initial commit

.gitignore

@@ -0,0 +1 @@
+.vercel

.vercelignore

@@ -0,0 +1,2 @@
+venv
+.idea

.vscode/launch.json

@@ -0,0 +1,17 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "args": ["2022-12-06"],
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": true
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "python.testing.pytestArgs": [
+        "venv"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
+}

anki.json

@@ -0,0 +1 @@
+{"date": "2022-07-26T13:55:19.519504", "cookies": {"ankiweb": "eyJrIjogIkZicDI2UlFhb20xZkF4d04iLCAiYyI6IDEsICJ0IjogMTY1ODgzMjkxOX0.j3cM2VCQQZ7HW0i6cA4gJ1ysKGFl2jmQoUbEQ2CAid0"}}

anki.py

@@ -0,0 +1,255 @@
+import abc
+import datetime
+import json
+import logging
+import re
+import sys
+import typing
+from abc import abstractmethod
+from concurrent.futures import ThreadPoolExecutor
+from typing import List
+
+import httpx
+import typer
+from typer import Typer, Option
+
+import sentry_sdk
+
+sentry_sdk.init(
+    "https://915a1bc84eb94d34b36cfdef13b3f3e5@o271257.ingest.sentry.io/6040098",
+    # Set traces_sample_rate to 1.0 to capture 100%
+    # of transactions for performance monitoring.
+    # We recommend adjusting this value in production.
+    traces_sample_rate=1.0,
+)
+
+logging.basicConfig(level=logging.DEBUG)
+
+
+class CookieStorage(abc.ABC):
+    @abstractmethod
+    def save_cookies(self, cookies: dict):
+        raise NotImplemented
+
+    @abstractmethod
+    def load_cookies(self) -> typing.Optional[dict]:
+        raise NotImplemented
+
+
+class FilesystemCookieStorage(CookieStorage):
+    def __init__(self, filename: str):
+        self.filename = filename
+
+    def save_cookies(self, cookies: dict):
+        with open(self.filename, "wt") as f:
+            json.dump({"date": datetime.datetime.now().isoformat(), "cookies": cookies}, f)
+
+    def load_cookies(self) -> typing.Optional[dict]:
+        try:
+            with open(self.filename) as f:
+                saved = json.load(f)
+        except FileNotFoundError:
+            return None
+        date_saved = datetime.datetime.fromisoformat(saved["date"])
+        if (datetime.datetime.now() - date_saved).days > 1:
+            return None
+        return saved["cookies"]
+
+
+class AnkiClient:
+    def __init__(self, cookie_storage: CookieStorage = None):
+        self._cookie_storage = cookie_storage or FilesystemCookieStorage("anki.json")
+        self._logger = logging.getLogger(self.__class__.__name__)
+        self._http = httpx.Client(
+            headers={
+                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4681.0 Safari/537.36",
+                "X-Requested-With": "XMLHttpRequest",
+            },
+            verify=False,
+            follow_redirects=False,
+        )
+
+    def _load_cookies(self):
+        cookies = self._cookie_storage.load_cookies()
+        if not cookies:
+            raise PermissionError("please login")
+        self._http.cookies = cookies
+
+    @property
+    def session(self) -> dict:
+        """
+        Returns session cookies
+
+        :return: session cookies
+        """
+        return self._cookie_storage.load_cookies()
+
+    def login(self, username: str, password: str) -> dict:
+        """
+        login and return cookies
+
+        :return: cookies
+        """
+        self._logger.debug("visiting login page")
+        res = self._http.get("https://ankiweb.net/account/login", follow_redirects=True)
+        res.raise_for_status()
+
+        self._logger.debug("submitting login form")
+        hidden_fields = dict(re.findall(r'.*type="hidden" name="([^"]+)" value="([^"]+)".*', res.text, re.MULTILINE))
+        res = self._http.post(
+            "https://ankiweb.net/account/login", data={**hidden_fields, "username": username, "password": password}, follow_redirects=True
+        )
+        res.raise_for_status()
+
+        cookies = {**self._http.cookies}
+        self._cookie_storage.save_cookies(cookies)
+        return cookies
+
+    def get_editor_context(self):
+        self._load_cookies()
+
+        add_info = self._http.get("https://ankiuser.net/edit/getAddInfo", follow_redirects=True).json()
+        note_types = {it["name"]: {"id": it["id"], "name": it["name"]} for it in add_info["notetypes"]}
+        decks = {it["name"]: {"id": it["id"], "name": it["name"]} for it in add_info["decks"]}
+
+        with ThreadPoolExecutor(max_workers=len(note_types)) as pool:
+
+            def fetch_note_fields(nid):
+                res = self._http.get(
+                    "https://ankiuser.net/edit/getNotetypeFields",
+                    params={"ntid": nid},
+                    follow_redirects=True,
+                )
+                if res.is_error:
+                    return []
+                fields_info = res.json()
+                return [f["name"] for f in fields_info["fields"]]
+
+            results = dict(
+                zip(note_types.keys(), pool.map(fetch_note_fields, [it["id"] for it in note_types.values()]))
+            )
+            for t in note_types:
+                note_types[t]["fields"] = results[t]
+
+        return note_types, decks
+
+    def create_note(self, note_type: str, deck: str, fields: dict, tags: str = ""):
+        self._load_cookies()
+
+        self._logger.debug("fetching note types")
+        add_info = self._http.get("https://ankiuser.net/edit/getAddInfo").json()
+        note_types = {it["name"]: it["id"] for it in add_info["notetypes"]}
+        decks = {it["name"]: it["id"] for it in add_info["decks"]}
+
+        self._logger.debug("fetching note fields")
+        fields_info = self._http.get(
+            "https://ankiuser.net/edit/getNotetypeFields", params={"ntid": note_types[note_type]}
+        ).json()
+        model_fields = [f["name"] for f in fields_info["fields"]]
+
+        editor_html = self._http.get("https://ankiuser.net/edit/").text
+        m = re.search(r"""new anki.Editor\('(ey.*)',.*\)""", editor_html)
+        csrf_token = m.group(1)
+
+        model_id = note_types[note_type]
+        self._logger.debug(f"found note type: {model_id}")
+        self._logger.debug(f"note fields: {model_fields}")
+
+        # unused fields must be an empty string, otherwise we get a 403
+        ordered_fields = [fields.get(f, "") for f in model_fields]
+        payload = {
+            "nid": "",
+            "data": json.dumps([ordered_fields, tags]),
+            "csrf_token": csrf_token,
+            "mid": model_id,
+            "deck": decks[deck],
+        }
+        self._logger.debug("saving note")
+        res = self._http.post("https://ankiuser.net/edit/save", data=payload)
+        res.raise_for_status()
+
+
+app = Typer(name="typer")
+state = {"session": "anki.json", "creds": "anki.creds.json"}
+
+
+@app.callback()
+def callback(session: str = "anki.json", creds: str = "anki.creds.json"):
+    state.update(session=session, creds=creds)
+
+
+def make_anki() -> AnkiClient:
+    anki = AnkiClient(FilesystemCookieStorage(state["session"]))
+    if not anki.session:
+        typer.echo("no valid session found, please login", err=True)
+        raise typer.Exit(1)
+    return anki
+
+
+def try_login():
+    try:
+        with open(state["creds"]) as f:
+            creds = json.load(f)
+            anki_login(username=creds["username"], password=creds["password"])
+    except FileNotFoundError:
+        pass
+
+
+@app.command("login")
+def anki_login(
+    username: str = Option(..., prompt=True),
+    password: str = Option(..., prompt=True),
+    force: bool = False,
+):
+    with open(state["creds"], "wt") as f:
+        json.dump({"username": username, "password": password}, f)
+    anki = AnkiClient(FilesystemCookieStorage(state["session"]))
+    if force or not anki.session:
+        anki.login(username=username, password=password)
+
+
+@app.command("info")
+def anki_info():
+    try_login()
+    anki = make_anki()
+    note_types, decks = anki.get_editor_context()
+    logging.info("note types:")
+    for _, t in note_types.items():
+        logging.info(t)
+    logging.info("decks:")
+    for d in decks.items():
+        logging.info(d)
+
+
+@app.command("note")
+def anki_create_note(
+    deck: str = Option(...),
+    type: str = Option(...),
+    fields: List[str] = Option(..., "-f", "--field", help="Fields in k=v format"),
+    tags: str = "",
+):
+    try_login()
+    anki = make_anki()
+    parsed_fields = dict([it.split("=", maxsplit=1) for it in fields])
+    anki.create_note(
+        deck=deck,
+        note_type=type,
+        fields=parsed_fields,
+        tags=tags,
+    )
+
+
+if __name__ == "__main__":
+    logging.debug(f"Called with {sys.argv=}")
+    # try_login()
+    # anki = make_anki()
+    # anki.create_note(
+    #     deck="Deutsch",
+    #     note_type="Card with Reverse and Examples",
+    #     fields={
+    #         "Front": "Es hat gerade an der Tür aufgeklingelt. Kannst du aufmachen?",
+    #         "Back": "The doorbell just rang. Can you get it?",
+    #         "Examples": "",
+    #     },
+    # )
+    app()

app.py

@@ -0,0 +1,84 @@
+import typing
+from typing import List, Dict
+
+from fastapi import FastAPI, Depends
+from pydantic import BaseModel
+from starlette.middleware.cors import CORSMiddleware
+from starlette.requests import Request
+from starlette.responses import Response, JSONResponse
+import httpx
+
+from anki import AnkiClient, CookieStorage
+
+app = FastAPI(
+    title='anki',
+    docs_url='/')
+app.add_middleware(CORSMiddleware, allow_origins=['*'], allow_methods=['*'], allow_headers=['*'])
+
+
+class SessionCookieStorage(CookieStorage):
+    def __init__(self, req: Request, res: Response):
+        self.res = res
+        self.req = req
+
+    def save_cookies(self, cookies: dict):
+        for k, v in cookies.items():
+            self.res.set_cookie(k, v, max_age=24 * 60 * 60)
+
+    def load_cookies(self) -> typing.Optional[dict]:
+        if 'ankiweb' not in self.req.cookies:
+            return None
+        return {**self.req.cookies}
+
+
+def get_anki_client(req: Request, res: Response) -> AnkiClient:
+    return AnkiClient(SessionCookieStorage(req, res))
+
+
+@app.post('/login', response_model=Dict[str, str])
+async def login(username: str, password: str, anki: AnkiClient = Depends(get_anki_client)):
+    anki.login(username, password)
+    return anki.session
+
+
+@app.get('/info', summary='Get note types and their fields', response_model=List[Dict])
+async def info(anki: AnkiClient = Depends(get_anki_client)):
+    return anki.get_editor_context()
+
+
+class CreateNote(BaseModel):
+    deck: str
+    note_type: str
+    note_fields: Dict[str, str]
+    tags: str = ''
+
+
+@app.post('/notes', summary='Create a note')
+async def create_note(input: CreateNote, anki: AnkiClient = Depends(get_anki_client)):
+    anki.create_note(note_type=input.note_type,
+                     deck=input.deck,
+                     fields=input.note_fields,
+                     tags=input.tags)
+
+
+@app.exception_handler(Exception)
+async def handle_errors(req: Request, error: Exception):
+    if isinstance(error, httpx.HTTPStatusError):
+        message = 'anki returned error'
+        status = error.response.status_code
+    elif isinstance(error, PermissionError):
+        message = str(error)
+        status = 401
+    else:
+        message = 'oops'
+        status = 500
+
+    return JSONResponse(content={
+        'message': message
+    }, status_code=status)
+
+
+if __name__ == '__main__':
+    import uvicorn
+
+    uvicorn.run(app, host='0.0.0.0')

aws_codecommit.py

@@ -0,0 +1,336 @@
+import itertools
+import json
+import logging
+import pprint
+import re
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor
+from urllib.parse import quote_plus
+
+import boto3
+import httpx
+
+import urllib.parse
+import subprocess
+import typing
+
+
+def list_repo_tags(*, repository_name: str, region: str, username: str, password: str) -> typing.List[dict]:
+    """
+    Fetches the list of tags in a Codecommit repo using git ls-remote command
+
+    :returns: List of {tag, commit} items
+    """
+    url = "https://{username}:{password}@git-codecommit.{region}.amazonaws.com/v1/repos/{repository_name}".format(
+        username=quote_plus(username),
+        password=quote_plus(password),  # passwords are base64-encoded and might contain "/"
+        region=region,
+        repository_name=repository_name,
+    )
+    result = subprocess.run(["git", "ls-remote", "--refs", "--tags", url], text=True, stdout=subprocess.PIPE)
+
+    tags = []
+    for line in result.stdout.splitlines(keepends=False):
+        # lines are formatted as:
+        # 070161940636bab5d934af2702f4248e5a13eb29	refs/tags/ui_sandbox_acc104
+        # and columns are delimited with "\t"
+        if "refs/tags/" in line:
+            commit_hash, tag_ref = line.split("\t", maxsplit=1)
+            tag = tag_ref.replace("refs/tags/", "")
+            tags.append({"commit": commit_hash, "tag": tag})
+
+    # natural-sort by tag name
+    # flora_prod_1 < ... < flora_prod_9 > flora_prod_10
+    # vs lexical sort would give:
+    # flora_prod_1 < flora_prod_10 < ... < flora_prod_9
+    tags = sorted(tags, key=lambda t: [int(it) if it.isdigit() else it for it in re.split(r"(\d+)", t["tag"])])
+
+    return tags
+
+
+def chunk(it, size):
+    it = iter(it)
+    sentinel = ()
+    return iter(lambda: tuple(itertools.islice(it, size)), sentinel)
+
+
+def summarize_tags(tags: list[dict], take: int = 10) -> list[dict]:
+    re_prefix = re.compile(r"^(\D*)")
+    re_digits = re.compile(r"(\d+)")
+    grouped = defaultdict(list)
+    for t in tags:
+        tag_name = t["tag"]
+        if m := re_prefix.search(tag_name):
+            prefix = m.group(1)
+            grouped[prefix].append(t)
+        else:
+            print("no match", m)
+    flattened = [*grouped.pop("", [])]
+    for g, items in grouped.items():
+        sorted_items = sorted(
+            items, key=lambda t: [int(it) if it.isdigit() else it for it in re_digits.split(t["tag"])], reverse=True
+        )
+        flattened.extend(sorted_items[:take])
+
+    return flattened
+
+
+abdus_creds = {
+    "username": "coder+1-at-400344683105",
+    "password": "4xdnGOtghKiT0M5CjulWf5raWwBSqT0Df2YkIY9Ws68=",
+}
+
+
+def get_commit_content():
+    client = boto3.client("codecommit")
+
+    res = client.get_file(
+        repositoryName="testing",
+        filePath="content.txt1",
+        commitSpecifier="vnext",
+    )
+    print(res)
+
+
+def main():
+    import logging
+    from botocore.endpoint import Endpoint
+
+    logging.basicConfig(level=logging.INFO)
+
+    raw_make_request = Endpoint.make_request
+
+    def intercepted_make_request(*args, **kwargs):
+        aws_response, parsed_response = raw_make_request(*args, **kwargs)
+        logging.info('AWS response: %s', parsed_response)
+        return aws_response, parsed_response
+
+    Endpoint.make_request = intercepted_make_request
+
+    session = boto3.Session(
+        aws_access_key_id="AKIAQKZTNEVLBHDG5Q3X",
+        aws_secret_access_key="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",
+        region_name="eu-central-1",
+    )
+    cc = session.client("rds")
+    _ = cc.describe_db_clusters()
+
+
+if __name__ == "__main__":
+    main()
+    exit()
+    # get_commit_content()
+    # flora b1cd47a221b84b2884b3d24198085a9f-f0dc2e747ff241649715c20d48232d8e
+    # oms 36363cc2f43b4cf5b1965af9f8925a4d-6deff091ce2d4c2483d8834c532a6f36
+    # omnitron 36363cc2f43b4cf5b1965af9f8925a4d-97400227a37940539c55489e1354ffc2
+    all_tags = list_repo_tags(
+        repository_name="b1cd47a221b84b2884b3d24198085a9f-f0dc2e747ff241649715c20d48232d8e",
+        username="sandbox-acc-api-user-at-023193265494",
+        password="sYPWKdhSx4JliXNI4ffRb4qWYBHLxA54nczSJ6CmYCY=",
+        region="eu-central-1",
+    )
+    print(all_tags)
+    exit(0)
+    all_tags.append({"tag": "1asd", "commit": "asdf"})
+    summarized = summarize_tags(all_tags, take=5)
+    commits = {t["commit"] for t in summarized}
+    session = boto3.Session(
+        aws_access_key_id="AKIAQKZTNEVLBHDG5Q3X",
+        aws_secret_access_key="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",
+        region_name="eu-central-1",
+    )
+    client = session.client("codecommit")
+
+    def get_commit_infos(commits: list[str]):
+        res = client.batch_get_commits(
+            repositoryName="36363cc2f43b4cf5b1965af9f8925a4d-97400227a37940539c55489e1354ffc2",
+            commitIds=commits,
+        )
+        return res["commits"]
+
+    with ThreadPoolExecutor() as pool:
+        results = list(pool.map(get_commit_infos, chunk(commits, 100)))
+
+    clone_ssh_url = repo["repositoryMetadata"]["cloneUrlSsh"]
+    print(repo)
+    exit(0)
+# exit()
+from botocore.awsrequest import AWSRequest
+from botocore.signers import RequestSigner
+
+# AWS Version 4 signing example
+
+# DynamoDB API (CreateTable)
+
+# See: http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
+# This version makes a POST request and passes request parameters
+# in the body (payload) of the request. Auth information is passed in
+# an Authorization header.
+import sys, os, base64, datetime, hashlib, hmac
+import requests  # pip install requests
+
+# ************* REQUEST VALUES *************
+method = "POST"
+service = "codecommit"
+host = "codecommit.eu-central-1.amazonaws.com"
+region = "eu-central-1"
+endpoint = "https://codecommit.eu-central-1.amazonaws.com/"
+# POST requests use a content type header. For DynamoDB,
+# the content is JSON.
+content_type = "application/x-amz-json-1.1"
+# DynamoDB requires an x-amz-target header that has this format:
+#     DynamoDB_<API version>.<operationName>
+amz_target = "CodeCommit_20150413.GetReferences"
+
+# Request parameters for CreateTable--passed in a JSON block.
+request_body = {"repositoryName": "36363cc2f43b4cf5b1965af9f8925a4d-6deff091ce2d4c2483d8834c532a6f36"}
+request_parameters = json.dumps(request_body)
+
+
+# Key derivation functions. See:
+# http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python
+def sign(key, msg):
+    return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
+
+
+def getSignatureKey(key, date_stamp, regionName, serviceName):
+    kDate = sign(("AWS4" + key).encode("utf-8"), date_stamp)
+    kRegion = sign(kDate, regionName)
+    kService = sign(kRegion, serviceName)
+    kSigning = sign(kService, "aws4_request")
+    return kSigning
+
+
+# Read AWS access key from env. variables or configuration file. Best practice is NOT
+# to embed credentials in code.
+access_key = os.environ.get("AWS_ACCESS_KEY_ID", "AKIAQKZTNEVLBHDG5Q3X")
+secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY", "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")
+if access_key is None or secret_key is None:
+    print("No access key is available.")
+    sys.exit()
+
+# Create a date for headers and the credential string
+t = datetime.datetime.utcnow()
+amz_date = t.strftime("%Y%m%dT%H%M%SZ")
+date_stamp = t.strftime("%Y%m%d")  # Date w/o time, used in credential scope
+
+# ************* TASK 1: CREATE A CANONICAL REQUEST *************
+# http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+
+# Step 1 is to define the verb (GET, POST, etc.)--already done.
+
+# Step 2: Create canonical URI--the part of the URI from domain to query
+# string (use '/' if no path)
+canonical_uri = "/"
+
+## Step 3: Create the canonical query string. In this example, request
+# parameters are passed in the body of the request and the query string
+# is blank.
+canonical_querystring = ""
+
+# Step 4: Create the canonical headers. Header names must be trimmed
+# and lowercase, and sorted in code point order from low to high.
+# Note that there is a trailing \n.
+canonical_headers = (
+    "content-type:"
+    + content_type
+    + "\n"
+    + "host:"
+    + host
+    + "\n"
+    + "x-amz-date:"
+    + amz_date
+    + "\n"
+    + "x-amz-target:"
+    + amz_target
+    + "\n"
+)
+
+# Step 5: Create the list of signed headers. This lists the headers
+# in the canonical_headers list, delimited with ";" and in alpha order.
+# Note: The request can include any headers; canonical_headers and
+# signed_headers include those that you want to be included in the
+# hash of the request. "Host" and "x-amz-date" are always required.
+# For DynamoDB, content-type and x-amz-target are also required.
+signed_headers = "content-type;host;x-amz-date;x-amz-target"
+
+# Step 6: Create payload hash. In this example, the payload (body of
+# the request) contains the request parameters.
+payload_hash = hashlib.sha256(request_parameters.encode("utf-8")).hexdigest()
+
+# Step 7: Combine elements to create canonical request
+canonical_request = (
+    method
+    + "\n"
+    + canonical_uri
+    + "\n"
+    + canonical_querystring
+    + "\n"
+    + canonical_headers
+    + "\n"
+    + signed_headers
+    + "\n"
+    + payload_hash
+)
+
+# ************* TASK 2: CREATE THE STRING TO SIGN*************
+# Match the algorithm to the hashing algorithm you use, either SHA-1 or
+# SHA-256 (recommended)
+algorithm = "AWS4-HMAC-SHA256"
+credential_scope = date_stamp + "/" + region + "/" + service + "/" + "aws4_request"
+string_to_sign = (
+    algorithm
+    + "\n"
+    + amz_date
+    + "\n"
+    + credential_scope
+    + "\n"
+    + hashlib.sha256(canonical_request.encode("utf-8")).hexdigest()
+)
+
+# ************* TASK 3: CALCULATE THE SIGNATURE *************
+# Create the signing key using the function defined above.
+signing_key = getSignatureKey(secret_key, date_stamp, region, service)
+
+# Sign the string_to_sign using the signing_key
+signature = hmac.new(signing_key, (string_to_sign).encode("utf-8"), hashlib.sha256).hexdigest()
+
+# ************* TASK 4: ADD SIGNING INFORMATION TO THE REQUEST *************
+# Put the signature information in a header named Authorization.
+authorization_header = (
+    algorithm
+    + " "
+    + "Credential="
+    + access_key
+    + "/"
+    + credential_scope
+    + ", "
+    + "SignedHeaders="
+    + signed_headers
+    + ", "
+    + "Signature="
+    + signature
+)
+
+# For DynamoDB, the request can include any headers, but MUST include "host", "x-amz-date",
+# "x-amz-target", "content-type", and "Authorization". Except for the authorization
+# header, the headers must be included in the canonical_headers and signed_headers values, as
+# noted earlier. Order here is not significant.
+# # Python note: The 'host' header is added automatically by the Python 'requests' library.
+headers = {
+    "User-Agent": "aws-sdk-js/2.627.0 promise",
+    "Content-Type": content_type,
+    "X-Amz-Date": amz_date,
+    "X-Amz-Target": amz_target,
+    "Authorization": authorization_header,
+}
+
+# ************* SEND THE REQUEST *************
+print("\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++")
+print("Request URL = " + endpoint)
+
+r = httpx.post(endpoint, json=request_body, headers=headers)
+
+print("\nRESPONSE++++++++++++++++++++++++++++++++++++")
+print("Response code: %d\n" % r.status_code)
+print(r.text)

aws_elasticache.py

@@ -0,0 +1,438 @@
+import dataclasses
+import functools
+import itertools
+import logging
+import re
+import time
+import typing
+import uuid
+from typing import Optional
+
+import boto3
+import botocore.exceptions
+from botocore.exceptions import ClientError
+
+logger = logging.getLogger(__name__)
+
+
+def retry(timeout: int, backoff_seconds: int = 10, bubble_errors: typing.List[typing.Type[Exception]] = None):
+    """
+    Tries calling a function {timeout} seconds until it succeeds or gives up and throw an error
+    :param timeout: Timeout in seconds
+    :param backoff_seconds: Wait time between retries
+    :param bubble_errors: List of exceptions to bubble up
+    :return: Wrapped function
+    """
+    bubble_errors = tuple(bubble_errors or [])
+
+    def wrapped(func: typing.Callable):
+        @functools.wraps(func)
+        def inner(*args, **kwargs):
+            waited = 0
+            while True:
+                try:
+                    logger.debug(f"Calling {func}")
+                    return func(*args, **kwargs)
+                except KeyboardInterrupt:
+                    raise
+                except tuple(bubble_errors):
+                    raise
+                except Exception as e:
+                    doze = min(timeout - waited, backoff_seconds)
+                    times = (timeout - waited) // backoff_seconds
+                    logger.debug(f"{func} failed, will try {times} times in {doze} sec")
+                    time.sleep(doze)
+                    waited += backoff_seconds
+                    if waited >= timeout:
+                        raise TimeoutError(f"Couldn't get a successful result from {func} in {timeout} seconds") from e
+
+        return inner
+
+    return wrapped
+
+
+@dataclasses.dataclass
+class RedisInstance:
+    cluster_id: str
+    host: str
+    port: int
+    arn: str
+    user_group_ids: typing.List[str] = dataclasses.field(default_factory=list)
+
+
+def paginate_by_marker(
+    func: callable,
+    list_field: str,
+) -> typing.Iterable:
+    """
+    Simplifies paginating over AWS results with a simpler interface than Paginators.
+    :param func: A function that accepts a parameter named `Marker`
+    :param list_field: Result field to iterate on
+    """
+    marker = ""
+    while True:
+        result = func(Marker=marker)
+
+        items = result.get(list_field, [])
+        yield from items
+
+        if "Marker" not in result:
+            break
+
+        marker = result["Marker"]
+
+
+@dataclasses.dataclass
+class RedisUser:
+    username: str
+    password: str
+    redis_key_prefix: str
+
+
+class CreateUserResult(typing.TypedDict):
+    user_group_id: str
+    users: typing.List[RedisUser]
+    status: str
+
+
+class RedisService:
+    def __init__(self, region: str):
+        self.elasticache = boto3.client("elasticache", region_name=region)
+
+    def create_redis_acl_user(
+        self,
+        replication_group_id: str,
+        users: typing.List[RedisUser],
+    ) -> None:
+        """
+        Creates a Redis ACL user. It will throw an error if the user group attached to the replication group is not ready.
+        This prevents us from quickly creating users. In that case, pass in a list of users instead.
+
+        It takes about 2 minutes until the user is ready.
+        """
+        repl_groups = self.elasticache.describe_replication_groups(ReplicationGroupId=replication_group_id)
+        user_group_id: str = repl_groups['ReplicationGroups'][0]['UserGroupIds'][0]
+        group = self.elasticache.describe_user_groups(UserGroupId=user_group_id)['UserGroups'][0]
+
+        """
+        Limits imposed by AWS:
+        
+        - User groups per replication group = 1
+        - Users per user group = 100
+        - Number of users = 1000
+        - Number of user groups = 100
+        https://docs.aws.amazon.com/AmazonElastiCache/latest/red-ug/Clusters.RBAC.html#Users-groups-to-RGs
+        
+        In short, we can't place more than 100 users on a single Redis instance.   
+        """
+
+        total_users = len(group['UserIds'])
+        if total_users >= 100:
+            raise Exception('Replication group reached the limit of 100 users')
+
+        self._update_or_create_user_group(user_group_id, users)
+
+    def delete_redis_acl_user(self, username: str) -> None:
+        logger.info(f'Deleting user {username}')
+        self.elasticache.delete_user(UserId=username)
+
+        # Even though we just deleted the user, it takes a couple of seconds until AWS updates user group status.
+        # So, we have to wait a bit before we can actually wait and check that the changes have propagated.
+        # Overall, it should take about 2-3 minutes to delete a user.
+
+    def create_redis_acl(
+        self,
+        *,
+        replication_group_id: str,
+        description: str = None,
+        node_type: str,
+        cache_subnet_group_name: str,
+        security_group_id: str,
+        tags: typing.Dict[str, str] = None,
+        users: typing.List[RedisUser],
+    ) -> dict:
+        user_group_id = f"{replication_group_id}-ug"
+        # we have to wait (~1m) until the user group is ready before we can create the ACL
+        self._update_or_create_user_group(user_group_id, users, wait=True)
+
+        cache_param_group = 'redis-acl-with-100-db'
+        self._ensure_cache_parameter_group(cache_param_group)
+
+        logger.info(f"Creating Redis replication group {replication_group_id=}")
+        default_replication_group_kwargs = dict(
+            ReplicasPerNodeGroup=0,
+            Engine="redis",
+            EngineVersion="6.x",
+            TransitEncryptionEnabled=True,  # must be True to use Redis ACL
+            MultiAZEnabled=False,
+            AutomaticFailoverEnabled=False,
+        )
+        repl_result = self.elasticache.create_replication_group(
+            **default_replication_group_kwargs,
+            ReplicationGroupId=replication_group_id,
+            ReplicationGroupDescription=description or f'{replication_group_id} replication group',
+            CacheNodeType=node_type,
+            CacheSubnetGroupName=cache_subnet_group_name,
+            CacheParameterGroupName=cache_param_group,
+            UserGroupIds=[
+                user_group_id,
+            ],
+            SecurityGroupIds=[
+                security_group_id,
+            ],
+            Tags=[{"Key": k, "Value": v} for k, v in (tags or {}).items()],
+        )
+
+        # replication group takes ~8m minutes to be ready
+
+        return {
+            "replication_group_id": repl_result["ReplicationGroup"]["ReplicationGroupId"],
+            'arn': repl_result["ReplicationGroup"]["ARN"],
+            'status': repl_result["ReplicationGroup"]["Status"],
+            'users': users,
+            '_response': repl_result,
+        }
+
+    def _update_or_create_user_group(
+        self, user_group_id: str, users: typing.List[RedisUser], wait: bool = False
+    ) -> None:
+        """
+        Creates or updates a user group with the given users. Waits until the user group is active.
+
+        :param user_group_id:
+        :param users:
+        """
+        # we have to add the `default` user for backwards compatibility
+        existing_users = self.elasticache.describe_users(
+            Filters=[{"Name": "user-id", "Values": [u.username for u in users]}],
+        )["Users"]
+        existing_user_ids = [it["UserId"] for it in existing_users]
+
+        created_user_ids = ["default"]
+        for user in users:
+            if user.username in existing_user_ids:
+                logger.info(f"User {user.username} is already exists")
+                created_user_ids.append(user.username)
+                continue
+
+            redis_acl = f"on +@all -@dangerous ~{user.redis_key_prefix}*"
+            logger.info(f"Creating tenant {user.username=}")
+            user_result = self.elasticache.create_user(
+                UserId=user.username,
+                UserName=user.username,
+                Passwords=[user.password],
+                AccessString=redis_acl,
+                Engine="redis",
+            )
+            created_user_ids.append(user_result["UserId"])
+
+        try:
+            group = self.elasticache.describe_user_groups(UserGroupId=user_group_id)["UserGroups"][0]
+            member_user_ids: typing.List[str] = group["UserIds"]
+            users_to_add = set(created_user_ids) - set(member_user_ids)
+
+            if users_to_add:
+                self.elasticache.modify_user_group(
+                    UserGroupId=user_group_id,
+                    UserIdsToAdd=list(users_to_add),
+                )
+                # this will take some time (~45s) until the changes propagate
+        except self.elasticache.exceptions.UserGroupNotFoundFault:
+            logger.info('User group does not exist, creating')
+            _ = self.elasticache.create_user_group(
+                UserGroupId=user_group_id,
+                Engine="redis",
+                UserIds=created_user_ids,
+            )
+            # user group creation takes around 60s
+            if wait:
+                self._wait_user_group(user_group_id)
+
+    @retry(timeout=60, backoff_seconds=5)
+    def _wait_user_group(self, user_group_id: str) -> None:
+        logger.info(f'Checking status of user group {user_group_id=}')
+        g = self.elasticache.describe_user_groups(UserGroupId=user_group_id)["UserGroups"][0]
+        assert g["Status"] == "active"
+
+    def is_redis_acl_ready(self, replication_group_id: str) -> bool:
+        repl = self.elasticache.describe_replication_groups(ReplicationGroupId=replication_group_id)[
+            'ReplicationGroups'
+        ][0]
+        is_redis_available = repl['Status'] == 'available'
+
+        user_group_id = repl['UserGroupIds'][0]
+        group = self.elasticache.describe_user_groups(UserGroupId=user_group_id)['UserGroups'][0]
+        is_group_available = group['Status'] == 'active'
+
+        return is_group_available and is_redis_available
+
+    def get_redis_acl(self, replication_group_id: str) -> dict:
+        redis = self.elasticache.describe_replication_groups(ReplicationGroupId=replication_group_id)[
+            'ReplicationGroups'
+        ][0]
+        return dict(
+            replication_group_id=redis['ReplicationGroupId'],
+            arn=redis['ARN'],
+            host=redis["NodeGroups"][0]["PrimaryEndpoint"]["Address"],
+            port=redis["NodeGroups"][0]["PrimaryEndpoint"]["Port"],
+        )
+
+    def delete_redis_acl(self, replication_group_id: str) -> None:
+        self.elasticache.delete_replication_group(ReplicationGroupId=replication_group_id)
+
+    def _ensure_cache_parameter_group(self, parameter_group_name: str) -> None:
+        try:
+            _ = self.elasticache.describe_cache_parameter_groups(CacheParameterGroupName=parameter_group_name)[
+                'CacheParameterGroups'
+            ][0]
+            return
+        except self.elasticache.exceptions.CacheParameterGroupNotFoundFault:
+            pass
+
+        logger.info(f"Creating cache parameter group {parameter_group_name=}")
+        _ = self.elasticache.create_cache_parameter_group(
+            CacheParameterGroupName=parameter_group_name,
+            CacheParameterGroupFamily="redis6.x",
+            Description="Redis parameter group for the Redis ACL",
+        )
+        _ = self.elasticache.modify_cache_parameter_group(
+            CacheParameterGroupName=parameter_group_name,
+            ParameterNameValues=[
+                {
+                    "ParameterName": "databases",
+                    "ParameterValue": "100",
+                },
+            ],
+        )
+
+
+def akinon_create_redis(
+    *,
+    k8s_cluster_name: str,
+    region: str,
+    node_type: str,
+    replicas: int,
+    owner_arn: str,
+    app_name: str,
+    role: str,
+):
+    subnet_group_name = f"{k8s_cluster_name}-redis-subg"
+    security_group_name = f"{k8s_cluster_name}-redis-sg"
+
+    # owner arn is formatted like:
+    # arn:aws:iam::412344683105:user/myusername
+    username = owner_arn.split("/")[1]
+    tags = {
+        "CostCenter": f"{username}-{app_name}-{role}-redis",
+    }
+
+    cache_cluster_id = uuid.uuid4().hex
+    r = RedisService(region)
+    return r.create_redis(
+        cluster_id=cache_cluster_id,
+        node_type=node_type,
+        replicas=replicas,
+        security_group_name=security_group_name,
+        subnet_group_name=subnet_group_name,
+        tags=tags,
+    )
+
+
+def akinon_create_redis_with_acl(
+    k8s_cluster_name: str,
+    owner_arn: str,
+    node_type: str,
+    app_name: str,
+    role: str,
+    users: typing.List[RedisUser],
+) -> RedisInstance:
+    cache_cluster_id = uuid.uuid4().hex
+    subnet_group_name = f"{k8s_cluster_name}-redis-subg"
+    security_group_name = f"{k8s_cluster_name}-redis-sg"
+
+    # owner arn is formatted as:
+    # arn:aws:iam::412344683105:user/myusername
+    username = owner_arn.split("/")[1]
+    tags = {
+        "CostCenter": f"{username}-{app_name}-{role}-redis",
+    }
+
+    return r.create_redis_acl(
+        cluster_id=cache_cluster_id,
+        node_type=node_type,
+        subnet_group_name=subnet_group_name,
+        security_group_name=security_group_name,
+        tags=tags,
+        users=users,
+    )
+
+
+def chunk(it, size):
+    it = iter(it)
+    sentinel = ()
+    return iter(lambda: tuple(itertools.islice(it, size)), sentinel)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format=f"%(asctime)s: {logging.BASIC_FORMAT}")
+    r = RedisService(region="eu-central-1")
+    # res = r.create_redis_acl(
+    #     replication_group_id="zerodev3",
+    #     description="zero redis dev",
+    #     node_type="cache.t3.micro",
+    #     cache_subnet_group_name="dev",
+    #     security_group_id="sg-c11a77ac",
+    #     tags={"CostCenter": "zero123"},
+    #     users=[RedisUser(username="u1", password="zeropassword1234", redis_key_prefix="zero:")],
+    # )
+    # print(res)
+
+    r.create_redis_acl_user(
+        replication_group_id='zerodev3',
+        users=[
+            RedisUser(
+                username='u2',
+                password='testtesttesttest123',
+                redis_key_prefix='test:',
+            )
+        ],
+    )
+    # r.delete_redis_acl_user('test6')
+    while True:
+        if r.is_redis_acl_ready('zerodev3'):
+            logger.info('ready')
+            print(r.get_redis_acl('zerodev3'))
+            break
+        logger.info('not ready')
+        time.sleep(10)
+    exit()
+
+    res = r.create_redis_acl(
+        replication_group_id="zerodev",
+        description="zero redis dev",
+        node_type="cache.t3.micro",
+        cache_subnet_group_name="dev",
+        security_group_id="sg-c11a77ac",
+        tags={"CostCenter": "zero123"},
+        users=[RedisUser(username="zerouser", password="zeropassword1234", redis_key_prefix="zero:")],
+    )
+    print(res)
+    #
+    # redis = r.get_redis_acl(cluster_arn="arn:aws:elasticache:eu-central-1:400344683105:replicationgroup:zero")
+    # r.add_redis_user(
+    #     cluster_id=redis.cluster_id,
+    #     users=[RedisUser(username="zero27", password="zero27password123", redis_key_prefix="zero27:")],
+    # )
+    # print(r.add_redis_user(cluster_id="zero1", users=[RedisUser("zero26", "zero26password123", "zero26:")]))
+    #
+    # print(
+    #     r.create_redis_with_acl(
+    #         cluster_id="zero",
+    #         cluster_description="zero redis",
+    #         node_type="cache.t3.micro",
+    #         subnet_group_name="testing-subnet",
+    #         security_group_name="default",
+    #         tags={"CostCenter": "zero123"},
+    #         users=[RedisUser(username="zerouser", password="zeropassword1234", redis_key_prefix="zero:")],
+    #     )
+    # )

aws_es.py

@@ -0,0 +1,238 @@
+import dataclasses
+import datetime
+import functools
+import json
+import logging
+import time
+import typing
+import uuid
+
+import boto3
+import botocore
+import requests
+from botocore.client import BaseClient
+from botocore.errorfactory import BaseClientExceptions
+from botocore.exceptions import ClientError
+
+logger = logging.getLogger(__name__)
+
+# fmt: off
+_response_describe_elasticsearch_domain_success = {'ResponseMetadata': {'RequestId': '38d7b555-2699-4868-9139-c47b1c2d70db', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '38d7b555-2699-4868-9139-c47b1c2d70db', 'content-type': 'application/json', 'content-length': '2314', 'date': 'Tue, 11 Jan 2022 07:52:24 GMT'}, 'RetryAttempts': 0}, 'DomainStatus': {'DomainId': '400344683105/es-d8a24850a0ae4b82af26', 'DomainName': 'es-d8a24850a0ae4b82af26', 'ARN': 'arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26', 'Created': True, 'Deleted': False, 'Endpoints': {'vpc': 'vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com'}, 'Processing': False, 'UpgradeProcessing': False, 'ElasticsearchVersion': '7.8', 'ElasticsearchClusterConfig': {'InstanceType': 't2.medium.elasticsearch', 'InstanceCount': 2, 'DedicatedMasterEnabled': False, 'ZoneAwarenessEnabled': True, 'ZoneAwarenessConfig': {'AvailabilityZoneCount': 2}, 'WarmEnabled': False, 'ColdStorageOptions': {'Enabled': False}}, 'EBSOptions': {'EBSEnabled': True, 'VolumeType': 'gp2', 'VolumeSize': 10}, 'AccessPolicies': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"es:*","Resource":"*"}]}', 'SnapshotOptions': {'AutomatedSnapshotStartHour': 0}, 'VPCOptions': {'VPCId': 'vpc-8c28b5e7', 'SubnetIds': ['subnet-004909d95a03c5fcd', 'subnet-b8f596f5'], 'AvailabilityZones': ['eu-central-1a', 'eu-central-1c'], 'SecurityGroupIds': ['sg-c11a77ac']}, 'CognitoOptions': {'Enabled': False}, 'EncryptionAtRestOptions': {'Enabled': False}, 'NodeToNodeEncryptionOptions': {'Enabled': False}, 'AdvancedOptions': {'override_main_response_version': 'false', 'rest.action.multi.allow_explicit_index': 'true'}, 'ServiceSoftwareOptions': {'CurrentVersion': 'R20211203-P2', 'NewVersion': '', 'UpdateAvailable': False, 'Cancellable': False, 'UpdateStatus': 'COMPLETED', 'Description': 'There is no software update available for this domain.', 'AutomatedUpdateDate': datetime.datetime(2021, 12, 14, 3, 38, 38), 'OptionalDeployment': False}, 'DomainEndpointOptions': {'EnforceHTTPS': True, 'TLSSecurityPolicy': 'Policy-Min-TLS-1-0-2019-07', 'CustomEndpointEnabled': False}, 'AdvancedSecurityOptions': {'Enabled': False, 'InternalUserDatabaseEnabled': False}, 'AutoTuneOptions': {'State': 'ENABLE_IN_PROGRESS'}}}
+_response_list_domain_names_success = {'ResponseMetadata': {'RequestId': '25923514-a426-4223-939a-4436f1d6e243', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '25923514-a426-4223-939a-4436f1d6e243', 'content-type': 'application/json', 'content-length': '87', 'date': 'Tue, 11 Jan 2022 07:53:37 GMT'}, 'RetryAttempts': 0}, 'DomainNames': [{'DomainName': 'es-d8a24850a0ae4b82af26', 'EngineType': 'Elasticsearch'}]}
+_response_describe_elasticsearch_domains_success = {'ResponseMetadata': {'RequestId': '61cfe5f4-42e7-4233-bca9-53f0ca62099d', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '61cfe5f4-42e7-4233-bca9-53f0ca62099d', 'content-type': 'application/json', 'content-length': '2320', 'date': 'Tue, 11 Jan 2022 07:55:25 GMT'}, 'RetryAttempts': 0}, 'DomainStatusList': [{'DomainId': '400344683105/es-d8a24850a0ae4b82af26', 'DomainName': 'es-d8a24850a0ae4b82af26', 'ARN': 'arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26', 'Created': True, 'Deleted': False, 'Endpoints': {'vpc': 'vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com'}, 'Processing': False, 'UpgradeProcessing': False, 'ElasticsearchVersion': '7.8', 'ElasticsearchClusterConfig': {'InstanceType': 't2.medium.elasticsearch', 'InstanceCount': 2, 'DedicatedMasterEnabled': False, 'ZoneAwarenessEnabled': True, 'ZoneAwarenessConfig': {'AvailabilityZoneCount': 2}, 'WarmEnabled': False, 'ColdStorageOptions': {'Enabled': False}}, 'EBSOptions': {'EBSEnabled': True, 'VolumeType': 'gp2', 'VolumeSize': 10}, 'AccessPolicies': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"es:*","Resource":"*"}]}', 'SnapshotOptions': {'AutomatedSnapshotStartHour': 0}, 'VPCOptions': {'VPCId': 'vpc-8c28b5e7', 'SubnetIds': ['subnet-004909d95a03c5fcd', 'subnet-b8f596f5'], 'AvailabilityZones': ['eu-central-1a', 'eu-central-1c'], 'SecurityGroupIds': ['sg-c11a77ac']}, 'CognitoOptions': {'Enabled': False}, 'EncryptionAtRestOptions': {'Enabled': False}, 'NodeToNodeEncryptionOptions': {'Enabled': False}, 'AdvancedOptions': {'override_main_response_version': 'false', 'rest.action.multi.allow_explicit_index': 'true'}, 'ServiceSoftwareOptions': {'CurrentVersion': 'R20211203-P2', 'NewVersion': '', 'UpdateAvailable': False, 'Cancellable': False, 'UpdateStatus': 'COMPLETED', 'Description': 'There is no software update available for this domain.', 'AutomatedUpdateDate': datetime.datetime(2021, 12, 14, 3, 38, 38), 'OptionalDeployment': False}, 'DomainEndpointOptions': {'EnforceHTTPS': True, 'TLSSecurityPolicy': 'Policy-Min-TLS-1-0-2019-07', 'CustomEndpointEnabled': False}, 'AdvancedSecurityOptions': {'Enabled': False, 'InternalUserDatabaseEnabled': False}, 'AutoTuneOptions': {'State': 'ENABLE_IN_PROGRESS'}}]}
+_response_delete_elasticsearch_domain_not_found = ClientError({'Error': {'Message': 'Domain not found: asd', 'Code': 'ResourceNotFoundException'}, 'ResponseMetadata': {'RequestId': 'd0669560-77f6-424c-bbc6-fb5878daa494', 'HTTPStatusCode': 409, 'HTTPHeaders': {'x-amzn-requestid': 'd0669560-77f6-424c-bbc6-fb5878daa494', 'x-amzn-errortype': 'ResourceNotFoundException', 'content-type': 'application/json', 'content-length': '35', 'date': 'Tue, 11 Jan 2022 08:06:07 GMT'}, 'RetryAttempts': 0}}, 'DeleteElasticsearchDomain')
+_response_delete_elasticsearch_domain_success = {'ResponseMetadata': {'RequestId': '1e9046f4-c4ee-4fda-8dfe-dd23f0161b7e', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '1e9046f4-c4ee-4fda-8dfe-dd23f0161b7e', 'content-type': 'application/json', 'content-length': '2312', 'date': 'Tue, 11 Jan 2022 08:31:33 GMT'}, 'RetryAttempts': 0}, 'DomainStatus': {'DomainId': '400344683105/es-d8a24850a0ae4b82af26', 'DomainName': 'es-d8a24850a0ae4b82af26', 'ARN': 'arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26', 'Created': True, 'Deleted': True, 'Endpoints': {'vpc': 'vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com'}, 'Processing': True, 'UpgradeProcessing': False, 'ElasticsearchVersion': '7.8', 'ElasticsearchClusterConfig': {'InstanceType': 't2.medium.elasticsearch', 'InstanceCount': 2, 'DedicatedMasterEnabled': False, 'ZoneAwarenessEnabled': True, 'ZoneAwarenessConfig': {'AvailabilityZoneCount': 2}, 'WarmEnabled': False, 'ColdStorageOptions': {'Enabled': False}}, 'EBSOptions': {'EBSEnabled': True, 'VolumeType': 'gp2', 'VolumeSize': 10}, 'AccessPolicies': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"es:*","Resource":"*"}]}', 'SnapshotOptions': {'AutomatedSnapshotStartHour': 0}, 'VPCOptions': {'VPCId': 'vpc-8c28b5e7', 'SubnetIds': ['subnet-004909d95a03c5fcd', 'subnet-b8f596f5'], 'AvailabilityZones': ['eu-central-1a', 'eu-central-1c'], 'SecurityGroupIds': ['sg-c11a77ac']}, 'CognitoOptions': {'Enabled': False}, 'EncryptionAtRestOptions': {'Enabled': False}, 'NodeToNodeEncryptionOptions': {'Enabled': False}, 'AdvancedOptions': {'override_main_response_version': 'false', 'rest.action.multi.allow_explicit_index': 'true'}, 'ServiceSoftwareOptions': {'CurrentVersion': 'R20211203-P2', 'NewVersion': '', 'UpdateAvailable': False, 'Cancellable': False, 'UpdateStatus': 'COMPLETED', 'Description': 'There is no software update available for this domain.', 'AutomatedUpdateDate': datetime.datetime(2021, 12, 14, 3, 38, 38), 'OptionalDeployment': False}, 'DomainEndpointOptions': {'EnforceHTTPS': True, 'TLSSecurityPolicy': 'Policy-Min-TLS-1-0-2019-07', 'CustomEndpointEnabled': False}, 'AdvancedSecurityOptions': {'Enabled': False, 'InternalUserDatabaseEnabled': False}, 'AutoTuneOptions': {'State': 'ENABLE_IN_PROGRESS'}}}
+# fmt: on
+
+
+def retry(timeout: int, backoff_seconds: int = 10, bubble_errors: typing.List[typing.Type[Exception]] = None):
+    """
+    Tries calling a function {timeout} seconds until it succeeds or gives up and throw an error
+    :param timeout: Timeout in seconds
+    :param backoff_seconds: Wait time between retries
+    :param bubble_errors: List of exceptions to bubble up
+    :return: Wrapped function
+    """
+    bubble_errors = tuple(bubble_errors or [])
+
+    def wrapped(func: typing.Callable):
+        @functools.wraps(func)
+        def func_retrier(*args, **kwargs):
+            waited = 0
+            started_at = datetime.datetime.now()
+            while True:
+                try:
+                    logger.debug(f"Calling {func}")
+                    result = func(*args, **kwargs)
+                    logger.debug(f"{func} completed in {(datetime.datetime.now() - started_at)}")
+                    return result
+                except KeyboardInterrupt:
+                    raise
+                except tuple(bubble_errors):
+                    raise
+                except Exception as e:
+                    doze = min(timeout - waited, backoff_seconds)
+                    times = (timeout - waited) // backoff_seconds
+                    logger.debug(f"{func} failed, will try {times} times in {doze} sec", exc_info=True)
+                    time.sleep(doze)
+                    waited += backoff_seconds
+                    if waited >= timeout:
+                        raise TimeoutError(f"Couldn't get a successful result from {func} in {timeout} seconds") from e
+
+        return func_retrier
+
+    return wrapped
+
+
+@dataclasses.dataclass
+class ElasticsearchInstance:
+    domain_name: str
+    endpoint: str
+    arn: str
+
+
+class ElasticsearchService:
+    domain_name_format = "es-{}"
+    default_version = "7.8"
+    valid_versions = {"7.10", "7.9", "7.8", "7.7", "7.4", "7.1", "6.8", "6.7", "6.5", "6.4", "6.3", "6.2", "6.0", "5.6", "5.5"}  # fmt: skip
+
+    def __init__(self, region: str):
+        self.es = boto3.client("es", region_name=region)
+
+    def create_instance(
+        self,
+        node_type: str,
+        total_nodes: int,
+        subnet_group_ids: typing.List[str],
+        security_group_id: str,
+        version: str = None,
+        tags: dict = None,
+        volume_size_gb: int = 30,
+        total_availability_zones: int = 3,
+    ) -> ElasticsearchInstance:
+        if not version:
+            version = self.default_version
+        assert version in self.valid_versions, "Invalid version"
+
+        domain_name = self.domain_name_format.format(uuid.uuid4().hex[:20])
+        logger.info(
+            "Creating a new Elasticsearch domain",
+            extra=dict(version=version, domain_name=domain_name, node_type=node_type, total_nodes=total_nodes),
+        )
+        es_result = self.es.create_elasticsearch_domain(
+            DomainName=domain_name,
+            ElasticsearchVersion=version,
+            ElasticsearchClusterConfig={
+                "InstanceType": node_type,
+                "InstanceCount": total_nodes,
+                "ZoneAwarenessEnabled": True,
+                "ZoneAwarenessConfig": {
+                    "AvailabilityZoneCount": total_availability_zones,
+                },
+            },
+            VPCOptions={
+                "SubnetIds": subnet_group_ids,
+                "SecurityGroupIds": [security_group_id],
+            },
+            EBSOptions={
+                "EBSEnabled": True,
+                "VolumeType": "gp2",
+                "VolumeSize": volume_size_gb,
+            },
+            AccessPolicies=json.dumps(
+                {
+                    "Version": "2012-10-17",
+                    "Statement": [{"Effect": "Allow", "Principal": "*", "Action": "es:*", "Resource": "*"}],
+                }
+            ),
+            DomainEndpointOptions={"EnforceHTTPS": True},
+            TagList=[{"Key": k, "Value": v} for k, v in (tags or {}).items()],
+        )
+        logger.info("Waiting until ES domain becomes available. This will take a while (~15 min)")
+        return self._find_instance(domain_name)
+
+    @retry(timeout=20 * 60, backoff_seconds=30)
+    def _find_instance(self, domain_name: str) -> ElasticsearchInstance:
+        result = self.es.describe_elasticsearch_domain(DomainName=domain_name)["DomainStatus"]
+        return ElasticsearchInstance(endpoint=result["Endpoints"]["vpc"], arn=result["ARN"], domain_name=domain_name)
+
+    def find_domain_name_by_arn(self, arn: str):
+        res = self.es.list_domain_names(EngineType='Elasticsearch')
+        domain_names = [it['DomainName'] for it in res['DomainNames']]
+
+        res = self.es.describe_elasticsearch_domains(DomainNames=domain_names)
+        domain_name_by_arn = {it['ARN']: it['DomainName'] for it in res['DomainStatusList']}
+
+        print(res)
+
+
+def main():
+    username, password = 'bgNNs4EWPTsuQJ24', '3asinFjikT5MgwRz!'
+
+
+"""
+GET _search
+{
+  "query": {
+    "match_all": {}
+  }
+}
+
+###
+PUT /my-index
+PUT /my-index2
+PUT /my-index3
+PUT /my-index3-4
+
+###
+DELETE /my-index*?expand_wildcards=all
+
+###
+DELETE /my-index1
+
+###
+GET /my-index
+GET /_stats/indexing
+###
+GET /_cat/indices?v
+
+###
+
+GET /kibana_sample_data_ecommerce/
+
+###
+GET /_all/_mapping
+
+## delete multiple
+PUT /my-index
+PUT /my-index2
+DELETE /my-index,my-index2
+GET /my-index
+
+###
+GET /_aliases
+
+###
+GET /_stats/
+
+
+user: qdSmWKS8p3VrUyBC
+pwd: un6f2vTrRjoa8orn!
+"""
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format=f"%(asctime)s: {logging.BASIC_FORMAT}")
+    logging.getLogger("botocore").setLevel(logging.INFO)
+
+    # ElasticsearchService('eu-central-1')
+    e = ElasticsearchService("eu-central-1")
+
+    esi = ElasticsearchInstance(
+        endpoint='vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com',
+        arn='arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26',
+        domain_name='es-d8a24850a0ae4b82af26',
+    )
+
+    # e.find_instance_by_arn(esi.arn)
+    # exit()
+    try:
+        es = boto3.client('es')
+        res = es.delete_elasticsearch_domain(DomainName=esi.domain_name)
+        print(res)
+    except ClientError as e:
+        error_code = e.__class__.__name__
+        print(error_code)
+
+    e = ElasticsearchService("eu-central-1")
+    # print(
+    #     e.create_instance(
+    #         node_type="t2.medium.elasticsearch",
+    #         total_nodes=2,
+    #         volume_size_gb=10,
+    #         total_availability_zones=2,
+    #         security_group_id="sg-c11a77ac",
+    #         subnet_group_ids=["subnet-b8f596f5", "subnet-004909d95a03c5fcd"],
+    #         tags={"CostCenter": "my"},
+    #     )
+    # )

aws_rds.py

@@ -0,0 +1,401 @@
+import dataclasses
+import datetime
+import functools
+import logging
+import time
+import typing
+import uuid
+
+import boto3
+
+# import mysql.connector.connection
+import psycopg2
+import pymysql
+from psycopg2 import sql
+import psycopg2.extensions
+
+logger = logging.getLogger(__name__)
+
+
+@dataclasses.dataclass
+class Credentials:
+    username: str
+    password: str
+
+    @classmethod
+    def new(cls):
+        username_format = "u{}"  # must start with a letter
+        return cls(
+            username=username_format.format(uuid.uuid4().hex),
+            password=uuid.uuid4().hex,
+        )  # make sure it starts with a letter
+
+
+@dataclasses.dataclass
+class DbConnection:
+    endpoint: str
+    port: int
+
+
+@dataclasses.dataclass
+class DbInstance(DbConnection):
+    cluster_id: str
+    arn: str
+    reader_endpoint: typing.Optional[str] = None
+    database_name: typing.Optional[str] = None
+    user: typing.Optional[Credentials] = None
+    master_user: typing.Optional[Credentials] = None
+
+
+def retry(timeout: int, backoff_seconds: int = 10, bubble_errors: typing.List[typing.Type[Exception]] = None):
+    """
+    Tries calling a function {timeout} seconds until it succeeds or gives up and throw an error
+    :param timeout: Timeout in seconds
+    :param backoff_seconds: Wait time between retries
+    :param bubble_errors: List of exceptions to bubble up
+    :return: Wrapped function
+    """
+    bubble_errors = tuple(bubble_errors or [])
+
+    def wrapped(func: typing.Callable):
+        @functools.wraps(func)
+        def func_retrier(*args, **kwargs):
+            waited = 0
+            started_at = datetime.datetime.now()
+            while True:
+                try:
+                    logger.debug(f"Calling {func}")
+                    result = func(*args, **kwargs)
+                    logger.debug(f"{func} completed in {(datetime.datetime.now() - started_at)}")
+                    return result
+                except KeyboardInterrupt:
+                    raise
+                except tuple(bubble_errors):
+                    raise
+                except Exception as e:
+                    doze = min(timeout - waited, backoff_seconds)
+                    times = (timeout - waited) // backoff_seconds
+                    logger.debug(f"{func} failed, will try {times} times in {doze} sec", exc_info=True)
+                    time.sleep(doze)
+                    waited += backoff_seconds
+                    if waited >= timeout:
+                        raise TimeoutError(f"Couldn't get a successful result from {func} in {timeout} seconds") from e
+
+        return func_retrier
+
+    return wrapped
+
+
+def paginate_by_marker(
+    func: callable,
+    list_field: str,
+) -> typing.Iterable:
+    """
+    Simplifies paginating over AWS results with a simpler interface than Paginators.
+    :param func: A function that accepts a parameter named `Marker`
+    :param list_field: Result field to iterate on
+    """
+    marker = ""
+    while True:
+        result = func(Marker=marker)
+
+        items = result.get(list_field, [])
+        yield from items
+
+        if "Marker" not in result:
+            break
+
+        marker = result["Marker"]
+
+
+class PostgresqlService:
+    db_type = "Postgresql"
+    db_engine = "aurora-postgresql"
+    db_engine_version = "10.11"
+    default_master_database_name = "postgres"
+    cluster_id_format = "pg-{}"  # must start with a letter
+    tenant_database_name_format = "db{}"  # must start with a letter
+
+    def __init__(self, region: str):
+        self.rds = boto3.client("rds", region_name=region)
+
+    def create_db(
+        self,
+        total_instances: int,
+        db_subnet_group_name: str,
+        node_type: str,
+        security_group_id: str,
+        engine_version: str = None,
+        backup_retention_days: int = 30,
+        public: bool = False,
+        tags: dict = None,
+    ):
+        if not engine_version:
+            engine_version = self.db_engine_version
+
+        cluster_id = self.cluster_id_format.format(uuid.uuid4().hex)
+
+        master_creds = Credentials.new()
+
+        logger.info(
+            f"Creating {self.db_type} cluster",
+            extra=dict(
+                cluster_id=cluster_id,
+                engine_version=engine_version,
+                db_subnet_group_name=db_subnet_group_name,
+                security_group_id=security_group_id,
+            ),
+        )
+
+        db_result = self.rds.create_db_cluster(
+            DBClusterIdentifier=cluster_id,
+            Engine=self.db_engine,
+            EngineVersion=engine_version,
+            MasterUsername=master_creds.username,
+            MasterUserPassword=master_creds.password,
+            Tags=[{"Key": k, "Value": v} for k, v in (tags or {}).items()],
+            DBSubnetGroupName=db_subnet_group_name,
+            VpcSecurityGroupIds=[security_group_id],
+            BackupRetentionPeriod=backup_retention_days,
+        )["DBCluster"]
+
+        endpoint = db_result["Endpoint"]
+        reader_endpoint = db_result["ReaderEndpoint"]
+        port = db_result["Port"]
+        arn = db_result["DBClusterArn"]
+
+        for i in range(total_instances):
+            instance_id = f"{cluster_id}-Instance-{i}"
+            logger.info(
+                "Creating instances on the cluster",
+                extra=dict(
+                    cluster_id=cluster_id,
+                    instance_id=instance_id,
+                    node_type=node_type,
+                ),
+            )
+            instance_result = self.rds.create_db_instance(
+                DBInstanceIdentifier=instance_id,
+                DBClusterIdentifier=cluster_id,
+                DBInstanceClass=node_type,
+                Engine=self.db_engine,
+                PubliclyAccessible=public,
+                Tags=[{"Key": k, "Value": v} for k, v in (tags or {}).items()],
+            )
+
+        tenant_database_name = self.tenant_database_name_format.format(uuid.uuid4().hex)
+        db = DbInstance(
+            cluster_id=cluster_id,
+            arn=arn,
+            endpoint=endpoint,
+            reader_endpoint=reader_endpoint,
+            port=port,
+            database_name=tenant_database_name,
+            master_user=master_creds,
+        )
+        print(db)  # TODO: remove
+
+        logger.info("Waiting until DB instances are online. This will take a while (~5 min)")
+        logger.info("Connecting DB instance using master credentials", extra=dict(endpoint=db.endpoint))
+        self._check_connection(
+            connection=db,
+            credentials=master_creds,
+        )
+
+        tenant_creds = Credentials.new()
+        logger.info(
+            "Connection successful. Creating a new database and user",
+            extra=dict(
+                endpoint=endpoint,
+                username=tenant_creds.username,
+                database_name=tenant_database_name,
+            ),
+        )
+        self.create_tenant(
+            connection=db,
+            master_user=master_creds,
+            database_name=tenant_database_name,
+            tenant_user=tenant_creds,
+        )
+        db.user = tenant_creds
+
+        return db
+
+    @retry(timeout=10 * 60)
+    def _check_connection(self, connection: DbConnection, credentials: Credentials, database_name: str = "postgres"):
+        psycopg2.connect(
+            host=connection.endpoint,
+            port=connection.port,
+            dbname=database_name,
+            user=credentials.username,
+            password=credentials.password,
+            connect_timeout=15,
+        ).close()
+
+    def _find_instance(self, endpoint: str) -> typing.Optional[DbInstance]:
+        func = functools.partial(
+            self.rds.describe_db_clusters, Filters=[{"Name": "engine", "Values": [self.db_engine]}], MaxRecords=100
+        )
+        cluster = None
+        for it in paginate_by_marker(func, "DBClusters"):
+            if it["Endpoint"] == endpoint:
+                cluster = it
+                break
+        if cluster:
+            return DbInstance(
+                endpoint=cluster["Endpoint"],
+                port=cluster["Port"],
+                cluster_id=cluster["DBClusterIdentifier"],
+                arn=cluster["DBClusterArn"],
+                reader_endpoint=cluster["ReaderEndpoint"],
+            )
+        return None
+
+    def create_tenant(
+        self,
+        connection: DbConnection,
+        master_user: Credentials,
+        tenant_user: Credentials,
+        database_name: typing.Optional[str] = None,
+    ) -> DbInstance:
+        if not database_name:
+            database_name = self.tenant_database_name_format.format(uuid.uuid4().hex)
+        try:
+            con = psycopg2.connect(
+                host=connection.endpoint,
+                port=connection.port,
+                dbname=self.default_master_database_name,
+                user=master_user.username,
+                password=master_user.password,
+                connect_timeout=30,
+            )
+            con.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+            cur = con.cursor()
+
+            cur.execute(
+                sql.SQL("CREATE DATABASE {}").format(
+                    sql.Identifier(database_name),
+                ),
+            )
+            cur.execute(
+                sql.SQL("CREATE USER {} WITH ENCRYPTED PASSWORD {}").format(
+                    sql.Identifier(tenant_user.username),
+                    sql.Placeholder(),
+                ),
+                [tenant_user.password],
+            )
+            cur.execute(
+                sql.SQL("GRANT ALL PRIVILEGES ON DATABASE {} TO {}").format(
+                    sql.Identifier(database_name),
+                    sql.Identifier(tenant_user.username),
+                )
+            )
+
+            # `with` block would create an implicit transaction
+            con.close()
+
+            db = self._find_instance(connection.endpoint)
+            db.database_name = database_name
+            db.user = tenant_user
+            db.master_user = master_user
+            return db
+        except psycopg2.Error:
+            logging.error(
+                f"Failed to create database and user",
+                exc_info=True,
+                extra=dict(endpoint=connection.endpoint, database_name=database_name),
+            )
+            raise
+
+
+class MysqlService(PostgresqlService):
+    db_type = "MySQL"
+    db_engine = "aurora-mysql"
+    db_engine_version = "5.7.12"
+    default_master_database_name = "mysql"
+    cluster_id_format = "mysql-{}"  # must start with a letter
+
+    @retry(timeout=10 * 60)
+    def _check_connection(self, connection: DbConnection, credentials: Credentials, database_name: str = "postgres"):
+        pymysql.connect(
+            host=connection.endpoint,
+            port=connection.port,
+            user=credentials.username,
+            password=credentials.password,
+            connect_timeout=15,
+        ).close()
+
+    def create_tenant(
+        self,
+        connection: DbConnection,
+        master_user: Credentials,
+        tenant_user: Credentials,
+        database_name: typing.Optional[str] = None,
+    ) -> DbInstance:
+        if not database_name:
+            database_name = self.tenant_database_name_format.format(uuid.uuid4().hex)
+
+        try:
+            with pymysql.connect(
+                host=connection.endpoint,
+                port=connection.port,
+                user=master_user.username,
+                password=master_user.password,
+                connect_timeout=15,
+            ) as con:
+                cur = con.cursor()
+                cur.execute(f"CREATE DATABASE {database_name}")
+                cur.execute(f"CREATE USER {tenant_user.username} IDENTIFIED BY '{tenant_user.password}'")
+                cur.execute(
+                    f"GRANT ALL PRIVILEGES ON {database_name}.* TO {tenant_user.username}@'%' IDENTIFIED BY '{tenant_user.password}'"
+                )
+
+                db = self._find_instance(endpoint=connection.endpoint)
+                db.database_name = database_name
+                db.master_user = master_user
+                db.user = tenant_user
+                return db
+        except pymysql.Error:
+            logger.error(
+                "Failed to create database and user",
+                exc_info=True,
+                extra=dict(endpoint=connection.endpoint, database_name=database_name),
+            )
+            raise
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format=f"%(asctime)s: {logging.BASIC_FORMAT}")
+    logging.getLogger("botocore").setLevel(logging.INFO)
+
+    # m = MysqlService("eu-central-1")
+    # db = m.create_db(
+    #     total_instances=1,
+    #     db_subnet_group_name="mydbsubnet",
+    #     security_group_id="sg-c11a77ac",
+    #     node_type="db.t3.medium",
+    #     tags={"CostCenter": "hello"},
+    #     public=True,
+    # )
+    # print(db)
+
+    # p = PostgresqlService("eu-central-1")
+    # db = p.create_db(
+    #     total_instances=1,
+    #     db_subnet_group_name="mydbsubnet",
+    #     security_group_id="sg-c11a77ac",
+    #     node_type="db.t3.medium",
+    #     tags={"CostCenter": "hello"},
+    #     public=True,
+    # )
+    # p.create_tenant(
+    #     connection=DbConnection(
+    #         endpoint="pg0a085614d5eb4f34affd2a24b1fb18c4.cluster-c8v5rp0ouaey.eu-central-1.rds.amazonaws.com",
+    #         port=5432,
+    #     ),
+    #     master_user=Credentials(
+    #         username="u7d0f9621f5a5419288a4ef7ce9fd5fe", password="ca9473879b794dd3a0b0f0d2a8d4aebb"
+    #     ),
+    #     tenant_user=Credentials.new(),
+    #     database_name="mydb",
+    # )

aws_redis_acl.py

@@ -0,0 +1,107 @@
+import logging
+import typing
+
+import boto3
+import botocore.client
+
+
+def paginate_by_marker(
+    func: callable,
+    list_field: str,
+) -> typing.Iterable:
+    """
+    Simplifies paginating over AWS results with a simpler interface than Paginators.
+    :param func: A function that accepts a parameter named `Marker`
+    :param list_field: Result field to iterate on
+    """
+    marker = ""
+    while True:
+        result = func(Marker=marker)
+
+        items = result.get(list_field, [])
+        yield from items
+
+        if "Marker" not in result:
+            break
+
+        marker = result["Marker"]
+
+
+logger = logging.getLogger(__name__)
+
+
+class Rediser:
+    @classmethod
+    def new(cls):
+        return cls(boto3.client("elasticache", region_name='eu-central-1'))
+
+    def __init__(self, elasticache: botocore.client.BaseClient):
+        self.elasticache = elasticache
+
+    def _get_redis_acl(
+        self,
+        cluster_id: str = None,
+        status: str = "available",
+    ) -> typing.Optional[dict]:
+        """
+        Finds a Redis replication group by its ID
+
+        :param cluster_id: Replication group ID
+        :param status: Replication status. One of available/starting/modifying/deleting.
+        :return: RedisInstance or throws LookupError error
+        """
+        try:
+            redis = self.elasticache.describe_replication_groups(ReplicationGroupId=cluster_id)["ReplicationGroups"][0]
+            if redis["Status"] != status:
+                raise Exception(f"Found redis with {cluster_id=} but its status={redis['Status']}")
+        except self.elasticache.exceptions.ReplicationGroupNotFoundFault:
+            return None
+
+        return dict(
+            cluster_id=redis["ReplicationGroupId"],
+            host=redis["NodeGroups"][0]["PrimaryEndpoint"]["Address"],
+            port=redis["NodeGroups"][0]["PrimaryEndpoint"]["Port"],
+            arn=redis["ARN"],
+            user_group_ids=redis["UserGroupIds"],
+        )
+
+    def create_redis_acl(
+        self,
+        *,
+        cluster_id: str,
+        cluster_description: typing.Optional[str] = None,
+        node_type: str,
+        subnet_group_name: str,
+        security_group_id: str,
+    ):
+        redis = self._get_redis_acl(cluster_id=cluster_id)
+
+        if not redis:
+            res = self.elasticache.create_replication_group(
+                ReplicationGroupId=cluster_id,
+                ReplicationGroupDescription=cluster_description or cluster_id,
+                ReplicasPerNodeGroup=0,
+                Engine="redis",
+                EngineVersion="6.x",
+                CacheNodeType=node_type,
+                CacheSubnetGroupName=subnet_group_name,
+                TransitEncryptionEnabled=True,  # must be True to use Redis ACL
+                UserGroupIds=[
+                    user_group_id,
+                ],
+                SecurityGroupIds=[
+                    security_group_id,
+                ],
+                Tags=[{"Key": k, "Value": v} for k, v in tags.items()],
+                MultiAZEnabled=False,
+                AutomaticFailoverEnabled=False,
+            )
+
+
+def main():
+    r = Rediser.new()
+    res = r.create_redis_acl(cluster_id='hey', node_type='cache.t2.micro')
+
+
+if __name__ == '__main__':
+    main()

aws_ses.py

@@ -0,0 +1,202 @@
+import concurrent.futures
+import functools
+import logging
+import pprint
+import time
+import typing
+
+import boto3
+import botocore.client
+
+DnsRecord = typing.TypedDict(
+    "DnsRecord",
+    {
+        "purpose": str,
+        "type": str,
+        "name": str,
+        "value": str,
+        "priority": typing.Optional[str],
+    },
+)
+VerificationStatus = typing.Literal["Pending", "Success", "Failed", "TemporaryFailure", "NotStarted"]
+
+logger = logging.getLogger(__name__)
+
+
+def retry(tries: int = 5, delay: int = 1.5):
+    def wrapper(fn):
+        @functools.wraps(fn)
+        def wrapped(*args, **kwargs):
+            nonlocal tries
+            while tries:
+                try:
+                    return fn(*args, **kwargs)
+                except Exception:
+                    logger.exception(f'{fn} returned error, retrying {tries} more times')
+                time.sleep(delay)
+                tries -= 1
+
+        return wrapped
+
+    return wrapper
+
+
+class AwsSesService:
+    def __init__(self, client: typing.Optional[botocore.client.BaseClient] = None, timeout: int = 5):
+        self.ses = client or boto3.client("ses")
+        self.region = self.ses._client_config.region_name
+        self.timeout = timeout
+
+    # @retry()
+    def configure_domain_identity(
+        self, domain: str, mail_subdomain: typing.Optional[str] = None
+    ) -> typing.List[DnsRecord]:
+        """
+        Creates up mail identities for sending email through a domain.
+
+        DKIM-signed messages help receiving mail servers validate that a message was not forged or altered in transit.
+
+        :param domain: domain name to set up email identity
+        :return: DNS records that need to be set
+        """
+
+        res_id = self.ses.verify_domain_identity(Domain=domain)
+        id_token = res_id["VerificationToken"]
+
+        res_dkim = self.ses.verify_domain_dkim(Domain=domain)
+        dns_records = [
+            {
+                "purpose": "identity",
+                "type": "TXT",
+                "name": domain,
+                "value": id_token,
+            },
+            *[
+                {
+                    "purpose": "dkim",
+                    "type": "CNAME",
+                    "name": f"{dkim_token}._domainkey.{domain}",
+                    "value": f"{dkim_token}.dkim.amazonses.com",
+                }
+                for dkim_token in res_dkim["DkimTokens"]
+            ],
+        ]
+        if mail_subdomain:
+            more_dns = self.configure_from_domain(domain, mail_subdomain)
+            dns_records.extend(more_dns)
+        return dns_records
+
+    @retry()
+    def check_dkim_verification_status(self, domain: str) -> VerificationStatus:
+        """
+        Returns the DKIM verification status for an email identity.
+
+        :param domain: domain used to set up the mail identity
+        :return: one of {Pending, Success, Failed, TemporaryFailure, NotStarted}
+        """
+        res = self.ses.get_identity_dkim_attributes(Identities=[domain])
+        return res["DkimAttributes"][domain]["DkimVerificationStatus"]
+
+    @retry()
+    def check_id_verification_status(self, domain: str) -> VerificationStatus:
+        """
+        Returns the id verification status of an email identity.
+
+        :param domain: domain used to set up the mail identity
+        :return: one of {Pending, Success, Failed, TemporaryFailure, NotStarted}
+        """
+        res = self.ses.get_identity_verification_attributes(Identities=[domain])
+        return res["VerificationAttributes"][domain]["VerificationStatus"]
+
+    def configure_from_domain(self, domain: str, mail_subdomain: str) -> typing.List[DnsRecord]:
+        """
+        Allows using `mail_subdomain` as FROM address when sending emails.
+        Messages sent through Amazon SES will be marked as originating from your domain instead of a subdomain of amazon.com.
+
+        :param domain: mail identity
+        :param mail_subdomain: a subdomain
+        :return: DNS records that need to be set
+        """
+        _ = self.ses.set_identity_mail_from_domain(
+            Identity=domain,
+            MailFromDomain=mail_subdomain,
+            BehaviorOnMXFailure="UseDefaultValue",  # uses $region.amazonses.com if MX records not present on the mail subdomain
+        )
+        return [
+            {
+                "purpose": "mail_from_domain",
+                "type": "MX",
+                "name": mail_subdomain,
+                "value": f"feedback-smtp.{self.region}.amazonses.com",
+                "priority": "10",
+            },
+            {
+                "purpose": "mail_from_domain",
+                "type": "TXT",
+                "name": mail_subdomain,
+                "value": f'"v=spf1 include:amazonses.com ~all"',
+            },
+        ]
+
+    @retry()
+    def check_custom_from_domain_status(self, domain: str) -> VerificationStatus:
+        """
+        Returns the status of custom email FROM request verification.
+
+        :param domain: domain used to set up email identity
+        :return: one of {Pending, Success, Failed, TemporaryFailure}
+        """
+        res = self.ses.get_identity_mail_from_domain_attributes(Identities=[domain])
+        return res["MailFromDomainAttributes"][domain].get("MailFromDomainStatus", 'NotStarted')
+
+    def bulk_check_domain_verification(self, identity_domain: str, timeout: int = 5) -> dict:
+        tasks = {
+            "status_id": functools.partial(self.check_id_verification_status, identity_domain),
+            "status_dkim": functools.partial(self.check_dkim_verification_status, identity_domain),
+            "status_custom": functools.partial(self.check_custom_from_domain_status, identity_domain),
+        }
+        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
+            futures = [pool.submit(t) for _, t in tasks.items()]
+            results = dict(zip(tasks.keys(), [f.result(timeout=timeout) for f in futures]))
+
+        return {
+            **results,
+            "identity_domain": identity_domain,
+        }
+
+    @retry()
+    def bulk_configure_email_domains(
+        self,
+        identity_domain: str,
+        mail_subdomain: typing.Optional[str] = None,
+        timeout: int = 10,
+    ) -> dict:
+        tasks = {
+            "dns": functools.partial(self.configure_domain_identity, identity_domain, mail_subdomain),
+        }
+
+        # send all requests in parallel
+        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
+            futures = [pool.submit(t) for _, t in tasks.items()]
+            results = dict(zip(tasks.keys(), [f.result(timeout=timeout) for f in futures]))
+        return {
+            **results,
+            "identity_domain": identity_domain,
+            "mail_subdomain": mail_subdomain,
+        }
+
+
+if __name__ == "__main__":
+    ses = AwsSesService()
+    logging.basicConfig(level=logging.INFO)
+    # print(ses.bulk_configure_email_domains('abdus100.dev', 'mail.abdus100.dev'))
+    # print(ses.check_custom_from_domain_status('abdus101.dev'))
+
+    tasks = [
+        functools.partial(ses.bulk_check_domain_verification, d) for d in ['abdus.dev', 'abdus1.dev', 'abdus2.dev']
+    ] * 3
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=20) as pool:
+        futures = [pool.submit(t) for t in tasks]
+        results = [f.result(timeout=5) for f in futures]
+    pprint.pp(results)

aws_ses_test.py

@@ -0,0 +1,133 @@
+from unittest import mock
+
+import pytest
+
+from aws_ses import AwsSesService
+
+# fmt: off
+_response_verify_domain_dkim = {'DkimTokens': ['token1', 'token2', 'token3'], 'ResponseMetadata': {'RequestId': '6e4da72e-3063-41da-9434-dae7ef19c78e', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:27:40 GMT', 'content-type': 'text/xml', 'content-length': '469', 'connection': 'keep-alive', 'x-amzn-requestid': '6e4da72e-3063-41da-9434-dae7ef19c78e'}, 'RetryAttempts': 0}}
+_response_verify_domain_identity = {'VerificationToken': 'token', 'ResponseMetadata': {'RequestId': 'b733069c-115f-4457-b6cb-14e44f29d17f', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:27:12 GMT', 'content-type': 'text/xml', 'content-length': '370', 'connection': 'keep-alive', 'x-amzn-requestid': 'b733069c-115f-4457-b6cb-14e44f29d17f'}, 'RetryAttempts': 0}}
+_response_get_identity_dkim_attributes = {'DkimAttributes': {'example.com': {'DkimEnabled': True, 'DkimVerificationStatus': 'Success', 'DkimTokens': ['bgobmdqs2vbxr6ygwbmg753knfu6tos7', 'm6o6w6twpmwzwe3igpxxo7qsvwp3hlgj', 'zqfz4ekdelp7m2u2phsyad3h5dsfle4v']}}, 'ResponseMetadata': {'RequestId': 'ba466be1-79bc-499b-9766-ca1acd7d1c08', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 16:49:53 GMT', 'content-type': 'text/xml', 'content-length': '833', 'connection': 'keep-alive', 'x-amzn-requestid': 'ba466be1-79bc-499b-9766-ca1acd7d1c08'}, 'RetryAttempts': 0}}
+_response_get_identity_dkim_attributes_pending = {'DkimAttributes': {'example.com': {'DkimEnabled': True, 'DkimVerificationStatus': 'Pending', 'DkimTokens': ['bgobmdqs2vbxr6ygwbmg753knfu6tos7', 'm6o6w6twpmwzwe3igpxxo7qsvwp3hlgj', 'zqfz4ekdelp7m2u2phsyad3h5dsfle4v']}}, 'ResponseMetadata': {'RequestId': 'b24732d9-01e1-4266-8b34-550a9ecf634d', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:30:03 GMT', 'content-type': 'text/xml', 'content-length': '833', 'connection': 'keep-alive', 'x-amzn-requestid': 'b24732d9-01e1-4266-8b34-550a9ecf634d'}, 'RetryAttempts': 0}}
+_response_get_identity_verification_attributes = {'VerificationAttributes': {'example.com': {'VerificationStatus': 'Success', 'VerificationToken': 'XzMvLQqOPdC0iJ02YTdjqWBJr7FTqhGtAsEcfpUwfoA='}}, 'ResponseMetadata': {'RequestId': '022c3e6e-a6e8-426f-bf4d-149f71ca7913', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 16:56:19 GMT', 'content-type': 'text/xml', 'content-length': '637', 'connection': 'keep-alive', 'x-amzn-requestid': '022c3e6e-a6e8-426f-bf4d-149f71ca7913'}, 'RetryAttempts': 0}}
+_response_get_identity_verification_attributes_pending = {'VerificationAttributes': {'example.com': {'VerificationStatus': 'Pending', 'VerificationToken': 'XzMvLQqOPdC0iJ02YTdjqWBJr7FTqhGtAsEcfpUwfoA='}}, 'ResponseMetadata': {'RequestId': '022c3e6e-a6e8-426f-bf4d-149f71ca7913', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 16:56:19 GMT', 'content-type': 'text/xml', 'content-length': '637', 'connection': 'keep-alive', 'x-amzn-requestid': '022c3e6e-a6e8-426f-bf4d-149f71ca7913'}, 'RetryAttempts': 0}}
+_response_set_identity_mail_from_domain = {'ResponseMetadata': {'RequestId': 'c04acd33-40be-46cd-9960-c9e51397dc48', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:03:50 GMT', 'content-type': 'text/xml', 'content-length': '266', 'connection': 'keep-alive', 'x-amzn-requestid': 'c04acd33-40be-46cd-9960-c9e51397dc48'}, 'RetryAttempts': 0}}
+_response_get_identity_mail_from_domain_attributes = {'MailFromDomainAttributes': {'example.com': {'MailFromDomain': 'email.abdus.dev', 'MailFromDomainStatus': 'Success', 'BehaviorOnMXFailure': 'UseDefaultValue'}}, 'ResponseMetadata': {'RequestId': '63049966-c062-47f4-8238-145bd533d880', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:37:06 GMT', 'content-type': 'text/xml', 'content-length': '687', 'connection': 'keep-alive', 'x-amzn-requestid': '63049966-c062-47f4-8238-145bd533d880'}, 'RetryAttempts': 0}}
+_response_get_identity_mail_from_domain_attributes_pending = {'MailFromDomainAttributes': {'example.com': {'MailFromDomain': 'email.abdus.dev', 'MailFromDomainStatus': 'Pending', 'BehaviorOnMXFailure': 'UseDefaultValue'}}, 'ResponseMetadata': {'RequestId': '63049966-c062-47f4-8238-145bd533d880', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:37:06 GMT', 'content-type': 'text/xml', 'content-length': '687', 'connection': 'keep-alive', 'x-amzn-requestid': '63049966-c062-47f4-8238-145bd533d880'}, 'RetryAttempts': 0}}
+# fmt: on
+
+
+@pytest.fixture()
+def mock_ses() -> mock.MagicMock:
+    with mock.patch("boto3.client") as m:
+        m_ses = m.return_value
+        yield m_ses
+
+
+def test_init_domain_verification(mock_ses):
+    m_verify_id = mock_ses.verify_domain_identity
+    m_verify_dkim = mock_ses.verify_domain_dkim
+    m_verify_id.return_value = _response_verify_domain_identity
+    m_verify_dkim.return_value = _response_verify_domain_dkim
+    domain = "example.com"
+
+    dns = AwsSesService(mock_ses).configure_domain_identity(domain)
+
+    m_verify_id.assert_called_once_with(Domain=domain)
+    m_verify_dkim.assert_called_once_with(Domain=domain)
+    assert dns == [
+        {
+            "type": "TXT",
+            "name": domain,
+            "value": "token",
+        },
+        {
+            "type": "CNAME",
+            "name": f"token1._domainkey.{domain}",
+            "value": f"token1.dkim.amazonses.com",
+        },
+        {
+            "type": "CNAME",
+            "name": f"token2._domainkey.{domain}",
+            "value": f"token2.dkim.amazonses.com",
+        },
+        {
+            "type": "CNAME",
+            "name": f"token3._domainkey.{domain}",
+            "value": f"token3.dkim.amazonses.com",
+        },
+    ]
+
+
+def test_get_dkim_status(mock_ses):
+    m = mock_ses.get_identity_dkim_attributes
+    m.return_value = _response_get_identity_dkim_attributes_pending
+    domain = "example.com"
+
+    status = AwsSesService(mock_ses).check_dkim_verification_status(domain)
+
+    assert status == "Pending"
+    m.assert_called_once_with(Identities=[domain])
+
+    # ===
+
+    m.reset_mock()
+    m.return_value = _response_get_identity_dkim_attributes
+
+    status = AwsSesService(mock_ses).check_dkim_verification_status(domain)
+    assert status == "Success"
+
+
+def test_get_id_status(mock_ses):
+    m = mock_ses.get_identity_verification_attributes
+    m.return_value = _response_get_identity_verification_attributes
+
+    domain = "example.com"
+    status = AwsSesService(mock_ses).check_id_verification_status(domain)
+
+    assert status == "Success"
+    m.assert_called_once_with(Identities=[domain])
+
+
+def test_set_custom_from_domain(mock_ses):
+    mock_ses._client_config.region_name = "fake-region"
+    m = mock_ses.set_identity_mail_from_domain
+    m.return_value = _response_set_identity_mail_from_domain
+
+    dns = AwsSesService(mock_ses).configure_from_domain("example.com", "mail.example.com")
+
+    assert dns == [
+        {
+            "type": "MX",
+            "name": "mail.example.com",
+            "value": f"feedback-smtp.fake-region.amazonses.com",
+            "priority": "10",
+        },
+        {"type": "TXT", "name": "mail.example.com", "value": f'"v=spf1 include:amazonses.com ~all"'},
+    ]
+    m.assert_called_once_with(
+        Identity="example.com",
+        MailFromDomain="mail.example.com",
+        BehaviorOnMXFailure="UseDefaultValue",
+    )
+
+
+def test_get_mail_from_status(mock_ses):
+    m = mock_ses.get_identity_mail_from_domain_attributes
+    m.return_value = _response_get_identity_mail_from_domain_attributes_pending
+
+    domain = "example.com"
+    status = AwsSesService(mock_ses).check_custom_from_domain_status(domain)
+
+    assert status == "Pending"
+    m.assert_called_once_with(Identities=[domain])
+
+    # ===
+
+    m.reset_mock()
+    m.return_value = _response_get_identity_mail_from_domain_attributes
+
+    status = AwsSesService(mock_ses).check_custom_from_domain_status(domain)
+
+    assert status == "Success"
+    m.assert_called_once_with(Identities=[domain])

berlin.py

@@ -0,0 +1,48 @@
+import contextlib
+
+from playwright.sync_api import sync_playwright, Page
+from time import sleep
+
+
+@contextlib.contextmanager
+def launch_browser() -> Page:
+    with sync_playwright() as playwright:
+        with playwright.chromium.launch(headless=False) as browser:
+            with browser.new_context() as context:
+                with context.new_page() as page:
+                    yield page
+
+
+def book_slot():
+    with launch_browser() as page:
+        page.goto("https://service.berlin.de/dienstleistung/120686/")
+        page.get_by_role("complementary").get_by_role("link", name="Termin berlinweit suchen").click()
+
+        with page.expect_navigation(url='https://service.berlin.de/terminvereinbarung/termin/time/*', timeout=0):
+            pass
+
+        with page.expect_navigation(url='https://service.berlin.de/terminvereinbarung/termin/register/*', timeout=0):
+            pass
+
+        page.get_by_label("Ihr Vor- und Nachname *").fill("Paulina Gutkowska")
+        page.get_by_label("Ihre E-Mail Adresse *").fill("gutpaula11@gmail.com")
+        if page.is_visible("Ihre Telefonnummer oder Mobilfunknummer *"):
+            page.get_by_label("Ihre Telefonnummer oder Mobilfunknummer *").fill("+48501099723")
+        page.get_by_role("combobox").select_option("0")
+        page.get_by_label(
+            "Ich erkläre mich mit den Nutzungsbedingungen einverstanden und akzeptiere diese. (Pflichtangabe. Termineintrag nur bei Einverständnis möglich) *"
+        ).check()
+
+        with page.expect_navigation():
+            page.get_by_role("button", name="Termin eintragen").click()
+        
+        sleep(10)
+        input()
+
+
+def main():
+    book_slot()
+
+
+if __name__ == '__main__':
+    main()

berlin_anmeldung.py

@@ -0,0 +1,75 @@
+import contextlib
+import datetime
+import locale
+
+import httpx
+from bs4 import BeautifulSoup
+
+import berlin
+
+http = httpx.Client(
+    headers={
+        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36'
+    },
+    follow_redirects=True,
+)
+
+
+@contextlib.contextmanager
+def override_locale(category: int, val: str) -> None:
+    prev = locale.getlocale(category)
+    locale.setlocale(category, val)
+    yield
+    locale.setlocale(category, prev)
+
+
+def get_available_slots():
+    res = http.get('https://service.berlin.de/dienstleistung/120686/')
+    res.raise_for_status()
+
+    soup = BeautifulSoup(res.text, 'html.parser')
+    link = soup.select_one('[role="complementary"] .zmstermin-multi a')
+    calendar_url = link.attrs['href']
+
+    try:
+        # redirect fails, we just need the cookies
+        _ = http.get(calendar_url)
+    except:
+        pass
+    res = http.get('https://service.berlin.de/terminvereinbarung/termin/day/')
+    res.raise_for_status()
+
+    return extract_days(res.text)
+
+
+def extract_days(html: str) -> list[datetime.date]:
+    soup = BeautifulSoup(html, 'html.parser')
+
+    found = []
+    for it in soup.select('.calendar-month-table .buchbar'):
+        month_year = it.find_parent(attrs={'class': 'calendar-month-table'}).select_one('thead .month').text.strip()
+        day = it.text.strip()
+        found.append(f'{day} {month_year}')
+
+    with override_locale(locale.LC_TIME, 'de_DE'):
+        days = [datetime.datetime.strptime(it, '%d %B %Y').date() for it in found]
+
+    return days
+
+
+def main():
+    days = get_available_slots()
+    good_days = [it for it in days if it < datetime.date(2023, 1, 27)]
+
+    if not good_days:
+        print('no slots')
+        return
+
+    print('found slots')
+    berlin.book_slot()
+    for it in days:
+        print(it)
+
+
+if __name__ == '__main__':
+    main()

berlin_immigration.py

@@ -0,0 +1,139 @@
+import argparse
+import asyncio
+import contextlib
+import datetime
+import logging
+import random
+import re
+import subprocess
+import time
+import urllib.parse
+from pathlib import Path
+
+from playwright.async_api import async_playwright, Browser, BrowserContext, Page
+
+
+@contextlib.asynccontextmanager
+async def launch_browser(headless: bool = False) -> Page:
+    async with async_playwright() as playwright:
+        browser = await playwright.chromium.launch(headless=headless)
+        browser: Browser
+        async with browser:
+            ctx = await browser.new_context()
+            ctx: BrowserContext
+            async with ctx:
+                page = await ctx.new_page()
+                async with page:
+                    yield page
+
+
+class Error(Exception):
+    pass
+
+
+def alert(text: str):
+    subprocess.run(['open', f'alfred://runtrigger/dev.abdus.integrations/remind/?argument={urllib.parse.quote(text)}'])
+
+
+dump_dir = Path('~/Desktop').expanduser() / 'immigration'
+dump_dir.mkdir(parents=True, exist_ok=True)
+
+async def check_slots(page: Page, retry_count: int = 1) -> list[datetime.datetime]:
+    await page.add_init_script('''Object.defineProperty(navigator, 'webdriver', { get: () => false })''')
+    window_id = random.randint(1000, 9999)
+    req_id = random.randint(0, 999)
+    await page.goto(
+        f'https://otv.verwalt-berlin.de/ams/TerminBuchen/wizardng?dswid={window_id}&dsrid={req_id}',
+        wait_until='networkidle',
+    )
+    await page.check('[name="gelesen"]')
+    async with page.expect_navigation(url=re.compile('st=2')):
+        await page.click('[name="applicationForm:managedForm:proceed"]')
+
+    await page.get_by_role("combobox", name="Staatsangehörigkeit *").select_option(label="Türkei")
+    await asyncio.sleep(0.3)
+
+    await page.get_by_role(
+        "combobox",
+        name="Anzahl der Personen, die einen Aufenthaltstitel beantragen (auch ausländische Ehepartner und Kinder) *",
+    ).select_option(label="eine Person")
+    await asyncio.sleep(0.3)
+
+    await page.get_by_role(
+        "combobox", name="Leben Sie in Berlin zusammen mit einem Familienangehörigen (z.B. Ehepartner, Kind) *"
+    ).select_option(label="nein")
+    await asyncio.sleep(0.3)
+
+    await page.get_by_text("Aufenthaltstitel - beantragen").click()
+    await page.locator("label").filter(has_text="Erwerbstätigkeit").click()
+    await page.get_by_text("Blaue Karte EU (§ 18b Abs. 2)").click()
+
+    stop_at = datetime.datetime.now() + datetime.timedelta(minutes=28)
+
+    async def go_forward():
+        async with page.expect_navigation(url=re.compile(r'st='), wait_until='networkidle', timeout=60_000):
+            await page.get_by_role("button", name="Weiter").click()
+        if 'st=2' in page.url:
+            error_message = await page.inner_text('.errorMessage')
+            if 'keine Termine frei' in error_message:
+                return False
+            raise Error(error_message)
+        return 'st=3' in page.url
+
+    while 'st=2' in page.url:
+        retry_count -= 1
+
+        async with page.expect_navigation(url=re.compile(r'st='), wait_until='networkidle', timeout=60_000):
+            await page.get_by_role("button", name="Weiter").click()
+
+        if datetime.datetime.now() > stop_at:
+            raise TimeoutError('could not find a slot')
+
+        if not retry_count:
+            return []
+
+        await asyncio.sleep(random.randint(4, 10))
+        continue
+
+    if 'st=3' in page.url:
+        alert('empty slot found')
+        now = int(datetime.datetime.now().timestamp())
+        html_path = dump_dir / f'immigration_{now}.html'
+        html_path.write_text(await page.content())
+        await page.screenshot(path=dump_dir/f'immigration_{now}.png')
+
+        await page.pause()
+
+    input()
+
+
+def parse_args():
+    arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    arger.add_argument(
+        '--headless', '--silent', '--quiet', action='store_true', dest='headless', help='Run in headless mode'
+    )
+    arger.add_argument(
+        '--retry-count', '--retry', type=int, default=1, help='Run in headless mode'
+    )
+
+    args, _ = arger.parse_known_args()
+    return args
+
+
+async def main():
+    args = parse_args()
+    async with launch_browser(headless=args.headless) as page:
+        try:
+            slots = await check_slots(page, retry_count=args.retry_count)
+            if not slots:
+                print('no slots yet')
+                return
+        except Exception as e:
+            logging.exception('got an error')
+            await page.pause()
+
+
+if __name__ == '__main__':
+    asyncio.run(main())
+    logging.basicConfig(level=logging.INFO)
+    main()

deluge_import.py

@@ -0,0 +1,121 @@
+import logging
+import random
+import subprocess
+import httpx
+from pathlib import Path
+import torf
+import time
+
+class Deluge:
+    def __init__(self, session: httpx.Client) -> None:
+        self.session = session
+
+    @classmethod
+    def new(cls) -> 'Deluge':
+        logging.debug("Connecting to Deluge")
+        session = httpx.Client(base_url="https://t.zzzp.win/", timeout=10, auth=httpx.BasicAuth('abdus', 'xAsametk50'))
+        res = session.post("/json", json={"method": "auth.login", "params": ["xAsametk50"], "id": cls.make_id()})
+        res.raise_for_status()
+
+        # find first available host
+        res = session.post("/json", json={"method": "web.get_hosts", "params": [], "id": cls.make_id()})
+        res.raise_for_status()
+        host_id = res.json()["result"][0][0]
+
+        # connect to a host
+        res = session.post("/json", json={"method": "web.connect", "params": [host_id], "id": cls.make_id()})
+        res.raise_for_status()
+
+        return cls(session)
+
+    @classmethod
+    def make_id(cls) -> int:
+        return time.time_ns()
+
+    def add(self, torrent_path: Path, download_path: Path, paused: bool = True, skip_check: bool = True):
+        # upload torrent
+        logging.debug("Uploading torrent file")
+        with torrent_path.open("rb") as f:
+            res = self.session.post("/upload", files={"file": f})
+        res.raise_for_status()
+        remote_path: str = res.json()["files"][0]
+
+        # add torrent
+        logging.debug("Adding torrent file")
+        res = self.session.post(
+            "/json",
+            timeout=1,
+            json={
+                "method": "web.add_torrents",
+                "params": [
+                    [
+                        {
+                            "path": remote_path,
+                            "options": {
+                                "file_priorities": [1],
+                                "add_paused": paused,
+                                "sequential_download": False,
+                                "pre_allocate_storage": False,
+                                "download_location": str(download_path),
+                                # "move_completed": False,
+                                # "move_completed_path": "/root/Downloads",
+                                # "prioritize_first_last_pieces": True,
+                                "seed_mode": skip_check,
+                                # "super_seeding": False,
+                            },
+                        }
+                    ]
+                ],
+                "id": self.make_id(),
+            },
+        )
+
+        res.raise_for_status()
+
+
+def find_torrents() -> list[Path]:
+    files = list(Path('/tmp/deluge/config/state').glob('*.torrent'))
+    random.shuffle(files)
+    return files
+
+def find_existing_torrents() -> dict[str, Path]:
+    lines = subprocess.check_output(['ssh', 'klein', '--', 'find', '/mnt/box/files/_torrents', '-maxdepth', '1']).decode()
+    existing = {}
+    lines = lines.strip().splitlines(keepends=False)
+    for line in lines:
+        if line.startswith('.'):
+            continue
+        p = Path(line)
+        existing[p.name] = p
+    return existing
+
+
+def main():
+    existing = find_existing_torrents()
+    d = Deluge.new()
+    done = set()
+    for torrent_path in find_torrents():
+        if torrent_path in done:
+            continue
+        if len(done) == len(existing):
+            return
+        with torrent_path.open('rb') as f:
+            t = torf.Torrent.read_stream(f)
+        if '.PRT' in t.name:
+            continue
+        if et := existing.get(t.name):
+            logging.debug(f'adding {t.name}')
+            download_path = Path("/dl")
+            try:
+                d.add(torrent_path, download_path=download_path)
+                torrent_path.unlink()
+                done.add(torrent_path)
+                logging.info(f'added: {t.name}')
+            except Exception as e:
+                pass
+
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format=f'%(asctime)s {logging.BASIC_FORMAT}')
+    main()

duration_parser.py

@@ -0,0 +1,40 @@
+import sys
+import re
+import math
+import datetime
+
+
+def parse_duration(text: str) -> datetime.timedelta:
+    re_duration = re.compile(r'(\d+(?:\.\d+)?(?:sec|min|hour|hr|h|m|s))')
+    re_digits = re.compile(r'([\d.]+)')
+    time_scales = {
+        's': 1,
+        'sec': 1,
+        'm': 60,
+        'min': 60,
+        'h': 3600,
+        'hr': 3600,
+    }
+
+    parts = re_duration.findall(text)
+    if parts:
+        total_secs = 0
+        for it in parts:
+            if m := re_digits.search(it):
+                _, end = m.span()
+                scale = it[end:]
+                total_secs += math.ceil(float(m.group(1)) * time_scales[scale])
+        return datetime.timedelta(seconds=total_secs)
+
+    parts = [float(it) for it in text.split(':')]
+    if len(parts) == 1:
+        return datetime.timedelta(seconds=parts[0])
+    if len(parts) == 2:
+        return datetime.timedelta(minutes=parts[0], seconds=parts[1])
+    if len(parts) == 3:
+        return datetime.timedelta(hours=parts[0], minutes=parts[1], seconds=parts[2])
+
+
+if __name__ == '__main__':
+    parsed = parse_duration(sys.argv[1])
+    seconds = int(parsed.total_seconds())

emp.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3.9
+from dataclasses import dataclass
+import json
+from pathlib import Path
+import subprocess
+import typing
+from playwright.sync_api import Playwright, sync_playwright, Browser
+from typer import Option, Typer
+
+
+class Storage(typing.Protocol):
+    def get(self, key: str) -> typing.Optional[typing.Any]:
+        ...
+
+    def set(self, key: str, value) -> None:
+        ...
+
+
+@dataclass
+class FileStorage:
+    path: Path
+
+    def get(self, key: str):
+        try:
+            return json.loads(self.path.read_text()).get(key)
+        except FileNotFoundError:
+            return None
+
+    def set(self, key: str, value) -> None:
+        try:
+            data = json.loads(self.path.read_text())
+        except FileNotFoundError:
+            data = {}
+        data[key] = value
+        self.path.write_text(json.dumps(data))
+
+
+class Emp:
+    def __init__(self, browser: Browser, storage: Storage) -> None:
+        self.browser = browser
+        self.storage = storage
+
+    def ensure_session(self, username: str, password: str) -> None:
+        if self.storage.get("cookies") is None:
+            self.login(username, password)
+
+    def login(self, username: str, password: str) -> typing.Dict[str, str]:
+        """
+        logins and returns session cookies
+        """
+        context = self.browser.new_context()
+
+        # Open new page
+        page = context.new_page()
+
+        # Go to https://www.empornium.is/
+        page.goto("https://www.empornium.is/login")
+
+        page.locator('[placeholder="Username"]').fill(username)
+        page.locator('[placeholder="Password"]').fill(password)
+        # Click text=Stay logged in
+        page.locator("text=Stay logged in").click()
+
+        # Click input:has-text("login")
+        page.locator('input:has-text("login")').click()
+
+        cookies = {c["name"]: c["value"] for c in context.cookies()}
+        self.storage.set("cookies", {"sid": cookies["sid"]})
+
+        return {
+            "sid": cookies["sid"],
+        }
+
+    def prepare_post(
+        self,
+        torrent_path: Path,
+        title: str,
+        tags: str,
+        description: str,
+        cover_image_url: str,
+        category: typing.Optional[str] = None,
+    ) -> None:
+        cookies = self.storage.get("cookies")
+        assert cookies, "You must login first"
+
+        context = self.browser.new_context()
+        context.add_cookies(
+            [{"name": k, "value": v, "domain": "www.empornium.is", "path": "/"} for k, v in cookies.items()]
+        )
+
+        page = context.new_page()
+
+        page.goto("https://www.empornium.is/upload.php")
+
+        if torrent_path.is_file():
+            page.locator('input[name="file_input"]').set_input_files(torrent_path.expanduser().resolve())
+            page.locator('text="check for dupes"').click()
+
+        # Select category
+        if category:
+            page.locator('select[name="category"]').select_option(label=category)
+
+        page.locator('input[name="title"]').fill(title)
+        page.locator('textarea[name="taglist"]').fill(tags)
+        page.locator('input[name="image"]').fill(cover_image_url)
+        page.locator('textarea[name="desc"]').fill(description)
+
+        # Click text=Preview
+        page.locator("text=Preview").click()
+
+
+def submit_post():
+    storage = FileStorage(Path("emp.json"))
+
+    with sync_playwright() as playwright:
+        browser = playwright.chromium.launch(headless=False)
+        emp = Emp(browser=browser, storage=storage)
+        emp.ensure_session(username="zzzp", password="9arjs9za2o")
+
+        emp.prepare_post(
+            torrent_path=Path("~/Downloads/v.torrent"),
+            title="A Title",
+            tags="tag.1 tag.2",
+            description="Some description",
+            cover_image_url="https://images.com/image.jpg",
+            category="Anal",
+        )
+        input("Press Enter to continue...")
+        browser.close()
+
+
+cli = Typer(name="emp")
+
+
+torrent_cli = Typer(name="torrent")
+cli.add_typer(torrent_cli)
+
+
+@torrent_cli.callback("torrent")
+def make_torrent(paths: typing.List[Path], announce_url: str = Option(..., envvar="ANNOUNCE_URL")):
+    if len(paths) == 1 and paths[0].is_dir():
+        dir_path = paths[0]
+    
+    dir_path = dir_path.expanduser().resolve()
+    proc = subprocess.run(
+        [
+            "torrentify",
+            f"-announce={announce_url}",
+            f"-comment=created by zzzp",
+            f"-created-by=zzzp",
+            f"-name={dir_path.name}",
+            str(dir_path),
+        ],
+        check=True,
+        capture_output=True,
+    )
+    torrent_path = dir_path / f"{dir_path.name}.torrent"
+    torrent_path.write_bytes(proc.stdout)
+
+
+@torrent_cli.command()
+def clone():
+    pass
+
+
+if __name__ == "__main__":
+    cli()

enc.py

@@ -0,0 +1,479 @@
+#!/usr/bin/env python3.9
+import argparse
+import dataclasses
+import datetime
+import json
+import logging
+import re
+import subprocess
+import sys
+import threading
+import time
+import typing
+from pathlib import Path
+
+import math
+import rich.progress
+
+logger = logging.getLogger(__name__)
+
+
+@dataclasses.dataclass
+class FfprobeResult:
+    duration_sec: float
+    duration_human: str
+    codec: str
+    fps: float
+    size_bytes: int
+    width: int
+    height: int
+    bitrate: int
+    container: str
+    ar: float
+    sample_ar: float
+
+
+def ffprobe(video: Path) -> FfprobeResult:
+    logger.debug("executing ffprobe")
+    proc = subprocess.run(
+        args=[
+            "ffprobe",
+            "-v",
+            "error",
+            "-show_streams",
+            "-show_format",
+            "-print_format",
+            "json",
+            str(video),
+        ],
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    try:
+        proc.check_returncode()
+    except subprocess.CalledProcessError:
+        logger.error("failed to run ffprobe", exc_info=True)
+        raise
+    output: dict = json.loads(proc.stdout)
+    video_stream: dict = [s for s in output.get("streams", []) if s.get("codec_type") == "video"][0]
+    video_format: dict = output.get("format", {})
+    width, height = video_stream.get("width"), video_stream.get("height")
+    codec = video_stream.get("codec_name")
+    duration = float(video_format.get("duration"))
+    fps_long = eval(video_stream.get("r_frame_rate", ""))
+    fps = float(f"{fps_long:.3f}")
+    size = int(video_format.get("size", 0))
+    duration_time = datetime.timedelta(seconds=math.ceil(duration))
+    bitrate = int(video_stream.get("bit_rate", video_format.get("bit_rate", 0)))
+
+    try:
+        sample_w, sample_h = list(map(int, video_stream["sample_aspect_ratio"].split(":")))
+        sample_ar = (sample_w / sample_h) or 1
+    except:
+        sample_ar = 1
+
+    logger.debug(f"dimensions={width}x{height} duration={duration_time}, fps={fps}, size={size // 1_048_576}MB")
+    # total_frames = video_stream.get("nb_frames")
+
+    return FfprobeResult(
+        duration_sec=duration,
+        duration_human=str(duration_time),
+        codec=codec,
+        fps=fps,
+        size_bytes=size,
+        width=width,
+        bitrate=bitrate,
+        height=height,
+        container=video.suffix.lstrip(".").lower(),
+        ar=width / height,
+        sample_ar=sample_ar,
+        # total_frames=total_frames,
+    )
+
+
+def make_thumbnail_tile(
+    video: Path,
+    image_path: Path = None,
+    columns: int = 3,
+    interval_seconds: int = 60,
+    tile_width: int = 540,
+    skip_first_sec: int = 10,
+    skip_if_exists: bool = False,
+) -> Path:
+    if not image_path:
+        image_path = video.parent / f"{video.stem}.thumbnail.jpg"
+    if image_path.is_file() and skip_if_exists:
+        logger.debug("thumbnail already exists")
+        return image_path
+
+    info = ffprobe(video)
+
+    min_frames = columns * 3
+    sec_per_frame = min(interval_seconds, math.ceil(info.duration_sec / min_frames))
+    rows = math.ceil(info.duration_sec // sec_per_frame / columns)
+    tile = f"{columns}x{rows}"
+
+    scaled_width, scaled_height = tile_width, math.ceil(tile_width / info.ar / info.sample_ar)
+    scale = f"{scaled_width}:{scaled_height}"
+
+    proc = subprocess.run(
+        # fmt: off
+        args=[
+            "ffmpeg",
+            "-v", "error",
+            "-skip_frame", "nokey",
+            "-ss", f"{skip_first_sec}",
+            "-i", str(video.absolute()),
+            "-vf", f"fps=1/{sec_per_frame},scale={scale},tile={tile}",
+            "-frames", "1",
+            "-y", str(image_path.absolute()),
+        ],
+        # fmt: on
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        timeout=2000,
+    )
+    proc.check_returncode()
+    return image_path
+
+
+@dataclasses.dataclass
+class EncodeProgress:
+    percent: float
+    fps_avg: int
+    eta: str
+    current_size: int
+
+    @property
+    def encoded_mb(self) -> int:
+        return math.ceil(self.current_size / 1_048_576)
+
+    @property
+    def estimated_mb(self) -> int:
+        return math.ceil(self.encoded_mb / (self.percent / 100))
+
+
+@dataclasses.dataclass
+class UploadProgress:
+    uploaded_mb: int
+    speed: str
+    percent: float
+
+
+def noop(*args):
+    pass
+
+
+def upload_file(
+    file_path: Path,
+    destination: str,
+    watch: bool = False,
+    on_progress: typing.Callable[[UploadProgress], None] = noop,
+    on_resync: typing.Callable[[], None] = noop,
+):
+    size = file_path.stat().st_size
+    while True:
+        proc = subprocess.Popen(
+            ["rsync", "--bwlimit", "1100", "-rvPa", str(file_path), destination],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+
+        def watch_progress(p: subprocess.Popen):
+            #  5233864   0%    1.93MB/s    0:21:14
+            #  12,450,960  30%    2.49MB/s    0:00:11
+            pattern = re.compile(r"\b(?P<uploaded>[\d,]+)\s+(?P<percent>[\d]+)%\s+(?P<speed>\d+\.\d+[kmg]B/s)\b", re.I)
+            for line in p.stdout:
+                if m := pattern.search(line):
+                    parsed = m.groupdict()
+                    progress = UploadProgress(
+                        uploaded_mb=int(parsed["uploaded"].replace(",", "")) // 1_048_576,
+                        speed=parsed["speed"],
+                        percent=float(parsed["percent"]),
+                    )
+                    try:
+                        on_progress(progress)
+                    except:
+                        pass
+
+        t = threading.Thread(target=watch_progress, args=(proc,))
+        t.start()
+
+        if proc.wait() != 0:
+            code = proc.poll()
+            err = proc.stderr.read()
+            logger.warning(f"rsync failed with {code=}. error: {err}")
+            return
+
+        if not watch:
+            return
+
+        time.sleep(5)
+        current_size = file_path.stat().st_size
+        diff = current_size - size
+        size = current_size
+
+        if not diff:
+            return
+
+        logger.info(f"filesize changed {diff} bytes, re-syncing...")
+        on_resync()
+
+
+def encode_video_handbrake(
+    video_path: Path,
+    save_path: Path,
+    quality: int = 30,
+    from_time: typing.Optional[int] = None,
+    duration: typing.Optional[int] = None,
+    on_progress: typing.Callable[[EncodeProgress], None] = noop,
+    extra_args: typing.List[str] = None,
+    denoise: bool = False,
+    is_10bit: bool = True,
+):
+    # fmt: off
+    args = [
+        'HandbrakeCLI',
+        '--format', 'av_mp4',
+        '--input', str(video_path),
+        *(['--start-at', f'duration:{from_time}'] if from_time else []),
+        *(['--stop-at', f'duration:{duration}'] if duration else []),
+        '--output', str(save_path),
+        '--optimize',
+        '--encoder', *['vt_h265_10bit' if is_10bit else 'vt_h265'],
+        '--quality', str(quality),
+        '--vfr',
+        '--aencoder', 'ac3',
+        '--ab', '160',
+        '--non-anamorphic',
+        *(['--hqdn3d', 'light'] if denoise else []),
+        *(extra_args or []),
+        # '--json',
+        '--verbose', '0'
+    ]
+    # fmt: on
+    # subprocess.run(args, check=True)
+    # return
+    logger.debug("executing handbrake with args: %s", args)
+    p = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
+
+    last_progress: typing.Optional[EncodeProgress] = None
+
+    def parse_progress(p: subprocess.Popen):
+        # Encoding: task 1 of 1, 11.96 % (210.12 fps, avg 206.32 fps, ETA 00h06m14s)
+        re_progress = re.compile(r"(?P<percent>[\d.]+) % \(.+ avg (?P<fps_avg>[\d.]+) fps, ETA (?P<eta>[^)]+)")
+        for line in p.stdout:
+            if m := re_progress.search(line.strip()):
+                parsed = m.groupdict()
+                percent = round(float(parsed["percent"]), 1)
+                fps_avg = math.floor(float(parsed["fps_avg"]))
+                eta = parsed["eta"]
+
+                nonlocal last_progress
+                last_progress = EncodeProgress(percent, fps_avg, eta, current_size=save_path.stat().st_size)
+
+                try:
+                    on_progress(last_progress)
+                except:
+                    pass
+
+    t = threading.Thread(target=parse_progress, args=(p,))
+    t.start()
+
+    try:
+        if p.wait() != 0:
+            logger.error(f"handbrake failed: {p.stderr.read()}")
+    except KeyboardInterrupt:
+        p.kill()
+        if last_progress:
+            logger.info(f"cancelled at {last_progress.percent}%")
+            logger.info(f"encoded file is {last_progress.encoded_mb}MB, estimated: {last_progress.estimated_mb}MB")
+        raise
+
+
+def encode_video_ffmpeg(
+    video_path: Path,
+    save_path: Path,
+    quality: int = 30,
+    from_time: typing.Optional[int] = None,
+    duration: typing.Optional[int] = None,
+    on_progress: typing.Callable[[EncodeProgress], None] = noop,
+    extra_args: typing.List[str] = None,
+    is_10bit: bool = True,
+    **kwargs,
+):
+    # fmt: off
+    args = [
+        'ffmpeg',
+        '-y',
+        # '-v', 'quiet',
+        '-progress', 'pipe:1',
+        '-stats_period', '3',
+        *(['-ss', str(from_time)] if from_time else []),
+        *(['-t', str(duration)] if duration else []),
+        '-i', str(video_path),
+        '-c:v', 'hevc_videotoolbox',
+        '-q:v', f'{quality}',
+        '-profile:v', *['main10' if is_10bit else 'main'],
+        '-map_metadata', '0',
+        '-metadata', f'title={video_path.stem}',
+        *(extra_args or []),
+        str(save_path),
+    ]
+    # fmt: on
+
+    logger.info(f"calling ffmpeg with {args=}")
+
+    proc = subprocess.run(args, check=True)
+
+
+def parse_args():
+    arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    arger.add_argument("video_path", type=Path, help="path to video file")
+    arger.add_argument("-e", "--encoder", choices=["handbrake", "ffmpeg"], default="handbrake", help="encoder engine")
+    arger.add_argument("-q", "--quality", type=float, default=25, help="x265 quality factor")
+    arger.add_argument("--output-dir", "-o", dest="output_dir", type=Path, help="Dir to save encoded files")
+    arger.add_argument("--rsync", dest="upload_target", help="rsync encoded file to a host")
+    arger.add_argument("--validate", action="store_true", default=False, help="perform validations before starting")
+    arger.add_argument("--denoise", action="store_true", default=False, help="Enable denoise filter (Handbrake only)")
+    arger.add_argument("--10bit", action="store_true", dest="is_10bit", help="Encode using 10-bit profile")
+    arger.add_argument("--8bit", action="store_false", dest="is_10bit", help="Encode using 8-bit profile")
+    arger.set_defaults(is_10bit=True)
+
+    def parse_time(val: str) -> int:
+        parts = val.split(":")
+        if len(parts) == 1:
+            return int(val)
+        elif len(parts) == 2:
+            return int(parts[0]) * 60 + int(parts[1])
+        return int(parts[0]) * 60 + int(parts[1]) * 60 + int(parts[2])
+
+    arger.add_argument(
+        "--from",
+        dest="from_time",
+        default=0,
+        type=parse_time,
+        help="Start encoding from this time. Example 05:00 or 300",
+    )
+    arger.add_argument("--duration", type=parse_time, help="Stop encoding at this time. Example 07:00 or 420")
+
+    if len(sys.argv[1:]) < 1:
+        arger.print_help()
+        exit(1)
+
+    args, extra_args = arger.parse_known_args()
+    if extra_args and extra_args[0] == "--":
+        extra_args = extra_args[1:]
+
+    return args, extra_args
+
+
+def generate_filename(video_path: Path) -> str:
+    probe = ffprobe(video_path)
+    if 1900 <= probe.width <= 2000:
+        hd = "1080p"
+    elif 1200 <= probe.width <= 1400:
+        hd = "720p"
+    elif 3000 <= probe.width:
+        hd = "4K"
+    else:
+        hd = None
+    new_stem = re.sub(r"(\[\d+[pk]])", "", video_path.stem)
+    new_stem = f"{new_stem.strip()} [{hd}, x265]"
+    return new_stem
+
+
+def main():
+    args, extra_args = parse_args()
+
+    video_path: Path = args.video_path
+    video_path = video_path.expanduser().resolve()
+    if not video_path.is_file():
+        logger.error("No such file")
+        exit(1)
+
+    if args.validate and len(video_path.name) >= 128:
+        logger.error("Filename is too long")
+        exit(1)
+
+    save_path = video_path.with_stem(generate_filename(video_path)).with_suffix(".mp4")
+    if video_path == save_path:
+        save_path = save_path.with_suffix(".reencoded" + video_path.suffix)
+
+    output_dir: Path = args.output_dir
+    if not output_dir:
+        output_dir = save_path.parent / "_reenc"
+
+    output_dir = output_dir.expanduser().resolve()
+    output_dir.mkdir(exist_ok=True, parents=True)
+    save_path = output_dir / save_path.name
+
+    bar = rich.progress.Progress(refresh_per_second=2)
+    task_encode = bar.add_task("encoding", visible=False)
+    up_task = bar.add_task("uploading", visible=False, start=False)
+
+    total_encoded_mb = 0
+
+    def sync_later():
+        logger.info("syncing in 10s")
+        time.sleep(10)
+        bar.update(up_task, visible=True)
+        bar.start_task(up_task)
+        upload_file(
+            save_path,
+            args.upload_target,
+            watch=True,
+            on_progress=lambda p: bar.update(
+                up_task,
+                description=f"upload: {p.speed}, {p.uploaded_mb:3}MB/{total_encoded_mb:3}MB",
+                completed=p.uploaded_mb / total_encoded_mb * 100,
+            ),
+            on_resync=lambda: bar.reset(up_task, description="upload: re-syncing"),
+        )
+
+    t = threading.Thread(target=sync_later)
+    if args.upload_target:
+        t.start()
+
+    encoder = encode_video_ffmpeg if args.encoder == "ffmpeg" else encode_video_handbrake
+
+    with bar:
+
+        def _on_encode_progress(p: EncodeProgress):
+            nonlocal total_encoded_mb
+            total_encoded_mb = p.encoded_mb
+            bar.update(
+                task_encode,
+                completed=p.percent,
+                description=f"encode: {p.fps_avg:3}fps, {p.encoded_mb:3}MB/{p.estimated_mb:3}MB",
+            )
+
+        try:
+            bar.update(task_encode, visible=True)
+            encoder(
+                video_path,
+                quality=args.quality,
+                save_path=save_path,
+                extra_args=extra_args,
+                on_progress=_on_encode_progress,
+                from_time=args.from_time,
+                duration=args.duration,
+                denoise=args.denoise,
+                is_10bit=args.is_10bit,
+            )
+        except KeyboardInterrupt:
+            logger.info("cancelled")
+            exit(1)
+
+        logger.info(f"finished encoding {video_path}")
+        if args.upload_target:
+            t.join()
+            image_path = make_thumbnail_tile(save_path, image_path=save_path.with_suffix(".jpg"), skip_if_exists=True)
+            upload_file(image_path, args.upload_target)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.DEBUG)
+    main()

ffmpeg.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3.9
+
+import dataclasses
+import json
+import logging
+import subprocess
+import typing
+from datetime import timedelta
+from math import ceil
+from pathlib import Path
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclasses.dataclass
+class FfprobeResult:
+    duration_sec: float
+    duration_human: str
+    codec: str
+    fps: float
+    size_bytes: int
+    width: int
+    height: int
+    bitrate: int
+    container: str
+    ar: float
+    sample_ar: float
+    tags: typing.Dict[str, str]
+
+
+def ffprobe(video: Path) -> FfprobeResult:
+    logger.debug("executing ffprobe")
+    proc = subprocess.run(
+        args=[
+            "ffprobe",
+            "-v",
+            "error",
+            "-show_streams",
+            "-show_format",
+            "-print_format",
+            "json",
+            str(video),
+        ],
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    try:
+        proc.check_returncode()
+    except subprocess.CalledProcessError:
+        logger.error("failed to run ffprobe", exc_info=True)
+        raise
+    output: dict = json.loads(proc.stdout)
+    video_stream: dict = [s for s in output.get("streams", []) if s.get("codec_type") == "video"][0]
+    video_format: dict = output.get("format", {})
+    width, height = video_stream.get("width"), video_stream.get("height")
+    profile = video_stream.get("profile")
+    codec = video_stream.get("codec_name")
+    duration = float(video_format.get("duration"))
+    fps_long = eval(video_stream.get("r_frame_rate", ""))
+    fps = float(f"{fps_long:.3f}")
+    size = int(video_format.get("size", 0))
+    duration_time = timedelta(seconds=ceil(duration))
+    bitrate = int(video_stream.get("bit_rate", video_format.get("bit_rate", 0)))
+    tags = video_format.get("tags", {})
+
+    try:
+        sample_w, sample_h = list(map(int, video_stream["sample_aspect_ratio"].split(":")))
+        sample_ar = (sample_w / sample_h) or 1
+    except:
+        sample_ar = 1
+
+    logger.debug(f"dimensions={width}x{height} duration={duration_time}")
+    # total_frames = video_stream.get("nb_frames")
+
+    return FfprobeResult(
+        duration_sec=duration,
+        duration_human=str(duration_time),
+        codec=codec,
+        fps=fps,
+        size_bytes=size,
+        width=width,
+        bitrate=bitrate,
+        height=height,
+        container=video.suffix.lstrip(".").lower(),
+        ar=width / height,
+        sample_ar=sample_ar,
+        tags=tags,
+        # total_frames=total_frames,
+    )
+
+
+def strip_metadata(video_path: Path, save_path: Path) -> Path:
+    media_info = ffprobe(video_path)
+    # check if the first video stream has hevc codec
+    if media_info["streams"][0]["codec_name"] == "hevc":
+        logger.info(f"Video codec is hevc, adding hvc1 tag")
+        extra_args = ["-tag:v", "hvc1"]
+    else:
+        extra_args = []
+
+    # fmt: off
+    args = [
+        'ffmpeg',
+        '-i',
+        str(video_path),
+        '-c:v', 'copy',
+        '-movflags', '+faststart', 
+        '-map_metadata', '-1',
+        *extra_args,
+        '-c:a', 'copy',
+        '-y',
+        str(save_path),
+    ]
+    # fmt: on
+    logger.info(f"calling ffmpeg with {args=}")
+    subprocess.run(args, check=True)
+    return save_path
+
+
+def make_thumbnail_tile(
+    video: Path,
+    image_path: Path = None,
+    columns: int = 3,
+    interval_seconds: int = 60,
+    tile_width: int = 540,
+    skip_first_sec: int = 10,
+    skip_if_exists: bool = False,
+) -> Path:
+    if not image_path:
+        image_path = video.parent / f"{video.stem}.thumbnail.jpg"
+    if image_path.is_file() and skip_if_exists:
+        logger.debug("thumbnail already exists")
+        return image_path
+
+    info = ffprobe(video)
+
+    min_frames = columns * 3
+
+    sec_per_frame = min(interval_seconds, ceil(info.duration_sec / min_frames))
+
+    rows = ceil(info.duration_sec // sec_per_frame / columns)
+    tile = f"{columns}x{rows}"
+
+    scaled_width, scaled_height = tile_width, ceil(tile_width / info.ar / info.sample_ar)
+    scale = f"{scaled_width}:{scaled_height}"
+
+    # font_path = Path(__file__).parent / 'iosevka.ttf'
+    # timestamp_filter = rf"drawtext=r=1:timecode='00\:00\:00\:00':fontsize=16:fontcolor=white:x=10:y=10:box=1:boxcolor=black@0.5"
+    # timestamp_filter = fr"drawtext=text='%{{(pts\\+{skip_first_sec})\:hms}}':fontsize=16:fontcolor=white:x=10:y=10:box=1:boxcolor=black@0.5"
+
+    proc = subprocess.run(
+        # fmt: off
+        args=[
+            "ffmpeg",
+            "-v", "error",
+            "-skip_frame", "nokey",
+            "-ss", f"{skip_first_sec}",
+            "-i", str(video.absolute()),
+            # "-vf", f"fps=1/{sec_per_frame},scale={scale},{timestamp_filter},tile={tile}",
+            "-vf", f"fps=1/{sec_per_frame},scale={scale},tile={tile}",
+            "-frames", "1",
+            "-y", str(image_path.absolute()),
+        ],
+        # fmt: on
+        stdout=subprocess.DEVNULL,
+        timeout=2000,
+    )
+    proc.check_returncode()
+    return image_path
+
+
+if __name__ == "__main__":
+    import sys
+
+    logging.basicConfig(level=logging.DEBUG)
+    for it in sys.argv[1:]:
+        video = Path(it)
+
+        if not video.exists():
+            continue
+
+        logger.info("creating thumbnail for %s", video)
+        make_thumbnail_tile(video)

ffmpeg_concat.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3.9
+import argparse
+import logging
+import subprocess
+import sys
+from pathlib import Path
+
+
+def parse_args(argv: list[str]):
+    arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    arger.add_argument("files", nargs="+", type=Path, help="Video paths")
+    arger.add_argument("-o", dest="output_path", type=Path, help="Output path", required=True)
+    if len(argv) == 0:
+        arger.print_help()
+        exit(1)
+
+    return arger.parse_args(argv)
+
+
+def combine_videos(videos: list[Path], save_path: Path) -> Path:
+    """
+    Combine multiple videos into one.
+    :param videos: List of video paths.
+    :param save_path: Path to save the combined video.
+    :return: Path to the combined video.
+    """
+
+    # file '/path/to/file1'
+    # file '/path/to/file2'
+    # file '/path/to/file3'
+    #
+    # $ ffmpeg -f concat -safe 0 -i mylist.txt -c copy output.mp4
+
+    # ffmpeg -f concat -safe 0 -i mylist.txt -c copy output.mp4
+    stdin = "\n".join(f"file '{video.absolute()}'" for video in videos if video.is_file())
+    # fmt: off
+    args = [
+        'ffmpeg',
+        '-protocol_whitelist', 'file,pipe',
+        '-f', 'concat',
+        '-safe', '0',
+        '-i', '-',
+        '-c', 'copy',
+        str(save_path),
+    ]
+    # fmt: on
+    logging.info(f"calling ffmpeg with {args=} and {stdin=}")
+    subprocess.run(args, input=stdin.encode(), check=True)
+    return save_path
+
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+    args = parse_args(sys.argv[1:])
+    paths = [Path(f) for f in args.files]
+    combine_videos(videos=paths, save_path=args.output_path)
+
+
+if __name__ == "__main__":
+    main()

ffmpeg_gif.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3.9
+import argparse
+import datetime
+import hashlib
+import math
+import re
+
+import subprocess
+import sys
+import typing
+from pathlib import Path
+
+import ffmpeg
+
+
+def create_gif(
+    video_path: Path,
+    from_time: str,
+    to_time: typing.Optional[str] = None,
+    save_path: typing.Optional[Path] = None,
+    duration: typing.Optional[int] = 10,
+    fps: int = 15,
+    width: int = 320,
+) -> Path:
+    if not save_path:
+        # strip extra decimals
+        time_clean = re.sub(r"\.(\d)\d+", r".\1", from_time).replace(":", "")
+        save_path = video_path.with_name(f"{video_path.stem}_{time_clean}.gif")
+
+    info = ffmpeg.ffprobe(video_path)
+    scaled_width, scaled_height = width, math.ceil(width / info.ar / info.sample_ar)
+
+    subprocess.run(
+        [
+            "ffmpeg",
+            "-v",
+            "warning",
+            "-y",
+            "-ss",
+            from_time,
+            *(["-to", to_time] if to_time else ["-t", str(duration)]),
+            "-ignore_chapters",
+            "1",
+            "-i",
+            str(video_path),
+            "-filter_complex",
+            # f"fps={fps},scale={scale}:-1:flags=lanczos[x];[x]split[x1][x2];[x1]palettegen[p];[x2][p]paletteuse",
+            f"[0:v] fps={fps},scale={scaled_width}:{scaled_height},split [a][b];[a] palettegen=stats_mode=full [p];[b][p] paletteuse=new=1",
+            str(save_path),
+        ],
+        check=True,
+    )
+
+    return save_path
+
+
+def optimize_gif(gif_path: Path) -> Path:
+    subprocess.run(
+        [
+            "gifsicle",
+            "--lossy=98",
+            "--optimize=3",
+            "--batch",
+            "-i",
+            str(gif_path),
+        ],
+        check=True,
+    )
+    return gif_path
+
+
+def parse_args(argv: typing.List[str] = None) -> argparse.Namespace:
+    arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    arger.add_argument("video_path", type=str, help="Path to video file")
+    arger.add_argument(
+        "--start", "-s", dest="from_time", type=str, help="Time to start from, e.g. 02:59", required=True
+    )
+    arger.add_argument("--duration", "-d", default=3, type=int, help="Duration of gif in seconds")
+    arger.add_argument("--fps", default=15, type=int, help="Frames per second")
+    arger.add_argument("--width", default=400, type=int, help="Image width")
+    arger.add_argument("--optimize", default=True, action="store_true", help="Reduce GIF filesize")
+
+    if len(argv) == 0:
+        argv = ["--help"]
+
+    return arger.parse_args(argv)
+
+
+def md5(text: str) -> str:
+    return hashlib.md5(text.encode("utf-8")).hexdigest()
+
+
+def main():
+    args = parse_args(sys.argv[1:])
+    video_path = Path(args.video_path)
+
+    from_time = args.from_time
+    try:
+        time = datetime.timedelta(seconds=round(float(from_time), 1))
+        from_time = str(time)
+        if time.total_seconds() < 3600:
+            from_time = from_time[2:]
+    except ValueError:
+        pass
+
+    save_path = None
+    if str(args.video_path).startswith("http"):
+        save_path = Path("~/Downloads").expanduser() / f"{md5(str(video_path))}__{from_time.replace(':', '')}.gif"
+
+    gif_path = create_gif(
+        video_path=video_path,
+        save_path=save_path,
+        from_time=from_time,
+        duration=args.duration,
+        fps=args.fps,
+        width=args.width,
+    )
+    if args.optimize:
+        try:
+            gif_path = optimize_gif(gif_path)
+        except:
+            pass
+    print(gif_path.resolve())
+
+
+if __name__ == "__main__":
+    main()

ffmpeg_gif_summary.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3.9
+import argparse
+import datetime
+import math
+
+import subprocess
+import sys
+import typing
+from pathlib import Path
+import re
+
+import ffmpeg_gif
+
+def parse_duration(text: str) -> datetime.timedelta:
+    re_duration = re.compile(r"(\d+(?:\.\d+)?(?:sec|min|hour|hr|h|m|s))")
+    re_digits = re.compile(r"([\d.]+)")
+    time_scales = {
+        "s": 1,
+        "sec": 1,
+        "m": 60,
+        "min": 60,
+        "h": 3600,
+        "hr": 3600,
+    }
+
+    parts = re_duration.findall(text)
+    if parts:
+        total_secs = 0
+        for it in parts:
+            if m := re_digits.search(it):
+                _, end = m.span()
+                scale = it[end:]
+                total_secs += math.ceil(float(m.group(1)) * time_scales[scale])
+        return datetime.timedelta(seconds=total_secs)
+
+    parts = [float(it) for it in text.split(":")]
+    if len(parts) == 1:
+        return datetime.timedelta(seconds=parts[0])
+    if len(parts) == 2:
+        return datetime.timedelta(minutes=parts[0], seconds=parts[1])
+    if len(parts) == 3:
+        return datetime.timedelta(hours=parts[0], minutes=parts[1], seconds=parts[2])
+
+
+
+def parse_args(argv: typing.List[str] = None) -> argparse.Namespace:
+    arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    arger.add_argument("video_path", type=Path, help="Path to video file")
+    arger.add_argument("--interval", "-i", default=60, type=int, help="Interval between GIFs in seconds")
+    arger.add_argument("--duration", "-d", default=3, type=int, help="GIF duration in seconds")
+    arger.add_argument("--fps", default=18, type=float, help="Frames per second")
+    arger.add_argument("--width", default=400, type=int, help="Image width")
+    arger.add_argument("--optimize", default=True, action="store_true", help="Reduce GIF filesize")
+    arger.add_argument("--from", dest='from_time', type=parse_duration, help="Start from timestamp")
+    arger.add_argument("--to", dest='to_time', type=parse_duration, help="Stop at timestamp")
+
+    if len(argv) == 0:
+        argv = ["--help"]
+
+    return arger.parse_args(argv)
+
+
+def get_video_duration(video_path: Path) -> datetime.timedelta:
+    # fmt: off
+    args = [
+        'ffprobe',
+        '-v', 'error',
+        '-show_entries', 'format=duration',
+        '-of', 'default=noprint_wrappers=1:nokey=1',
+        video_path,
+    ]
+    # fmt: on
+    p = subprocess.run(args, stdout=subprocess.PIPE, check=True)
+    return datetime.timedelta(seconds=round(float(p.stdout), 1))
+
+
+def summarize(video_path: Path, width: int = 400, gif_duration: int = 3, fps: int = 15, interval_seconds: int = 60, optimize: bool = True,
+from_time: typing.Optional[datetime.timedelta] = None, to_time: typing.Optional[datetime.timedelta] = None) -> None:
+    duration = get_video_duration(video_path)
+    
+    if not from_time:
+        from_time = datetime.timedelta(minutes=2)
+    if not to_time:
+        to_time = duration
+
+    count = math.ceil((to_time - from_time).total_seconds() // interval_seconds)
+    interval = (to_time - from_time) / count
+    for i in range(count):
+        start = from_time + datetime.timedelta(seconds=round(interval.total_seconds() * i, 1))
+        end = start + datetime.timedelta(seconds=gif_duration)
+        if end >= to_time:
+            break
+        gif_path = ffmpeg_gif.create_gif(
+            video_path=video_path,
+            from_time=str(start),
+            to_time=str(end),
+            width=width,
+            fps=fps,
+            duration=duration,
+        )
+        if optimize:
+            gif_path = ffmpeg_gif.optimize_gif(gif_path)
+        print(str(gif_path.absolute()))
+
+def main():
+    args = parse_args(sys.argv[1:])
+    video_path: Path = args.video_path
+    summarize(
+        video_path=video_path,
+        width=args.width,
+        gif_duration=args.duration,
+        fps=args.fps,
+        optimize=args.optimize,
+        interval_seconds=args.interval,
+        from_time=args.from_time,
+        to_time=args.to_time,
+    )
+
+
+if __name__ == "__main__":
+    main()

ffmpeg_resolution.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3.9
+
+from pathlib import Path
+import subprocess
+import sys
+from ffmpeg import ffprobe
+import logging
+
+
+def get_resolution(video_path: Path) -> str:
+    try:
+        info = ffprobe(video_path)
+    except subprocess.CalledProcessError:
+        return None
+
+    if 700 <= info.height <= 800:
+        return "720p"
+    elif 1000 <= info.height <= 1200:
+        return "1080p"
+    elif info.height > 2000:
+        return "4K"
+    return None
+
+
+def main():
+    for it in sys.argv[1:]:
+        video_path = Path(it)
+        if not video_path.is_file():
+            continue
+        resolution = get_resolution(video_path)
+        if resolution is None:
+            logging.info("unknown resolution: %s", video_path)
+            continue
+        if resolution in video_path.stem:
+            continue
+
+        logging.info("%s: %s", video_path.stem, resolution)
+        new_path = video_path.with_stem(f"{video_path.stem} [{resolution}]")
+        video_path.rename(new_path)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    main()

ffmpeg_strip.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3.9
+import argparse
+import json
+import logging
+import subprocess
+import sys
+from pathlib import Path
+import typing
+
+
+def parse_args(argv: list[str]):
+    arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    arger.add_argument("video_path", type=Path, help="Video path")
+    arger.add_argument("save_path", nargs="?", type=Path, help="Output path")
+    if len(argv) == 0:
+        arger.print_help()
+        exit(1)
+
+    return arger.parse_args(argv)
+
+
+def ffprobe(video_path: Path) -> dict:
+    proc = subprocess.run(
+        [
+            "ffprobe",
+            "-v",
+            "quiet",
+            "-print_format",
+            "json",
+            "-show_format",
+            "-show_streams",
+            str(video_path),
+        ],
+        check=True,
+        capture_output=True,
+    )
+    return json.loads(proc.stdout)
+
+
+def clean_video(video_path: Path, save_path: Path) -> Path:
+    media_info = ffprobe(video_path)
+    # check if the first video stream has hevc codec
+    if media_info["streams"][0]["codec_name"] == "hevc":
+        logging.info(f"Video codec is hevc, adding hvc1 tag")
+        extra_args = ["-tag:v", "hvc1"]
+    else:
+        extra_args = []
+
+    # fmt: off
+    args = [
+        'ffmpeg',
+        '-i',
+        str(video_path),
+        '-c:v', 'copy',
+        # '-movflags', '+faststart', 
+        '-map_metadata', '-1',
+        *extra_args,
+        '-c:a', 'copy',
+        '-y',
+        str(save_path),
+    ]
+    # fmt: on
+    logging.info(f"calling ffmpeg with {args=}")
+    subprocess.run(args, check=True)
+    return save_path
+
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+    args = parse_args(sys.argv[1:])
+    save_path: typing.Optional[Path] = args.save_path
+    if save_path and save_path.resolve().is_dir():
+        save_path = save_path / args.video_path.name
+    if save_path is None:
+        save_path = args.video_path.with_suffix(".clean.mp4")
+
+    clean_video(video_path=args.video_path, save_path=save_path)
+
+
+if __name__ == "__main__":
+    main()

file_renamer.py

@@ -0,0 +1,304 @@
+#!/usr/bin/env python3.10
+import argparse
+import dataclasses
+import datetime
+import json
+import logging
+import re
+import subprocess
+import typing
+from pathlib import Path
+
+script_path = Path(__file__)
+db_filename = script_path.with_name(f'{script_path.stem}.known.txt')
+
+
+def save_actors(actors: typing.Iterable[str]):
+    existing = load_actors()
+    all_actors = sorted({*existing, *actors})
+    with db_filename.open('w', encoding='utf-8', newline='\n') as f:
+        f.writelines(all_actors)
+
+
+def load_actors() -> set[str]:
+    try:
+        with db_filename.open('w', encoding='utf-8', newline='\n') as f:
+            return {line.strip() for line in f}
+    except FileNotFoundError:
+        return set()
+
+
+re_date_us = re.compile(r"\b(\d{2})\D(\d{2})\D(\d{2})\b")  # 12/31/21
+re_date_iso = re.compile(r"\b(\d{4})\D?(\d{2})\D?(\d{2})\b")  # 2020-12-31
+re_date_iso_short = re.compile(r"(\d{2})\D?(\d{2})\D?(\d{2})")  # 20-12-31"'
+re_date_iso_rev = re.compile(r"\b(\d{2})\D?(\d{2})\D?(\d{4})\b")  # 31-12-2020
+re_date_iso_rev_short = re.compile(r"\b(\d{2})\D(\d{2})\D(\d{2})\b")  # 31.12.21
+
+
+class Pattern:
+    def __init__(self, pattern: re.Pattern):
+        self.re = pattern if isinstance(pattern, re.Pattern) else re.compile(pattern)
+
+    def __eq__(self, other):
+        if isinstance(other, str):
+            return self.re.search(other)
+        raise ValueError
+
+
+@dataclasses.dataclass
+class ParsedDate:
+    date: datetime.date
+    start: int
+    end: int
+
+
+def parse_date(filename: str) -> ParsedDate | None:
+    def from_us_format():
+        match = re_date_us.search(filename)
+        m, d, y = map(int, match.groups())
+        return ParsedDate(datetime.date(y + 2000, m, d), start=match.start(), end=match.end())
+
+    def from_iso():
+        match = re_date_iso.search(filename)
+        y, m, d = map(int, match.groups())
+        return ParsedDate(datetime.date(y, m, d), start=match.start(), end=match.end())
+
+    def from_iso_short():
+        match = re_date_iso_short.search(filename)
+        y, m, d = map(int, match.groups())
+        return ParsedDate(datetime.date(y + 2000, m, d), start=match.start(), end=match.end())
+
+    def from_iso_reversed():
+        match = re_date_iso_rev.search(filename)
+        d, m, y = map(int, match.groups())
+        return ParsedDate(datetime.date(y, m, d), start=match.start(), end=match.end())
+
+    def from_iso_reversed_short():
+        match = re_date_iso_rev_short.search(filename)
+        d, m, y = map(int, match.groups())
+        return ParsedDate(datetime.date(y + 2000, m, d), start=match.start(), end=match.end())
+
+    candidates = []
+    for fn in [
+        from_us_format,
+        from_iso,
+        from_iso_short,
+        from_iso_reversed,
+        from_iso_reversed_short,
+    ]:
+        try:
+            candidates.append(fn())
+        except (ValueError, AttributeError):
+            pass
+
+    today = datetime.date.today()
+    future_threshold = today + datetime.timedelta(days=60)
+    past_threshold = datetime.date(2010, 1, 1)
+    candidates = [it for it in candidates if past_threshold < it.date < future_threshold]
+
+    if not candidates:
+        return
+
+    return candidates[0]
+
+
+@dataclasses.dataclass
+class Release:
+    actors: list[str]
+    studio: str | None = None
+    title: str | None = None
+    released_at: datetime.date | None = None
+
+    def to_filename(self):
+        parts = []
+        if self.actors:
+            parts.append(', '.join(self.actors))
+        if self.studio:
+            parts.append(f'@{self.studio}')
+        if self.title:
+            parts.append(self.title)
+        if self.released_at:
+            parts.append(self.released_at.isoformat())
+        return ' -- '.join(parts)
+
+
+def trash(path: Path):
+    filename = f'"{path}"'
+    cmd = ['osascript', '-e', f'tell app "Finder" to move (POSIX file {filename}) to trash']
+    subprocess.run(cmd).check_returncode()
+
+
+def filenames_to_actors():
+    sources = [
+        Path(r'/Users/abdus/Downloads/temp/'),
+        Path(r'/Volumes/BANDAID/_temp/__reenc/'),
+        Path(r'/Volumes/BANDAID/_temp/'),
+    ]
+    actors = set()
+    for it in sources:
+        for f in it.glob('*.mp4'):
+            if not f.is_file():
+                continue
+            if r := parse_release(f.name):
+                actors.update(r.actors)
+    return actors
+
+
+def parse_release(filename: str, known_actors: set[str] | None = None):
+    if not known_actors:
+        known_actors = set()
+    filename = re.sub(r'\.\w{3,4}$', '', filename)
+
+    def from_own():
+        remaining = re.sub(r'\s+\[[^]]+]$', '', filename)
+        remaining = re.sub(r'\s+\[([^]]+|\d+\w)(,\s*[^]]+)?]', '', remaining)
+        remaining = re.sub(r'\s*--\s*', ' -- ', remaining)
+
+        match remaining.split(' -- '):
+            case [actors, studio, title, date] if studio.startswith('@'):
+                return Release(
+                    actors=sorted(actors.split(', ')),
+                    studio=studio.removeprefix('@'),
+                    title=title,
+                    released_at=datetime.date.fromisoformat(date),
+                )
+            case [actors, studio, date] if studio.startswith('@') and (parsed := parse_date(date)):
+                return Release(
+                    actors=sorted(actors.split(', ')),
+                    studio=studio.removeprefix('@'),
+                    title=None,
+                    released_at=parsed.date,
+                )
+            case [actors, studio, title] if studio.startswith('@'):
+                return Release(
+                    actors=sorted(actors.split(', ')),
+                    studio=studio.removeprefix('@'),
+                    title=title,
+                    released_at=None,
+                )
+            case [actors, title, date] if (parsed := parse_date(date)):
+                return Release(
+                    actors=sorted(actors.split(', ')),
+                    studio=None,
+                    title=title,
+                    released_at=parsed.date,
+                )
+            case [actors, studio] if studio.startswith('@'):
+                return Release(
+                    actors=sorted(actors.split(', ')),
+                    studio=studio,
+                )
+            case [actors, title]:
+                return Release(
+                    actors=sorted(actors.split(', ')),
+                    title=title,
+                )
+
+    def from_prt():
+        assert '.PRT' in filename
+        remaining = re.sub(r'\.(720p|1080p|HEVC|x265|PRT|XXX)', ' ', filename)
+        studio = remaining[:remaining.index('.')]
+        parsed_date = parse_date(remaining)
+        remaining = remaining[parsed_date.end:]
+        remaining = re.sub(r'[. ]+', ' ', remaining).strip()
+
+        actors = []
+        title = None
+
+        match remaining.split(' '):
+            case [a_first, a_last, 'And', b_first, b_last]:
+                actors = [f'{a_first} {a_last}', f'{b_first} {b_last}']
+            case [a_first, a_last, 'And', b_first, b_last, *rest]:
+                actors = [f'{a_first} {a_last}', f'{b_first} {b_last}']
+                title = ' '.join(rest)
+            case [first, last]:
+                actors = [f'{first} {last}']
+            case [first, last, *rest]:
+                actors = [f'{first} {last}']
+                title = ' '.join(rest)
+            case [first]:
+                actors = [first]
+
+        return Release(
+            studio=studio,
+            actors=sorted(actors),
+            title=title,
+            released_at=parsed_date.date,
+        )
+
+    def from_galaxxxy():
+        pass
+
+    for fn in [
+        from_own,
+        from_prt,
+        from_galaxxxy,
+    ]:
+        try:
+            if res := fn():
+                return res
+        except (ValueError, AttributeError, AssertionError):
+            pass
+
+
+def parse_args():
+    arger = argparse.ArgumentParser()
+    arger.add_argument('filenames', nargs='+', type=lambda v: Path(v).resolve(), help='Filenames')
+    return arger.parse_args()
+
+
+@dataclasses.dataclass
+class MoveOp:
+    source: Path
+    target: Path
+
+    def to_dict(self) -> dict:
+        return {
+            'source': str(self.source),
+            'target': str(self.target),
+        }
+
+
+def write_undo_script(save_dir: Path, ops: list[MoveOp]):
+    now = datetime.datetime.now().isoformat().replace(':', '')
+    save_path = save_dir / f'undo_{now}.json'
+    with save_path.open('w', encoding='utf-8') as f:
+        for it in ops:
+            f.write(json.dumps(it.to_dict()) + '\n')
+
+def fix_filename():
+    pass
+
+
+def main():
+    actors = load_actors()
+
+    args = parse_args()
+    paths: list[Path] = args.filenames
+
+    jobs = []
+    for it in paths:
+        if parsed := parse_release(it.name, known_actors=actors):
+            logging.debug(f'filename={it} release={parsed}')
+            suggested = parsed.to_filename()
+            target = it.with_stem(suggested)
+            if target.is_file():
+                logging.error('target already exists')
+                continue
+            jobs.append(MoveOp(source=it, target=target))
+
+    if not jobs:
+        return
+
+    write_undo_script(paths[0].parent, jobs)
+
+    for it in jobs:
+        it.target.hardlink_to(it.source)
+
+    for it in jobs:
+        trash(it.source)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.DEBUG)
+    main()

file_renamer_test.py

@@ -0,0 +1,120 @@
+import datetime
+
+import pytest
+
+import file_renamer
+from file_renamer import Release
+
+
+@pytest.mark.parametrize(
+    ["filename", "date"],
+    [
+        [
+            "Tushy.21.09.13.Jia.Lissa.XXX.720p.WEB.x264-GalaXXXy",
+            datetime.date(2021, 9, 13),
+        ],
+        [
+            "[AdultTime] Aidra Fox (Lady Gonzo - 28.02.2019) [720p HEVC].mkv",
+            datetime.date(2019, 2, 28),
+        ],
+        [
+            "15.01.10_-_Aspen_Ora_-_Cock_Crazy_-_x265.mp4",
+            datetime.date(2015, 1, 10),
+        ],
+        [
+            "Karma Rx, Valerica Steele - Valerica & Karma’s Anal Slutfest -- AllAnal 20210912",
+            datetime.date(2021, 9, 12),
+        ],
+        [
+            "Scarlett Hampton - Taking Scarlett’s Ass For a Ride 09_15_21.mp4",
+            datetime.date(2021, 9, 15),
+        ],
+        [
+            "Sia_Siberia & LittleReislin - Best Valentine's Day gift ever Release Date _ 12_10_20.mp4",
+            datetime.date(2020, 12, 10),
+        ],
+        [
+            "Kenna James - Succubus 09_09_21.mp4",
+            datetime.date(2021, 9, 9),
+        ],
+        [
+            "Hot Bitch Dreaming about being Hard Fucked by her Stepbrother - Diana Daniels_Diana Daniels_720p.mp4",
+            None,
+        ],
+        [
+            "DadCrush.21.10.09.Ailee.Anne.My.Stepdaughters.Hot.XXX.1080p.HEVC.x265.PRT.mp4",
+            datetime.date(2021, 10, 9)
+        ]
+    ],
+)
+def test_parse_date(filename: str, date: datetime.date | None):
+    parsed = file_renamer.parse_date(filename)
+
+    if not date:
+        assert parsed is None
+        return
+
+    assert parsed.date == date
+
+
+@pytest.mark.parametrize(['filename', 'expected'], [
+    [
+        'Rebecca Volpetti -- @RealityKings -- Driving Him Crazy [1080p, x265] -- 2019-04-10',
+        Release(actors=['Rebecca Volpetti'], studio='RealityKings', title='Driving Him Crazy', released_at=datetime.date(2019, 4, 10)),
+    ],
+    [
+        'Liz Jordan -- @Lubed -- Sopping Oil -- 2023-02-07 [1080p, x265]',
+        Release(actors=['Liz Jordan'], studio='Lubed', title='Sopping Oil', released_at=datetime.date(2023, 2, 7)),
+    ],
+    [
+        'Liz Jordan--@Lubed--Sopping Oil--2023-02-07 [1080p, x265]',
+        Release(actors=['Liz Jordan'], studio='Lubed', title='Sopping Oil', released_at=datetime.date(2023, 2, 7)),
+    ],
+    [
+        'Liz Jordan -- @Lubed -- 2023-02-07 [1080p, x265]',
+        Release(actors=['Liz Jordan'], studio='Lubed', title=None, released_at=datetime.date(2023, 2, 7)),
+    ],
+    [
+        'Liz Jordan -- Sopping Oil -- 2023-02-07 [1080p, x265]',
+        Release(actors=['Liz Jordan'], studio=None, title='Sopping Oil', released_at=datetime.date(2023, 2, 7)),
+    ],
+    [
+        'Liz Jordan -- Sopping Oil',
+        Release(actors=['Liz Jordan'], studio=None, title='Sopping Oil', released_at=None),
+    ],
+    [
+        'DadCrush.21.10.09.Ailee.Anne.My.Stepdaughters.Hot.XXX.1080p.HEVC.x265.PRT.mp4',
+        Release(studio='DadCrush', actors=['Ailee Anne'], title='My Stepdaughters Hot', released_at=datetime.date(2021, 10, 9))
+    ],
+    [
+        'Deeper.21.11.11.Kenzie.Anne.XXX.1080p.HEVC.x265.PRT.mkv',
+        Release(studio='Deeper', actors=['Kenzie Anne'], title=None, released_at=datetime.date(2021, 11, 11)),
+    ],
+    [
+        'DevilsFilm.22.03.19.Kira.Noir.Wife.Swap.Schemes.2.XXX.1080p.HEVC.x265.PRT.mkv',
+        Release(studio='DevilsFilm', actors=['Kira Noir'], title='Wife Swap Schemes 2', released_at=datetime.date(2022, 3, 19)),
+    ],
+    [
+        'DogHouseDigital.22.04.06.Maddy.May.Full.Service.Massage.XXX.1080p.HEVC.x265.PRT.mkv',
+        Release(studio='DogHouseDigital', actors=['Maddy May'], title='Full Service Massage', released_at=datetime.date(2022, 4, 6)),
+    ],
+    [
+        'EvilAngel.22.04.05.Diana.Grace.XXX.1080p.HEVC.x265.PRT.mkv',
+        Release(studio='EvilAngel', actors=['Diana Grace'], title=None, released_at=datetime.date(2022, 4, 5)),
+    ],
+    [
+        'Slayed.21.10.07.Izzy.Lush.And.Aidra.Fox.XXX.1080p.HEVC.x265.PRT.mkv',
+        Release(studio='Slayed', actors=['Aidra Fox', 'Izzy Lush'], title=None, released_at=datetime.date(2021, 10, 7)),
+    ],
+    [
+        'ExploitedCollegeGirls.22.06.30.Gaby.XXX.1080p.HEVC.x265.PRT.mkv',
+        Release(studio='ExploitedCollegeGirls', actors=['Gaby'], title=None, released_at=datetime.date(2022, 6, 30)),
+    ],
+    [
+        'DoctorAdventures.21.05.26.Jamie.Michelle.Nurse.Jamie.Knows.Best.PRT.mp4',
+        Release(studio='DoctorAdventures', actors=['Jamie Michelle'], title='Nurse Jamie Knows Best', released_at=datetime.date(2021, 5, 26)),
+    ],
+])
+def test_parse_release(filename: str, expected):
+    parsed = file_renamer.parse_release(filename)
+    assert parsed == expected

fpup.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3.9
+import argparse
+import concurrent.futures
+import json
+import re
+import sys
+import typing
+from pathlib import Path
+from typing import Optional
+import logging
+
+
+import httpx
+
+
+def upload_image(image_path: Path) -> dict:
+    session = httpx.Client(timeout=20)
+    logging.info(f"uploading {image_path.name}")
+
+    with image_path.open("rb") as f:
+        res = session.post(
+            "https://fapping.empornium.sx/upload.php",
+            files={
+                "ImageUp": f,
+            },
+        )
+
+    res = session.get("https://fapping.empornium.sx/uploaded/", follow_redirects=True)
+    if m := re.search(r"var ImagesUp = (\[.+\]);", res.text, re.MULTILINE):
+        data: list[dict] = json.loads(m.group(1))
+        logging.info(f"upload result on the page={data}")
+
+        return dict(
+            image_url=data[0]["image_url"],
+            thumbnail_url=data[0]["image_thumb_url"],
+        )
+
+    logging.error(f"failed to find upload result on the page. html={res.text}")
+
+    raise ValueError(f"failed to parse upload result for {image_path.name}")
+
+
+def parse_args(argv: Optional[typing.Sequence[str]] = None) -> argparse.Namespace:
+    arger = argparse.ArgumentParser()
+    arger.add_argument("image_paths", nargs="+", action="store", type=Path, help="Path to image to upload")
+    if not argv or len(argv) == 0:
+        arger.print_help()
+        sys.exit(1)
+
+    return arger.parse_args(argv)
+
+
+def main():
+    args = parse_args(sys.argv[1:])
+    image_paths = args.image_paths
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
+        results = list(pool.map(upload_image, image_paths))
+    for it in results:
+        print(it["image_url"])
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    main()

gardener.py

@@ -0,0 +1,207 @@
+import dataclasses
+import logging
+import os
+import webbrowser
+
+
+def force_import(module: str):
+    import importlib
+    import subprocess
+    import sys
+
+    try:
+        return importlib.import_module(module)
+    except ModuleNotFoundError:
+        subprocess.run([sys.executable, "-m", "pip", "install", module])
+        importlib.invalidate_caches()
+        return importlib.import_module(module)
+
+
+try:
+    import httpx
+except ImportError:
+    torf = force_import("httpx")
+
+try:
+    import typer
+except ImportError:
+    typer = force_import("typer")
+try:
+    import rich
+except ImportError:
+    rich = force_import("rich")
+try:
+    import inquirer
+except ImportError:
+    inquirer = force_import("inquirer")
+
+from rich import print
+import rich.table
+
+JIRA_EMAIL = os.getenv('JIRA_EMAIL', 'abdussamet.kocak@akinon.com')
+JIRA_TOKEN = os.getenv('JIRA_TOKEN', 'iAZhlJ1AdGwckbVllhlx48C4')
+JIRA_URL = os.getenv('JIRA_URL', 'https://omniplatform.atlassian.net/')
+
+if not JIRA_TOKEN:
+    logging.info('')
+    webbrowser.open('https://id.atlassian.com/manage-profile/security/api-tokens')
+
+
+@dataclasses.dataclass
+class Issue:
+    id: str
+    key: str
+    summary: str
+
+
+@dataclasses.dataclass
+class PullRequest:
+    id: str
+    name: str
+    repository: str
+    branch: str
+
+
+class Jira:
+    def __init__(self, email: str, token: str):
+        self.client = httpx.Client(
+            base_url=JIRA_URL,
+            auth=httpx.BasicAuth(email, token),
+        )
+
+    def list_issues_for_release(self) -> list[Issue]:
+        mergeable_status = 'In Review'
+        res = self.client.post(
+            '/rest/api/2/search',
+            json={
+                'jql': f'project = COM AND status = "{mergeable_status}" AND "Team[Dropdown]" = Backend ORDER BY created DESC'
+            },
+        )
+
+        res.raise_for_status()
+        data = res.json()
+        development_type = ['Story', 'Bug', 'Task']
+        return [
+            Issue(id=it['id'], key=it['key'], summary=it['fields']['summary'])
+            for it in data['issues']
+            if it['fields']['issuetype']['name'] in development_type
+        ]
+
+    def get_approved_prs(self, issue_id: str):
+        res = self.client.post(
+            f'/jsw/graphql',
+            params={'operation': 'DevDetailsDialog'},
+            json={
+                "operationName": "DevDetailsDialog",
+                "query": """
+                    query DevDetailsDialog ($issueId: ID!) {
+                        developmentInformation(issueId: $issueId){
+                            details {
+                                instanceTypes {
+                                    repository {
+                                        name
+                                        branches {
+                                            name
+                                            pullRequests {
+                                                name
+                                                status
+                                                lastUpdate
+                                            }
+                                            reviews {
+                                                state
+                                                id
+                                            }
+                                        }
+                                        pullRequests {
+                                            id
+                                            name
+                                            branchName
+                                            status
+                                            reviewers{
+                                                name
+                                                isApproved
+                                            }
+                                        }
+                                    }
+                                    danglingPullRequests {
+                                        id
+                                        name
+                                        branchName
+                                        status
+                                        reviewers{
+                                            name
+                                            isApproved
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }    
+                """,
+                "variables": {"issueId": issue_id},
+            },
+        )
+        res.raise_for_status()
+        data = res.json()['data']
+        if not data['developmentInformation']['details']['instanceTypes']:
+            return []
+        repo_name = data['developmentInformation']['details']['instanceTypes'][0]['repository'][0]['name']
+        prs = data['developmentInformation']['details']['instanceTypes'][0]['danglingPullRequests']
+        if not prs:
+            repo_name = data['developmentInformation']['details']['instanceTypes'][0]['repository'][0]['name']
+            prs = data['developmentInformation']['details']['instanceTypes'][0]['repository'][0]['pullRequests']
+
+        return [
+            PullRequest(
+                id=it['id'],
+                name=it['name'],
+                repository=repo_name,
+                branch=it['branchName'],
+            )
+            for it in prs
+            if it['status'] != 'MERGED'
+        ]
+
+
+console = rich.console.Console()
+
+
+def render_issue(issue: Issue, prs: list[PullRequest]):
+    t = rich.table.Table(
+        title=f'{issue.key} -- {issue.summary}',
+        caption_justify='left',
+    )
+    t.add_column("Repository", no_wrap=True)
+    t.add_column("Branch", no_wrap=True)
+    t.add_column("Pull Request")
+
+    for it in prs:
+        t.add_row(it.repository, it.branch, it.name)
+
+    console.print(t)
+
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+    if not JIRA_EMAIL:
+        logging.error('JIRA_EMAIL is not set')
+        exit(1)
+    if not JIRA_TOKEN:
+        logging.error('JIRA_TOKEN is not set')
+        exit(1)
+
+    j = Jira(email=JIRA_EMAIL, token=JIRA_TOKEN)
+
+    logging.info('fetching issues in "ready to release" status')
+    issues = j.list_issues_for_release()
+    for it in issues:
+        logging.info(f'fetching pull requests for issue {it.key}')
+        prs = j.get_approved_prs(it.id)
+        if not prs:
+            logging.info(f'no non-merged pull requests found for issue {it.key}')
+            continue
+        render_issue(it, prs)
+
+
+if __name__ == '__main__':
+    main()

hetzner_kube.py

@@ -0,0 +1,237 @@
+#!/usr/bin/env python3.9
+import base64
+import json
+import logging
+import pprint
+import subprocess
+import time
+import webbrowser
+from pathlib import Path
+
+import httpx
+import typer
+
+hetzner_token = 'zdlkmAF9EswvOIEGvWgutULYTmUpa3c44YcKZOB0jymQfgvra7kRVJxuLiFTouM9'
+client = httpx.Client(base_url='https://api.hetzner.cloud/v1', headers={'Authorization': f'Bearer {hetzner_token}'})
+
+
+datacenters = [
+    {'id': 2, 'name': 'nbg1-dc3', 'description': 'Nuremberg 1 DC 3'},
+    {'id': 3, 'name': 'hel1-dc2', 'description': 'Helsinki 1 DC 2'},
+    {'id': 4, 'name': 'fsn1-dc14', 'description': 'Falkenstein 1 DC14'},
+    {'id': 5, 'name': 'ash-dc1', 'description': 'Ashburn DC1'},
+]
+
+server_types = {
+    'cx11': {'cores': 1, 'memory': 2.0, 'disk': 20},
+    'cx21': {'cores': 2, 'memory': 4.0, 'disk': 40},
+    'cx31': {'cores': 2, 'memory': 8.0, 'disk': 80},
+    'cx41': {'cores': 4, 'memory': 16.0, 'disk': 160},
+    'cx51': {'cores': 8, 'memory': 32.0, 'disk': 240},
+    'ccx11': {'cores': 2, 'memory': 8.0, 'disk': 80},
+    'ccx21': {'cores': 4, 'memory': 16.0, 'disk': 160},
+    'ccx31': {'cores': 8, 'memory': 32.0, 'disk': 240},
+    'ccx41': {'cores': 16, 'memory': 64.0, 'disk': 360},
+    'ccx51': {'cores': 32, 'memory': 128.0, 'disk': 600},
+    'cpx11': {'cores': 2, 'memory': 2.0, 'disk': 40},
+    'cpx21': {'cores': 3, 'memory': 4.0, 'disk': 80},
+    'cpx31': {'cores': 4, 'memory': 8.0, 'disk': 160},
+    'cpx41': {'cores': 8, 'memory': 16.0, 'disk': 240},
+    'cpx51': {'cores': 16, 'memory': 32.0, 'disk': 360},
+    'ccx12': {'cores': 2, 'memory': 8.0, 'disk': 80},
+    'ccx22': {'cores': 4, 'memory': 16.0, 'disk': 160},
+    'ccx32': {'cores': 8, 'memory': 32.0, 'disk': 240},
+    'ccx42': {'cores': 16, 'memory': 64.0, 'disk': 360},
+    'ccx52': {'cores': 32, 'memory': 128.0, 'disk': 600},
+    'ccx62': {'cores': 48, 'memory': 192.0, 'disk': 960},
+}
+
+
+def create_server(name: str, datacenter: str):
+    payload = {
+        "datacenter": datacenter,
+        "image": "ubuntu-20.04",
+        "name": name,
+        "server_type": "cpx21",
+        "ssh_keys": ["entropy"],
+    }
+
+    res = client.post('/servers', json=payload)
+    res.raise_for_status()
+
+    data = res.json()
+    server = data['server']
+    pprint.pprint(server)
+
+    return {
+        'id': server['id'],
+        'name': server['name'],
+        'ip': server['public_net']['ipv4']['ip'],
+    }
+
+
+def find_server(name: str) -> dict:
+    res = client.get('/servers')
+    res.raise_for_status()
+
+    data = res.json()
+
+    server = [it for it in data['servers'] if it['name'] == name][0]
+    return {
+        'id': server['id'],
+        'name': server['name'],
+        'ip': server['public_net']['ipv4']['ip'],
+    }
+
+
+def delete_server(name: str) -> None:
+    server = find_server(name)
+    logging.info(f'deleting server={name} with id={server["id"]}')
+
+    res = client.delete(f'/servers/{server["id"]}')
+    res.raise_for_status()
+
+    logging.info(f'deleted server={name}')
+
+
+def copy_to_clipboard(text: str) -> None:
+    subprocess.Popen(['pbcopy'], stdin=subprocess.PIPE).communicate(input=text.encode())
+
+
+class Server:
+    ssh_options = ['-o', 'StrictHostKeyChecking=no']
+
+    def __init__(self, ip_address: str, name: str = None):
+        self.ip = ip_address
+        self.ssh_host = f'root@{self.ip}'
+        self.name = name
+
+    @classmethod
+    def from_hetzner(cls, server: dict):
+        return cls(server['ip'], name=server['name'])
+
+    def run_command(self, cmd: str, return_output: bool = False) -> str:
+
+        logging.debug(f'running command={cmd} on {self.ssh_host}')
+        extra_args = {}
+        if return_output:
+            extra_args['stdout'] = subprocess.PIPE
+        p = subprocess.run(['ssh', *self.ssh_options, self.ssh_host, '--', cmd], text=True, **extra_args)
+        p.check_returncode()
+        return p.stdout
+
+    def install_package(self, packages: str) -> str:
+        return self.run_command(f'apt-get install -y {packages}')
+
+    def install_shell(self):
+        self.install_package('fish')
+        self.run_command('chsh -s /usr/bin/fish')
+
+    def install_microk8s(self):
+        self.install_package('snapd')
+        self.run_command('snap install microk8s --classic')
+        self.run_command('ufw allow in on cni0 && ufw allow out on cni0 && ufw default allow routed')
+        self.run_command('microk8s enable dashboard dns rbac ingress storage prometheus registry')
+
+    def get_microk8s_token(self):
+        raw_json = self.run_command('microk8s kubectl -n kube-system get secret -o json', return_output=True)
+        secrets = json.loads(raw_json)['items']
+
+        default_token_secret = [it for it in secrets if it['metadata']['name'].startswith('default-token')][0]
+        token_base64 = default_token_secret['data']['token']
+
+        token = base64.decodebytes(token_base64.encode()).decode()
+        return token
+
+    def show_microk8s_access_info(self):
+        logging.info('getting default service account token')
+        token = self.get_microk8s_token()
+        print(token)
+        logging.info('copying token to clipboard')
+        copy_to_clipboard(token)
+
+        logging.info('getting dashboard ip')
+        proxy = self.get_microk8s_dashboard_proxy()
+        print(proxy['ssh'])
+        copy_to_clipboard(proxy['ssh'])
+
+        logging.info('opening url in the browser')
+        print(proxy['url'])
+        webbrowser.open_new_tab(proxy['url'])
+
+    def get_microk8s_dashboard_proxy(self):
+        cluster_ip = self.run_command(
+            'microk8s kubectl get -n kube-system -o json service/kubernetes-dashboard -o jsonpath="{.spec.clusterIP}"',
+            return_output=True,
+        ).strip()
+        port, local_port = 443, 18433
+
+        return {
+            'ssh': f'ssh -vNL {local_port}:{cluster_ip}:{port} {self.ssh_host}',
+            'url': f'https://localhost:{local_port}',
+        }
+
+    def write_kubeconfig(self, filename: str):
+        logging.info(f'writing kubeconfig to {filename}')
+        kubeconfig = self.run_command('microk8s config', return_output=True)
+        file_path = Path(filename).expanduser()
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+        file_path.write_text(kubeconfig)
+
+    def install_micro_editor(self):
+        logging.info('installing micro editor')
+        self.run_command('curl https://getmic.ro | bash && mv ./micro /usr/bin/')
+
+    def setup(self):
+        self.run_command('apt-get update')
+
+        self.install_shell()
+        self.install_micro_editor()
+        self.install_microk8s()
+
+        logging.info('waiting until pods are online')
+        time.sleep(10)
+
+        self.show_microk8s_access_info()
+        self.write_kubeconfig(f'~/Desktop/kubeconfig-{self.name}.yml')
+
+
+cli = typer.Typer(invoke_without_command=False)
+
+
+@cli.command()
+def create(
+    name: str,
+    datacenter: str = typer.Option('nbg1-dc3'),
+):
+    create_server(name=name, datacenter=datacenter)
+
+
+@cli.command()
+def destroy(
+    name: str,
+):
+    delete_server(name=name)
+
+
+@cli.command()
+def setup(
+    name: str,
+):
+    server = find_server(name)
+    server = Server.from_hetzner(server)
+    server.setup()
+
+
+@cli.command()
+def show_k8s(
+    name: str,
+):
+    server = Server.from_hetzner(find_server(name))
+    server.show_microk8s_access_info()
+    server.write_kubeconfig(f'~/Desktop/kubeconfig-{server.name}.yml')
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.INFO)
+    cli()

hooksy/app.py

@@ -0,0 +1,80 @@
+import datetime
+import importlib
+import json
+import logging
+import typing
+import uuid
+from typing import Any
+
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from starlette import status
+import sentry_sdk
+from sentry_sdk.integrations.asgi import SentryAsgiMiddleware
+import seqlog
+
+app = FastAPI()
+
+
+class JsonDateEncoder(json.JSONEncoder):
+    def default(self, o: Any) -> Any:
+        if isinstance(o, datetime.datetime):
+            return o.isoformat()
+        return super().default(o)
+
+
+seqlog.log_to_seq(
+    server_url="http://log.abdus.dev:5341/",
+    api_key="ePewcJUxe8dST5EiC8Pw",
+    level=logging.INFO,
+    batch_size=10,
+    auto_flush_timeout=10,  # seconds
+    override_root_logger=True,
+    json_encoder_class=JsonDateEncoder,  # Optional; only specify this if you want to use a custom JSON encoder
+)
+sentry_sdk.init(
+    "https://ccb81025166c449d87e0646cede48448@o271257.ingest.sentry.io/6040020",
+    # Set traces_sample_rate to 1.0 to capture 100%
+    # of transactions for performance monitoring.
+    # We recommend adjusting this value in production.
+    traces_sample_rate=1.0,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class HookResponse(BaseModel):
+    id: str
+    result: typing.Optional[dict] = None
+    error: typing.Optional[str] = None
+
+
+@app.post("/{event}", response_model=HookResponse)
+def receive_hook(event: str, payload: dict):
+    logger.info(f"Received hook request for {event=} with {payload=}")
+    hook_id = uuid.uuid4().hex
+
+    try:
+        module = importlib.import_module(f"scripts.{event}")
+        func = getattr(module, "main")
+    except ModuleNotFoundError:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No such handler")
+
+    if not callable(func):
+        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Invalid script interface")
+
+    try:
+        logger.info(f"Running hook handler {func=}")
+        result = func(payload)
+        return HookResponse(id=hook_id, result=result, error=None)
+    except Exception as e:
+        logger.error(f"Hook handler threw error", exc_info=True)
+        return HookResponse(id=hook_id, result=None, error=str(e))
+
+
+app = SentryAsgiMiddleware(app)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app)

hooksy/scripts/handle_aria2_completed.py

@@ -0,0 +1,21 @@
+import logging
+import typing
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+Payload = typing.TypedDict(
+    "Payload",
+    {
+        "gid": str,
+        "num_files": int,
+        "path": str,
+        "files": list[typing.TypedDict("File", {"path": str, "size": int})],
+    },
+)
+
+
+def main(payload: Payload):
+    logging.info("got {torrent}", torrent=payload)
+    path = Path(payload["path"])

hooksy/scripts/handle_downloads.py

@@ -0,0 +1,26 @@
+import logging
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+def main(payload: dict = None):
+    if not payload:
+        payload = {}
+
+    links: list[dict] = payload.get("links", [])
+    if not links:
+        logger.info("No links, exiting")
+        return
+
+    logger.info("Got %d links", len(links))
+
+    for it in links:
+        download_path = Path(it["download_path"])
+        url: str = it.get("url", it.get("content_url"))
+        logger.info("Processing downloaded link {url} at {path}", path=download_path, url=url)
+
+        if download_path.suffix.lower() in {".zip", ".rar"}:
+            logger.info("Got an imageset!")
+
+    # raise Exception("oops!")

imgsheet.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python3.9
+import argparse
+import dataclasses
+import io
+import itertools
+import logging
+import math
+import statistics
+import sys
+import typing
+from pathlib import Path
+
+from PIL import Image
+from PIL import ImageDraw
+from PIL import ImageFont
+
+T = typing.TypeVar("T")
+
+logger = logging.getLogger("imgsheet")
+
+
+def chunk(it: typing.Iterable[T], size: int) -> typing.Iterable[typing.Iterable[T]]:
+    it = iter(it)
+    sentinel = ()
+    return iter(lambda: tuple(itertools.islice(it, size)), sentinel)
+
+
+@dataclasses.dataclass
+class PositionedImage:
+    path: Path
+    image_size: typing.Tuple[int, int]
+    size: typing.Tuple[int, int] = dataclasses.field(default_factory=lambda: (0, 0))
+    position: typing.Tuple[int, int] = dataclasses.field(default_factory=lambda: (0, 0))
+
+    @property
+    def top(self) -> int:
+        return self.position[1]
+
+    @property
+    def left(self) -> int:
+        return self.position[0]
+
+    @property
+    def bottom(self) -> int:
+        return self.height + self.top
+
+    @property
+    def right(self) -> int:
+        return self.width + self.left
+
+    @property
+    def width(self) -> int:
+        return self.size[0]
+
+    @property
+    def height(self) -> int:
+        return self.size[1]
+
+    def __post_init__(self):
+        self.size = self.image_size
+
+    def scale(self, scale: float) -> None:
+        self.size = (math.ceil(self.width * scale), math.ceil(self.height * scale))
+
+
+class ImageSheet:
+    def __init__(
+        self,
+        width: int,
+        columns: int = 6,
+        font_path: typing.Optional[Path] = None,
+        label_format: typing.Optional[str] = None,
+        label_font_size: typing.Optional[int] = 10,
+    ):
+        self.width = width
+        self.padding = 20
+        self.columns = columns
+        self.font_path = font_path
+        self.label_format = label_format
+        self.label_font_size = label_font_size
+
+    def create(self, image_paths: typing.Iterable[Path]) -> io.BytesIO:
+        images = []
+        for it in image_paths:
+            with Image.open(it) as img:
+                images.append(PositionedImage(it, img.size))
+
+        if self.font_path and self.font_path.is_file():
+            font = ImageFont.truetype(str(self.font_path), self.label_font_size)
+        else:
+            font = None
+
+        positioned = self._calculate_positions(images)
+        image_sheet = Image.new("RGB", (self.width, 0))
+        for row_images in positioned:
+            height = row_images[0].top + row_images[0].size[1]
+
+            extended_sheet = Image.new("RGB", (self.width, height + self.padding))
+            extended_sheet.paste(image_sheet, (0, 0))
+            image_sheet = extended_sheet
+
+            draw = ImageDraw.Draw(image_sheet)
+            for it in row_images:
+                img = Image.open(it.path)
+                img.thumbnail(it.size, Image.LANCZOS)
+                image_sheet.paste(img, it.position)
+
+                if font:
+                    # write filename on image
+                    text_x, text_y = it.left + 2, it.bottom + 2
+                    filesize = it.path.stat().st_size / 1_048_576
+
+                    text = self._render_text(
+                        width=it.image_size[0],
+                        height=it.image_size[1],
+                        size_mb=filesize,
+                        filename=it.path.name,
+                    )
+                    if text:
+                        draw.text((text_x, text_y), text, (255, 255, 255), font=font, align="left", anchor="la")
+
+                print(f"{it.path.name}\t{it.position}\t{it.size}")
+
+        f = io.BytesIO()
+        image_sheet.save(f, format="JPEG")
+        return f
+
+    def _render_text(self, width: int, height: int, size_mb: float, filename: str) -> typing.Optional[str]:
+        if not self.label_format:
+            return None
+
+        return self.label_format.format(width=width, height=height, size_mb=size_mb, filename=filename)
+
+    def _calculate_positions(self, images: list[PositionedImage]) -> list[list[PositionedImage]]:
+        cols = self.columns
+        width = self.width
+        padding = self.padding
+
+        positioned = []
+
+        rows = list(chunk(images, cols))
+        x, y = 0, 0
+        for row, row_images in enumerate(rows):
+            row_images = list(row_images)
+
+            usable_width = width - (len(row_images) - 1) * padding
+            max_height = max(image.height for image in row_images)
+            # check if there are enough images to fill the row
+            if len(row_images) < cols:
+                mean_height = statistics.median(prow[0].height for prow in positioned)
+                max_height = mean_height
+
+            for img in row_images:
+                y_scaling = max_height / img.height
+                img.scale(y_scaling)
+
+            if len(row_images) >= cols:
+                total_width = sum(image.width for image in row_images)
+                x_scaling = usable_width / total_width
+                max_height = math.ceil(max_height * x_scaling)
+                for img in row_images:
+                    img.scale(x_scaling)
+
+            is_last_row = row == len(rows) - 1
+            x = 0
+            for i, img in enumerate(row_images):
+                img.position = (x, y)
+                is_last_col = i == len(row_images) - 1
+                x = x + img.width + (0 if is_last_col else padding)
+            y = y + max_height + (0 if is_last_row else padding)
+
+            positioned.append(row_images)
+
+        return positioned
+
+
+def parse_args(argv: typing.List[str]) -> argparse.Namespace:
+    arger = argparse.ArgumentParser(
+        description="Create a sheet of images", formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    arger.add_argument("--width", "-w", type=int, default=2000, help="Width of the sheet")
+    arger.add_argument("--columns", "-c", type=int, default=4, help="Number of columns")
+    arger.add_argument("--output", "-o", type=Path, default=Path("image_sheet.jpg"), help="Output file")
+    arger.add_argument("--font", dest="font_path", type=Path, help="Label font to use")
+    arger.add_argument("--font-size", dest="font_size", default=10, type=int, help="Label font size")
+    arger.add_argument(
+        "--label",
+        dest="label_format",
+        default="{width}x{height} {size_mb:.1f}MB {filename}",
+        help="Label format. Available tokens: width, height, size_mb, filename",
+    )
+    arger.add_argument("images", nargs="+", type=Path, help="Images or image directory to include in the sheet")
+    if len(argv) == 0:
+        arger.print_help()
+        sys.exit(1)
+    return arger.parse_args(argv)
+
+
+def main():
+    args = parse_args(sys.argv[1:])
+
+    images: list[Path] = args.images
+    if len(images) == 1 and images[0].expanduser().is_dir():
+        images = sorted(it for it in images[0].resolve().glob("*.jpg"))
+
+    sheet = ImageSheet(
+        width=args.width,
+        columns=args.columns,
+        font_path=args.font_path,
+        label_font_size=args.font_size,
+        label_format=args.label_format,
+    )
+
+    sheet_img = sheet.create(images)
+    with args.output.open("wb") as f:
+        f.write(sheet_img.getvalue())
+
+
+if __name__ == "__main__":
+    main()

imgup.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3.9
+import argparse
+import concurrent.futures
+import dataclasses
+import functools
+import logging
+import os
+import random
+import re
+import sys
+import time
+import typing
+import uuid
+from pathlib import Path
+from typing import Optional
+
+import httpx
+
+
+IMGBB_TOKEN = os.getenv('IMGBB_TOKEN', '391f0559cc209b11f37eda0084bb3281')
+client = httpx.Client(base_url='https://api.imgbb.com/1/', params={'key': IMGBB_TOKEN})
+
+
+def upload_image(image_path: Path) -> str:
+    filename = f'i{image_path.suffix}'
+    with image_path.open('rb') as f:
+        res = client.post('/upload', files={'image': (filename, f)})
+    res.raise_for_status()
+    data = res.json()['data']
+    return data['url']
+
+
+def parse_args(argv: Optional[typing.Sequence[str]] = None) -> argparse.Namespace:
+    arger = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter)
+    arger.add_argument('image_paths', nargs='+', action='store', type=Path, help='Path to image to upload')
+    if not argv or len(argv) == 0:
+        arger.print_help()
+        sys.exit(1)
+
+    return arger.parse_args(argv)
+
+
+def main():
+    args = parse_args(sys.argv[1:])
+    image_paths = args.image_paths
+    with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
+        results = list(pool.map(upload_image, image_paths))
+    for url in results:
+        print(url)
+
+
+if __name__ == '__main__':
+    main()

jd2_hook.py

@@ -0,0 +1,35 @@
+import json
+import logging as logger
+import sys
+from pathlib import Path
+
+import file_renamer
+
+logger.basicConfig(
+    format=f"%(asctime)s {logger.BASIC_FORMAT}",
+    level=logger.DEBUG,
+    handlers=[
+        logger.FileHandler(Path(__file__).with_name("jd2_hook.log"), mode="a"),
+        logger.StreamHandler(),
+    ],
+)
+
+
+def main():
+    try:
+        payload = json.loads(sys.argv[1])
+    except (IndexError, ValueError):
+        logger.error("Expected JSON as first argument")
+        return 1
+
+    for link in payload["links"]:
+        path = link["download_path"]
+        f = Path(path)
+        logger.info("")
+        if f.is_file():
+            file_renamer.fix_dates([f])
+
+
+if __name__ == "__main__":
+    logger.debug(f"Got called with: {sys.argv}")
+    exit(main())

jkup.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3.9
+import argparse
+import concurrent.futures
+import dataclasses
+import re
+import sys
+import time
+import typing
+from pathlib import Path
+from typing import Optional
+import logging
+from functools import cache
+
+
+import httpx
+
+
+@dataclasses.dataclass
+class UploadResult:
+    image_url: str
+    thumbnail_url: str
+
+session = httpx.Client(timeout=20)
+
+@cache
+def get_auth_token():
+    logging.info('extracting auth token')
+    res = session.get("https://jerking.empornium.ph/?agree-consent", follow_redirects=True)
+    try:
+        auth_token = re.search(r'name="auth_token" value="([^"]+)"', res.text).group(1)
+    except AttributeError:
+        auth_token = re.search(r'auth_token = "([^"]+)"', res.text).group(1)
+
+    logging.info(f'got {auth_token=}')
+
+    return auth_token
+
+def upload_image(image_path: Path) -> UploadResult:
+    auth_token = get_auth_token()
+
+    logging.info(f'uploading {image_path.name}')
+
+    with image_path.open("rb") as f:
+        res = session.post(
+            "https://jerking.empornium.ph/json",
+            headers={"accept": "application/json"},
+            data={
+                "thumb_width": "400",
+                "thumb_height": "400",
+                "thumb_crop": "false",
+                "medium_width": "600",
+                "medium_crop": "false",
+                "type": "file",
+                "action": "upload",
+                "timestamp": str(round(time.time() * 1000)),
+                "auth_token": auth_token,
+                "nsfw": "1",
+            },
+            files={
+                "source": f,
+            },
+        )
+
+    data = res.json()
+    logging.info(f'got response={data}')
+    image_url = data["image"]["url"]
+    thumbnail_url = data["image"]["display_url"]
+
+    logging.info(f'uploaded {image_path.name}. url={image_url}')
+
+    return UploadResult(image_url, thumbnail_url)
+
+
+def parse_args(argv: Optional[typing.Sequence[str]] = None) -> argparse.Namespace:
+    arger = argparse.ArgumentParser()
+    arger.add_argument('image_paths', nargs='+', action='store', type=Path, help='Path to image to upload')
+    if not argv or len(argv) == 0:
+        arger.print_help()
+        sys.exit(1)
+
+    return arger.parse_args(argv)
+
+
+def main():
+    args = parse_args(sys.argv[1:])
+    image_paths = args.image_paths
+
+    get_auth_token()
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
+        results = list(pool.map(upload_image, image_paths))
+    for it in results:
+        print(it.image_url)
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.INFO)
+    main()

kube.py

@@ -0,0 +1,122 @@
+import base64
+import logging
+
+import kubernetes.client
+import urllib3.response
+import yaml
+from kubernetes.client import ApiException
+from kubernetes.client.rest import RESTResponse
+
+_result_deleted = {
+    'api_version': 'v1',
+    'code': None,
+    'details': None,
+    'kind': 'Namespace',
+    'message': None,
+    'metadata': {
+        '_continue': None,
+        'remaining_item_count': None,
+        'resource_version': '4424',
+        'self_link': '/api/v1/namespaces/mydep3',
+    },
+    'reason': None,
+    'status': "{'phase': 'Terminating'}",
+}
+_result_not_found = ApiException(
+    RESTResponse(
+        urllib3.response.HTTPResponse(
+            body=b'{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \\"mydep3\\" not found","reason":"NotFound","details":{"name":"mydep3","kind":"namespaces"},"code":404}',
+            status=404,
+        )
+    )
+)
+
+kubeconfig = yaml.load(
+    '''
+apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUREekNDQWZlZ0F3SUJBZ0lVVjhlN3pIa0Q5SitzcnJuN1Ywb0Zka2ZTaFQ0d0RRWUpLb1pJaHZjTkFRRUwKQlFBd0Z6RVZNQk1HQTFVRUF3d01NVEF1TVRVeUxqRTRNeTR4TUI0WERUSXlNREl3TnpBME1qY3dNVm9YRFRNeQpNREl3TlRBME1qY3dNVm93RnpFVk1CTUdBMVVFQXd3TU1UQXVNVFV5TGpFNE15NHhNSUlCSWpBTkJna3Foa2lHCjl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUE0UDNFSkozUXlxbzF4c2ducTh1dXY5Yk9yVks1QmQzOTBwWGcKUVlpMTBnZmd5UzBrWnZ6RFRFYXYzTmNKRnk0N2puYmdkVFF3em1YcHo5SFpKcFRZQUM0M0M1T2hEcHNFZzZEQQo2cVZYcmcrVjhObUlXelRMRk5wT2o2eUFvNnN0V3VqSkpyNDBkR0s1ZE5zT3QwMk9uMElOTkV1MHMraU5uc21tCkhNSFlhc3FXL0YvekFyR0psTTkwUjNtL05wZHdhMWxXcCtKY0lHYnQ3TmRyY21SV05HL3RyU05XQVZZTStrenQKVm02YUIweWZZdTRiUHB3bEtCVU43SjhqdCtJWWQwSmdiMmliUXd6V2VnTVdIU050b0ZtVEN1WVNqYTVTKzVheQpMeGE1R3BaY3B3UUo1dGcvRjFZNTU5QTZkS1gycjFtRzBzS2NIZDk0ZE1IT1FRYVRqd0lEQVFBQm8xTXdVVEFkCkJnTlZIUTRFRmdRVVM2T3l6MkZyNlBVR01Hdk9TSFhVckcvVitBb3dId1lEVlIwakJCZ3dGb0FVUzZPeXoyRnIKNlBVR01Hdk9TSFhVckcvVitBb3dEd1lEVlIwVEFRSC9CQVV3QXdFQi96QU5CZ2txaGtpRzl3MEJBUXNGQUFPQwpBUUVBU0d2bnY3NHNJb2hvcC9DWWpJSHpLMTJGWS9PUmZqUUNNR0tMRGhjN3dEMENXMFhjaHRaQmsrc3VLZVFuCnNJTTlub0k2TStJZGxWM2xCUlBpU1p6ZWpPV2FiZmpHak5MOWw1RGRyVjByNnBtN3p0ckNQU0lXdVY5N29LMTMKVnIwNlVpM2l3ZTM5M00rTlZmYjlxd2wzaGlMUGZ1c0hKM3ZJRGdKWjMyeTI0S2VMekc4QWJTcFJXYjVjWUcvRApQSmpaZGh4bzVjanF0SnZONXBCZ1JlYWdoZTRKdVZKRkh0VWgyYTQrR1F1QjQyVGNIZnFnaEF5TmtkblpIVWRjCmRjOWordFM2YW9xZTRHWEY2bkplcGhSalNaRXEwb1J4MStnR2tvUENkdkdoTGV2ZDRvaUV6eWprYzM1eWNYanQKOTNGOWQzaENxckhQRDBPaW4vQkN0Rk0vclE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
+    server: https://142.132.167.243:16443
+  name: microk8s-cluster
+contexts:
+- context:
+    cluster: microk8s-cluster
+    user: admin
+  name: microk8s
+current-context: microk8s
+kind: Config
+preferences: {}
+users:
+- name: admin
+  user:
+    token: VU96c211SXRDNlg0WUZOTUh0dHpYd2luNy84Ulh0Y2Nka09wWGlmOTQ1ST0K
+'''
+)
+
+logger = logging.getLogger(__name__)
+
+
+class KubernetesService:
+    def __init__(self, client: kubernetes.client.CoreV1Api):
+        self.client = client
+
+    @classmethod
+    def new(cls, kubeconfig: dict):
+        api_client = kubernetes.config.new_client_from_config_dict(kubeconfig)
+        v1_client = kubernetes.client.CoreV1Api(api_client)
+        return cls(v1_client)
+
+    def delete_namespace(self, namespace_name: str) -> None:
+        # just in case
+        assert (
+            namespace_name
+            not in [
+                'kube-system',
+                'kube-public',
+                'kube-node-lease',
+                'default',
+            ]
+            or not namespace_name.startswith('kube-')
+        ), 'protected namespaces cannot be deleted'
+
+        try:
+            _ = self.client.delete_namespace(namespace_name)
+        except ApiException as e:
+            if e.status != 404:  # nocover
+                raise
+            logger.error(
+                'Namespace not found',
+                extra={
+                    'namespace_name': namespace_name,
+                    'response_body': e.body,
+                },
+            )
+
+
+def main():
+    kubernetes.config.load_kube_config_from_dict(kubeconfig)
+    client = kubernetes.client.CoreV1Api()
+    env = {
+        'AWS_ACCESS_KEY': 'aws',
+        'AWS_ACCESS_SECRET_KEY': 'secret',
+    }
+    secret = client.read_namespaced_secret('env', 'default')
+    secret_names = list(secret.data.keys())
+    # ['AWS_ACCESS_KEY', 'AWS_ACCESS_SECRET_KEY']
+    secret = kubernetes.client.V1Secret(
+        api_version='v1',
+        metadata={'name': 'env'},
+        data={k: base64.b64encode(v.encode()).decode() for k, v in env.items()},
+        immutable=True,
+        kind='Secret',
+        type='Opaque',
+    )
+    res = client.create_namespaced_secret('default', secret)
+    print(res)
+
+
+#
+
+if __name__ == '__main__':
+    main()

kube_test.py

@@ -0,0 +1,54 @@
+from unittest import mock
+
+import kubernetes.client.api.core_v1_api
+import pytest
+
+import urllib3.response
+from kubernetes.client import ApiException
+from kubernetes.client.rest import RESTResponse
+from kube import KubernetesService
+
+# fmt: off
+_result_deleted = {'api_version': 'v1', 'code': None, 'details': None, 'kind': 'Namespace', 'message': None, 'metadata': {'_continue': None, 'remaining_item_count': None, 'resource_version': '4424', 'self_link': '/api/v1/namespaces/mydep3',}, 'reason': None, 'status': "{'phase': 'Terminating'}",}
+_result_not_found = ApiException(status=404, http_resp=RESTResponse(urllib3.response.HTTPResponse(body=b'{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \\"mydep3\\" not found","reason":"NotFound","details":{"name":"mydep3","kind":"namespaces"},"code":404}')))
+# fmt: on
+
+
+@pytest.fixture()
+def mock_k8s():
+    yield mock.MagicMock(autospec=kubernetes.client.api.core_v1_api.CoreV1Api)
+
+
+def test_delete_forbidden_namespace(mock_k8s):
+    service = KubernetesService(mock_k8s)
+
+    with pytest.raises(AssertionError):
+        service.delete_namespace('default')
+
+
+def test_delete_namespace(mock_k8s):
+    mock_k8s.delete_namespace.return_value = _result_deleted
+    service = KubernetesService(mock_k8s)
+
+    service.delete_namespace('app-uuid')
+
+    mock_k8s.delete_namespace.assert_called_once_with('app-uuid')
+
+
+def test_deleting_nonexistent_namespace_is_ok(mock_k8s):
+    mock_k8s.delete_namespace.side_effect = _result_not_found
+    service = KubernetesService(mock_k8s)
+
+    with mock.patch('logging.Logger.error') as m_log:
+        service.delete_namespace('app-uuid')
+
+    m_log.assert_called()
+
+
+def test_any_other_error_bubbles_up(mock_k8s):
+    mock_k8s.delete_namespace.side_effect = ApiException(status=500, http_resp=urllib3.response.HTTPResponse(body=b''))
+
+    service = KubernetesService(mock_k8s)
+
+    with pytest.raises(ApiException):
+        service.delete_namespace('app-uuid')

launchd_plist.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+import argparse
+import logging
+import plistlib
+import sys
+import typing
+from pathlib import Path
+
+
+def parse_args(argv: list[str]) -> typing.Tuple[argparse.Namespace, list[str]]:
+    def parse_env(env: str) -> typing.Tuple[str, str]:
+        return env.split("=", maxsplit=1)
+
+    def parse_path(path: str) -> Path:
+        return Path(path).expanduser().resolve()
+
+    arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    arger.add_argument(
+        "-e", "--env", dest="env", metavar="ENVVAR", type=parse_env, nargs="+", help="Environment variables as K=V list"
+    )
+    arger.add_argument("-w", "--cwd", dest="cwd", default=Path.cwd(), type=parse_path, help="Working directory")
+    arger.add_argument("-l", "--label", required=True, help="Label")
+    arger.add_argument(
+        "--install",
+        action="store_true",
+        help="Install the plist to ~/Library/LaunchAgents/",
+    )
+
+    if not argv:
+        arger.print_help()
+        sys.exit(1)
+
+    args, unknown = arger.parse_known_args()
+    if unknown and unknown[0] == "--":
+        unknown = unknown[1:]
+
+    if not unknown:
+        arger.print_help()
+        sys.exit(1)
+
+    return args, unknown
+
+
+def generate_plist(label: str, args: list[str], env: dict, cwd: Path) -> str:
+    doc = {
+        "Label": label,
+        "ProgramArguments": args,
+        "EnvironmentVariables": env,
+        "WorkingDirectory": str(cwd),
+        "RunAtLoad": True,
+        "StandardOutPath": str(cwd / f"{label}.stdout.log"),
+        "StandardErrorPath": str(cwd / f"{label}.stderr.log"),
+    }
+    return plistlib.dumps(
+        doc,
+        fmt=plistlib.FMT_XML,
+        sort_keys=True,
+    ).decode()
+
+
+def main():
+    args, unknown = parse_args(sys.argv[1:])
+    print(args, unknown)
+    plist_doc = generate_plist(
+        label=args.label,
+        args=unknown,
+        env=dict(args.env),
+        cwd=args.cwd,
+    )
+    print(plist_doc)
+
+    if args.install:
+        install_dir: Path = Path("~/Library/LaunchAgents/").expanduser()
+        logging.info(f"Installing plist to {install_dir}")
+
+        install_path = install_dir / f"{args.label}.plist"
+        install_path.write_text(plist_doc)
+
+        logging.info(f'Run "launchctl load -w {install_path}" to start the service')
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    main()

macos_wallpaper.py

@@ -0,0 +1,72 @@
+import contextlib
+import dataclasses
+import plistlib
+import sqlite3
+import subprocess
+from pathlib import Path
+from typing import Optional
+
+DB_PATH = Path("~/Library/Application Support/Dock/desktoppicture.db").expanduser()
+
+
+@dataclasses.dataclass
+class SpaceInfo:
+    uuid: str
+    image_path: Optional[Path]
+    data_id: Optional[Path]
+
+
+@contextlib.contextmanager
+def get_connection() -> sqlite3.Connection:
+    con = sqlite3.connect(str(DB_PATH))
+    con.row_factory = sqlite3.Row
+    yield con
+    con.close()
+
+
+def get_current_wallpapers() -> list[SpaceInfo]:
+    sql = """
+    select distinct s.space_uuid, pr.key, d.rowid, d.value from preferences pr
+    left join payload d on d.rowid = pr.data_id
+    left join pictures p on p.rowid = pr.picture_id
+    inner join spaces s on s.rowid = p.space_id
+    where key = 1
+    
+    order by space_uuid, key
+    """
+    spaces = list_spaces()
+
+    with get_connection() as con:
+        cur = con.execute(sql)
+
+        space_props = {}
+        for i, row in enumerate(cur):
+            space_uuid = row["space_uuid"]
+            data_id = row["rowid"]
+            image_path = row["value"]
+
+            if image_path:
+                image_path = Path(image_path).expanduser()
+            space_props[space_uuid] = SpaceInfo(uuid=space_uuid, image_path=image_path, data_id=data_id)
+    return [space_props.get(id, SpaceInfo(uuid="", image_path=None, data_id=None)) for id in spaces]
+
+
+def read_plist(plist_path):
+    with plist_path.open("rb") as f:
+        return plistlib.load(f)
+
+
+def list_spaces():
+    spaces_plist_path = Path("~/Library/Preferences/com.apple.spaces.plist").expanduser()
+    data = read_plist(spaces_plist_path)
+    spaces = data["SpacesDisplayConfiguration"]["Management Data"]["Monitors"][0]["Spaces"]
+    return [s["uuid"] for s in spaces]
+
+
+def set_wallpaper(image_path: Path):
+    script = f'tell application "Finder" to set desktop picture to "{image_path.expanduser().resolve()}" as POSIX file'
+    subprocess.run(["osascript", "-e", script], check=True)
+
+
+if __name__ == "__main__":
+    pass

make_kubeconfig.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cluster_name=sandbox
+namespace=default
+service_account=demo-readonly
+server=http://0.0.0.0:58253
+
+secret_name=$(kubectl --namespace $namespace get serviceaccount $service_account -o jsonpath='{.secrets[0].name}')
+ca=$(kubectl --namespace $namespace get secret/$secret_name -o jsonpath='{.data.ca\.crt}')
+token=$(kubectl --namespace $namespace get secret/$secret_name -o jsonpath='{.data.token}' | base64 --decode)
+
+echo "apiVersion: v1
+kind: Config
+clusters:
+  - name: ${cluster_name}
+    cluster:
+      certificate-authority-data: ${ca}
+      server: ${server}
+contexts:
+  - name: ${service_account}@${cluster_name}
+    context:
+      cluster: ${cluster_name}
+      namespace: ${namespace}
+      user: ${service_account}
+users:
+  - name: ${service_account}
+    user:
+      token: ${token}
+current-context: ${service_account}@${cluster_name}
+"

personio_attendance.py

@@ -0,0 +1,71 @@
+import datetime
+from pathlib import Path
+import sys
+import time
+
+from playwright.sync_api import BrowserContext, sync_playwright
+
+
+class creds:
+    employee_id: str = "9374380"
+    email: str = "abduessamet.kocak@refurbed.com"
+    password: str = "^6LMq$LX&2#c4y"
+
+
+def main():
+    try:
+        day = datetime.date.today().fromisoformat(sys.argv[1]).isoformat()
+    except:
+        day = "today"
+
+    with sync_playwright() as playwright:
+        data_dir = Path("/tmp/personio")
+        data_dir.mkdir(exist_ok=True, parents=True)
+        context = playwright.chromium.launch_persistent_context(headless=False, user_data_dir=data_dir)
+        with context:
+            fill_attendance(
+                context=context,
+                day=day,
+                work=("08:00", "17:00"),
+                pause=("12:00", "13:00"),
+            )
+
+
+def fill_attendance(context: BrowserContext, day: str, work: tuple[str, str], pause=[str, str]):
+    # Open new page
+    page = context.new_page()
+
+    url = f"https://refurbed.personio.de/attendance/employee/{creds.employee_id}/"
+    page.goto(url)
+    page.wait_for_load_state()
+
+    if "/login/index" in page.url:
+        page.locator('[placeholder="Email"]').fill(creds.email)
+        page.locator('[placeholder="Password"]').fill(creds.password)
+        page.locator('button:has-text("Login")').click()
+
+    card_selector = f'[data-test-id="day_{day}"] button'
+    if day == "today":
+        card_selector = '[data-test-id="today-cell"] button'
+
+    while True:
+        page.locator(card_selector).click()
+        if page.locator('[data-test-id="day-entry-dialog"]').is_visible():
+            break
+        time.sleep(0.05)
+
+    work_start, work_end = work
+    page.locator('[data-test-id="timerange-start"]').first.fill(work_start)
+    page.locator('[data-test-id="work-entry"] [data-test-id="timerange-end"]').fill(work_end)
+
+    pause_start, pause_end = pause
+    page.locator('[data-test-id="break-entry"] [data-test-id="timerange-start"]').fill(pause_start)
+    page.locator('[data-test-id="break-entry"] [data-test-id="timerange-end"]').fill(pause_end)
+
+    page.locator('[data-test-id="day-entry-save"]').click()
+    # wait until network req is sent
+    time.sleep(1)
+
+
+if __name__ == "__main__":
+    main()

plab_search.py

@@ -0,0 +1,188 @@
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import datetime
+import json
+import typing
+from pathlib import Path
+from pprint import pprint
+from urllib.parse import urljoin
+
+import bs4
+import httpx
+
+
+@dataclasses.dataclass()
+class SearchResult:
+    posted_at: datetime.datetime
+    title: str
+    size_mb: float
+    url: str
+
+
+class Auth(typing.Protocol):
+    def authenticate(self) -> dict[str, str]:
+        ...
+
+
+@dataclasses.dataclass()
+class CookieFileAuth:
+    file_path: Path
+
+    def authenticate(self) -> dict[str, str]:
+        with self.file_path.open() as f:
+            return json.load(f)
+
+
+@dataclasses.dataclass()
+class CredentialAuth:
+    username: str
+    password: str
+
+    def authenticate(self) -> dict[str, str]:
+        client = httpx.Client(base_url="https://pornolab.net/")
+        res = client.post(
+            "/forum/login.php",
+            data={
+                "login_username": self.username,
+                "login_password": self.password,
+                "login": "Вход",
+            },
+            follow_redirects=True,
+        )
+        res.raise_for_status()
+        soup = bs4.BeautifulSoup(res.text, "html.parser")
+        username_el = soup.select_one(".topmenu .med")
+        if self.username not in username_el.text.strip():
+            raise ValueError("failed to login")
+        return dict(client.cookies)
+
+
+@dataclasses.dataclass()
+class PersistedCookieAuth:
+    cookie_path: Path
+    auth: Auth
+
+    def authenticate(self) -> dict[str, str]:
+        cookie_auth = CookieFileAuth(self.cookie_path)
+        try:
+            return cookie_auth.authenticate()
+        except (ValueError, FileNotFoundError):
+            pass
+
+        cookies = self.auth.authenticate()
+        with self.cookie_path.open("w") as f:
+            json.dump(cookies, f)
+        return cookies
+
+
+class PlabAuthMiddleware(httpx.Auth):
+    def auth_flow(self, request: httpx.Request) -> typing.Generator[httpx.Request, httpx.Response, None]:
+        # if no cookies in req
+        # fill cookie
+        # check response
+        # reauth
+        # fill cookie
+
+        res = yield request
+        return super().auth_flow(request)
+
+
+class Plab:
+    def __init__(self, auth: Auth):
+        self.auth = auth
+        self.client = httpx.Client(base_url="https://pornolab.net/")
+
+    def search(self, term: str) -> list[SearchResult]:
+        self._ensure_session()
+        res = self.client.post(
+            "/forum/tracker.php",
+            data={
+                "max": "1",
+                "to": "1",
+                "nm": f'{term}',
+            },
+            follow_redirects=True,
+        )
+        res.raise_for_status()
+
+        if 'Введите ваше имя и пароль' in res.text:
+            raise
+
+        soup = bs4.BeautifulSoup(res.text, "html.parser")
+        rows = soup.select("#tor-tbl tbody tr")
+        items = []
+        for row in rows:
+            title_link = row.select_one(f'[href*="viewtopic.php?t="]')
+            if not title_link:
+                continue
+            size_mb = (
+                int(row.select_one('[href*="dl.php?t="]').find_previous_sibling(name='u').text.strip()) / 1_048_576
+            )
+            date = datetime.datetime.fromtimestamp(int(row.select_one('[title="Добавлен"] u').text.strip()))
+            items.append(
+                SearchResult(
+                    url=urljoin("https://pornolab.net/forum/", title_link["href"]),
+                    title=title_link.text.strip(),
+                    size_mb=size_mb,
+                    posted_at=date,
+                )
+            )
+        return items
+
+    def _ensure_session(self):
+        if self.client.cookies:
+            return
+        self.client.cookies = self.auth.authenticate()
+
+
+def dump_for_alfred(items: list[SearchResult]):
+    alfreds = []
+    for it in items:
+        alfreds.append(
+            {
+                "title": it.title,
+                "subtitle": f"{it.size_mb:.1f}MB -- {it.posted_at.isoformat()}",
+                "arg": it.url,
+                "text": {
+                    "copy": it.url,
+                    "largetype": f"{it.title}\nPosted at: {it.posted_at.isoformat()}\nSize: {it.size_mb:.1f}MB",
+                },
+                "match": f"{it.title} {it.posted_at.isoformat()}",
+            }
+        )
+    print(json.dumps({"items": alfreds}, indent=2))
+
+
+def parse_args():
+    arger = argparse.ArgumentParser()
+    arger.add_argument('term', help='Term to search for')
+    arger.add_argument('--alfred', action='store_true', help='Dump for Alfred')
+    return arger.parse_args()
+
+
+def main():
+    args = parse_args()
+
+    p = Plab(
+        auth=PersistedCookieAuth(
+            cookie_path=Path(__file__).parent / "plab.json",
+            auth=CredentialAuth(
+                username="zzzp",
+                password="3c45HzLMgSB7M5hJ",
+            ),
+        )
+    )
+
+    results = p.search(args.term)
+
+    if args.alfred:
+        dump_for_alfred(results)
+    else:
+        for it in results:
+            pprint(it)
+
+
+if __name__ == "__main__":
+    main()

playwright_playground.py

@@ -0,0 +1,55 @@
+from os import PathLike
+from pathlib import Path
+import re
+from playwright.sync_api import sync_playwright, Browser
+from contextlib import contextmanager
+
+
+@contextmanager
+def run_browser(data_dir: PathLike, extension_path: PathLike) -> Browser:
+    with sync_playwright() as pw:
+        with pw.chromium.launch_persistent_context(
+            user_data_dir=data_dir,
+            headless=False,
+            args=[
+                f"--disable-extensions-except={extension_path}",
+                f"--load-extension={extension_path}",
+            ],
+        ) as browser:
+            yield browser
+
+
+def find_sales_performance(browser: Browser, url: str) -> dict:
+    with browser.new_page() as page:
+        page.goto(url)
+        text = page.wait_for_selector("#h10-product-score").inner_text()
+    
+    amazon_id = re.search(r'/dp/([^?]+)', url).group(1)
+    score = float(re.search(r"(\d+\.?\d*)", text).group(1))
+    try:
+        sales = int(re.search(r"Sales\s*(\d+[,.]?\d*)", text, re.MULTILINE).group(1).replace(',', ""))
+    except:
+        sales = -1
+    
+    return {
+        'amazon_id': amazon_id,
+        'sales': sales,
+        'score': score,
+    }
+
+
+extension_path = Path(r"~/Desktop/amazon").expanduser().resolve()
+data_dir = Path(r'~/Desktop/amazondemo').expanduser()
+
+def main():
+    urls = [
+        "https://www.amazon.co.uk/dp/B07B9G7V3P?psc=1",
+    ]
+    with run_browser(data_dir=data_dir, extension_path=extension_path) as browser:
+        for url in urls:
+            perf = find_sales_performance(browser=browser, url=url)
+            print(perf)
+
+
+if __name__ == "__main__":
+    main()

realdebrid.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+import httpx
+import subprocess
+import logging
+import typer
+from pathlib import Path
+
+logging.basicConfig(
+    level=logging.DEBUG,
+    format="%(levelname)s %(asctime)s: %(message)s",
+    datefmt=logging.Formatter.default_time_format,
+)
+logging.getLogger("httpx").setLevel(logging.WARNING)
+
+# get api token from https://real-debrid.com/apitoken
+key = "P4SUGS4VSXLDIRXSOAB46DXMU2HVFJTOWYCACNULX4V7TOZRW2BA"
+
+cli = typer.Typer()
+
+http = httpx.Client(
+    base_url="https://api.real-debrid.com/rest/1.0/",
+    headers={"Authorization": f"Bearer {key}"},
+)
+
+
+def get_download_url(url: str) -> str:
+    res = http.post("/unrestrict/link", data={"link": url})
+    res.raise_for_status()
+    return res.json()["download"]
+
+
+def download_with_aria2(
+    url: str, cwd: Path = Path.cwd(), extra_args: list[str] = None
+) -> None:
+    if not extra_args:
+        extra_args = []
+    args = ["aria2c", "-x", "5", url, *extra_args]
+    logging.debug("calling aria2 with args=%r", args)
+    subprocess.run(args, text=True, cwd=str(cwd.resolve()))
+
+
+@cli.command(
+    context_settings={
+        "ignore_unknown_options": True,
+        "allow_extra_args": True,
+    }
+)
+def main(
+    ctx: typer.Context,
+    url: str,
+    cwd: Path = typer.Option(Path("."), "-o", "--out", "--cwd", writable=True),
+):
+    try:
+        logging.debug("generating premium link")
+        download_url = get_download_url(url)
+    except (httpx.HTTPStatusError, KeyError):
+        logging.error("cannot generate premium link")
+        raise typer.Exit(1)
+
+    logging.info("got url=%s", download_url)
+
+    logging.debug("passing url to aria2")
+    try:
+        download_with_aria2(download_url, cwd=cwd, extra_args=ctx.args)
+    except KeyboardInterrupt:
+        logging.error("download interrupted. exiting")
+        raise typer.Exit(2)
+
+
+if __name__ == "__main__":
+    cli()

requirements.txt

@@ -0,0 +1,6 @@
+beautifulsoup4==4.11.2
+boto3==1.19.1
+fastapi==0.86.0
+httpx==0.23.3
+pytest==6.2.5
+typer==0.6.1

rsync_it.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+import logging
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+
+def find_file_arg(argv: list[str]) -> Path:
+    for it in argv:
+        if it.startswith('-'):
+            continue
+        if Path(it).is_file():
+            return Path(it)
+
+
+def main():
+    argv = sys.argv[1:]
+    file_path = find_file_arg(argv)
+    if not file_path:
+        logging.error('file argument not passed')
+        return
+
+    size = file_path.stat().st_size
+    while True:
+        args = ['rsync', '--recursive', '--human-readable', '--progress', '--partial', *argv]
+        logging.debug(f'running rsync with {args=}')
+        p = subprocess.run(args)
+        p.check_returncode()
+
+        logging.debug('will check the file size in 20s')
+        time.sleep(20)
+        subprocess.run(['touch', str(file_path)])
+
+        new_size = file_path.stat().st_size
+
+        if new_size == size:
+            logging.info('file size has not changed, stopping')
+            break
+        logging.info(f'file size has changed {new_size - size} bytes, re-syncing')
+        size = new_size
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.INFO)
+    main()

tekton_api.py

@@ -0,0 +1,190 @@
+import functools
+import hashlib
+import json
+import textwrap
+import typing
+import uuid
+from pathlib import Path
+
+import kubernetes.config
+import yaml
+
+
+def make_client() -> kubernetes.client.CoreV1Api:
+    kubernetes.config.load_kube_config('/Users/abdus/Desktop/kubeconfig-kube.yml')
+    return kubernetes.client.CoreV1Api()
+
+
+class Kubernetes:
+    def __init__(self, client: kubernetes.client.CoreV1Api, namespace='default'):
+        self.client = client
+        self.custom_objects = kubernetes.client.CustomObjectsApi(self.client.api_client)
+        self.namespace = namespace
+
+    @classmethod
+    def from_kubeconfig(cls, kubeconfig_path: str, namespace: str) -> 'Kubernetes':
+        kubernetes.config.load_kube_config(kubeconfig_path)
+        return cls(kubernetes.client.CoreV1Api(), namespace=namespace)
+
+    def read_logs(self, pod_name: str, container_name: typing.Optional[str] = None, timestamps: bool = False) -> str:
+        return self.client.read_namespaced_pod_log(
+            pod_name,
+            self.namespace,
+            container=container_name,
+            timestamps=timestamps,
+            pretty=True,
+        )
+
+    def get_tekton_object(self, plural: str, name: str = None):
+        getter = functools.partial(
+            self.custom_objects.get_namespaced_custom_object,
+            group='tekton.dev',
+            version='v1beta1',
+            namespace=self.namespace,
+        )
+        return getter(plural=plural, name=name)
+
+    def list_tekton_object(self, plural: str):
+        getter = functools.partial(
+            self.custom_objects.list_namespaced_custom_object,
+            group='tekton.dev',
+            version='v1beta1',
+            namespace=self.namespace,
+        )
+        return getter(plural=plural)['items']
+
+    def create_from_yaml(self, yaml_text: str) -> list[dict]:
+        specs = list(yaml.safe_load_all(yaml_text))
+        return kubernetes.utils.create_from_yaml(
+            kubernetes.client.CustomObjectsApi(self.client.api_client),
+            yaml_objects=specs,
+            namespace=self.namespace,
+        )
+
+
+def read_logs():
+    task_hello_world = k8s.get_tekton_object(plural='tasks', name='echo-hello-world')
+    taskruns = k8s.list_tekton_object(plural='taskruns')
+
+    pod_name = taskruns['items'][0]['status']['podName']
+    container_name = taskruns['items'][0]['status']['containerName']
+    logs = k8s.read_logs(pod_name, container_name)
+    print(logs)
+
+
+k8s = Kubernetes.from_kubeconfig('/Users/abdus/Desktop/kubeconfig.yaml', namespace='api-server')
+
+
+def get_pipelineru_status():
+    result = k8s.get_tekton_object('pipelineruns', name='pr-2rr6t')
+    status = result['status']['conditions'][0]['status']
+    reason = result['status']['conditions'][0]['reason']
+    failed_taskruns = [
+        name for name, it in result['status']['taskRuns'].items() if it['status']['conditions'][0]['status'] == 'False'
+    ]
+
+    logs = []
+    for it in result['status']['taskRuns'].values():
+        task_name = it['pipelineTaskName']
+        task_status = it['status']['conditions'][0]['reason']
+        pod_name = it['status']['podName']
+
+        if task_status != 'Failed':
+            continue
+
+        # fetch all logs for all steps to get a better picture of the error
+        # (regardless of whether it is successful or not)
+        for i, step in enumerate(it['status']['steps'], start=1):
+            container_name = step['container']
+            header = f'# {task_name}, step {i}: {step["name"]}'
+            logs.append('#' * len(header))
+            logs.append(header)
+            logs.append('#' * len(header))
+            step_logs = k8s.read_logs(pod_name, container_name, timestamps=True)
+            logs.append(step_logs)
+
+    log_text = '\n'.join(logs)
+    return {
+        'pipelinerun_name': result['metadata']['name'],
+        'namespace': result['metadata']['namespace'],
+        'status': status,
+        'reason': reason,
+        'failed_taskruns': failed_taskruns,
+        'logs': log_text,
+    }
+
+
+def create_taskrun():
+    k = kubernetes.client.CustomObjectsApi()
+    res = k.create_namespaced_custom_object(
+        group='tekton.dev',
+        version='v1beta1',
+        namespace='default',
+        plural='taskruns',
+        body={
+            'apiVersion': 'tekton.dev/v1beta1',
+            'kind': 'TaskRun',
+            'metadata': {'name': 'testing1234' + uuid.uuid4().hex},
+            'spec': {
+                'taskRef': {'name': 'task-with-json'},
+                'params': [
+                    {
+                        'name': 'json_value',
+                        'value': '{"foo": "bar", "nested": {"a": "1"}}',
+                    }
+                ],
+            },
+        },
+    )
+    print(res)
+
+
+def migrate_pipelines():
+    k = kubernetes.client.CustomObjectsApi()
+
+    pipeline_yaml = Path('sample_pipeline.yaml').read_text()
+    parsed_pipeline = yaml.safe_load(pipeline_yaml)
+
+    params = dict(group='tekton.dev', version='v1beta1', namespace='default', plural='pipelines')
+    k.delete_namespaced_custom_object(
+        **params,
+        name=parsed_pipeline['metadata']['name'],
+    )
+    res = k.create_namespaced_custom_object(
+        **params,
+        body=parsed_pipeline,
+    )
+    return parsed_pipeline
+
+
+def trigger_pipeline():
+    k = kubernetes.client.CustomObjectsApi()
+    pipeline = migrate_pipelines()
+    pipeline_name = pipeline['metadata']['name']
+
+    payload = {
+        'image': 'nginx',
+        'json_value': {'nested': {'nested2': {'a': 1}}},
+    }
+
+    res = k.create_namespaced_custom_object(
+        group='tekton.dev',
+        version='v1beta1',
+        namespace='default',
+        plural='pipelineruns',
+        body={
+            'apiVersion': 'tekton.dev/v1beta1',
+            'kind': 'PipelineRun',
+            'metadata': {'name': 'deploy-app-' + uuid.uuid4().hex},
+            'spec': {
+                'pipelineRef': {'name': pipeline_name},
+                'params': [
+                    {'name': k, 'value': json.dumps(v) if not isinstance(v, str) else v} for k, v in payload.items()
+                ],
+            },
+        },
+    )
+
+
+if __name__ == '__main__':
+    main()

torrentify.py

@@ -0,0 +1,496 @@
+#!/usr/bin/env python3.9
+import argparse
+import json
+import logging
+import os
+import re
+import shutil
+import sys
+import time
+from turtle import back
+import typing
+from pathlib import Path
+from typing import Optional
+
+import ffmpeg
+from ffmpeg_strip import clean_video
+
+logging.basicConfig(level=logging.INFO, format=f"%(asctime)s {logging.BASIC_FORMAT}")
+
+
+def force_import(module: str):
+    import importlib
+    import subprocess
+    import sys
+
+    try:
+        return importlib.import_module(module)
+    except ModuleNotFoundError:
+        subprocess.run([sys.executable, "-m", "pip", "install", module])
+        importlib.invalidate_caches()
+        return importlib.import_module(module)
+
+
+try:
+    import torf
+except ImportError:
+    torf = force_import("torf")
+
+try:
+    import httpx
+except ImportError:
+    httpx = force_import("httpx")
+
+try:
+    import inquirer
+except ImportError:
+    inquirer = force_import("inquirer")
+
+TORRENT_TRACKER_URL = os.getenv(
+    "TORRENT_TRACKER_URL", "http://tracker.empornium.sx:2710/tegqucis10uanp672qh6nhn393xlkncs/announce"
+)
+TORRENT_CREATED_BY = os.getenv("TORRENT_CREATED_BY", "zzzp")
+TORRENT_DIR = Path(os.getenv("TORRENT_DIR", "/mnt/box/files/_torrents/_new2/"))
+
+VIDEO_EXTENSIONS = {".mp4", ".mpg", ".mkv"}
+
+
+def make_torrent(source_dir: Path, tracker_url: str = TORRENT_TRACKER_URL) -> Path:
+    torrent_path = source_dir / f"{source_dir.name}.torrent"
+    if torrent_path.is_file():
+        logging.info("Torrent file is already created")
+        return torrent_path
+
+    t = torf.Torrent(
+        path=str(source_dir.resolve()),
+        name=source_dir.resolve().name,
+        trackers=[tracker_url],
+        private=True,
+        created_by=TORRENT_CREATED_BY,
+        exclude_globs=["*.txt", "post.txt", "*.torrent", "*.gif"],
+    )
+    t.generate()
+    t.write(torrent_path, overwrite=True)
+
+    return torrent_path
+
+
+def human_size(size: int) -> str:
+    suffix = "B"
+    for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
+        if abs(size) < 1024.0:
+            return "%3.2f%s%s" % (size, unit, suffix)
+        size /= 1024.0
+    return "%.2f%s%s" % (size, "Yi", suffix)
+
+
+def find_video(root_dir: Path) -> Optional[Path]:
+    for ext in VIDEO_EXTENSIONS:
+        for f in root_dir.rglob(f"*{ext}"):
+            return f
+    return None
+
+
+def pick_name(video_path: Path) -> str:
+    try:
+        if parse_filename(video_path.stem):
+            return video_path.stem
+    except ValueError:
+        pass
+
+    candidates = set()
+
+    parent = video_path.parent
+    while parent.exists():
+        try:
+            if parse_filename(parent.name):
+                candidates.add(parent.name)
+        except ValueError:
+            pass
+        if parent.parent == parent:
+            break
+        parent = parent.parent
+
+    if len(candidates) == 1:
+        return candidates.pop()
+    elif len(candidates) > 1:
+        questions = [
+            inquirer.List(
+                "best_name",
+                message="Pick the best filename",
+                choices=[*candidates, "<custom>"],
+            ),
+        ]
+
+        answers = inquirer.prompt(questions) or {}
+        best_name = answers.get("best_name")
+        if best_name != "<custom>":
+            return best_name
+
+    while True:
+        try:
+            answers = inquirer.prompt([inquirer.Editor("best_name", message="New name", default=video_path.stem)])
+            if not answers:
+                raise Exception("Cancelled")
+            best_name = answers["best_name"].splitlines(keepends=False)[0].strip()
+            parse_filename(best_name)
+            return best_name
+        except ValueError:
+            continue
+
+
+def move_video(video_path: Path, new_name: typing.Optional[str], link: bool = False) -> Path:
+    """
+    Move video to the torrent directory and returns the video path
+    """
+    if new_name:
+        best_name = Path(new_name).name
+    else:
+        best_name = pick_name(video_path)
+    assert not best_name.startswith('.')
+    logging.info("using name: %s", best_name)
+
+    best_name = best_name.removesuffix('.nometadata')
+    if Path(best_name).suffix in VIDEO_EXTENSIONS:
+        best_name = Path(best_name).stem
+
+    renamed_path = video_path.with_stem(best_name)
+    if link:
+        video_path = video_path.link_to(renamed_path)
+    else:
+        video_path = video_path.rename(renamed_path)
+    video_path = renamed_path
+
+    new_loc = TORRENT_DIR / video_path.stem
+    new_loc.mkdir(parents=True, exist_ok=True)
+    target_path = new_loc / video_path.name
+    video_path.rename(target_path)
+
+    for f in video_path.parent.glob("*.jpg"):
+        if f.stem.startswith(video_path.stem):
+            f.rename(new_loc / f.name)
+            break
+
+    return target_path
+
+
+def upload_image(image_path: Path) -> str:
+    session = httpx.Client()
+    res = session.get("https://jerking.empornium.ph/?agree-consent", follow_redirects=True)
+    try:
+        auth_token = re.search(r'name="auth_token" value="([^"]+)"', res.text).group(1)
+    except AttributeError:
+        auth_token = re.search(r'auth_token = "([^"]+)"', res.text).group(1)
+
+    with image_path.open("rb") as f:
+        res = session.post(
+            "https://jerking.empornium.ph/json",
+            headers={"accept": "application/json"},
+            data={
+                "thumb_width": "160",
+                "thumb_height": "160",
+                "thumb_crop": "false",
+                "medium_width": "500",
+                "medium_crop": "false",
+                "type": "file",
+                "action": "upload",
+                "timestamp": str(round(time.time() * 1000)),
+                "auth_token": auth_token,
+                "nsfw": "0",
+            },
+            files={
+                "source": ("thumbs.jpg", f),
+            },
+        )
+
+    data = res.json()
+    image_url = data["image"]["url"]
+    thumbnail_url = data["image"]["display_url"]
+
+    return image_url
+
+
+ParsedFilename = typing.TypedDict('ParsedFilename', {'actors': list[str], 'studio': str, 'date': str, 'title': str, 'tags': list[str]})
+
+def parse_filename(filename: str) -> ParsedFilename:
+    parsed = {
+        "tags": [],
+    }
+    filename = re.sub(r"\s*\[\d+[^]]+]", "", filename)
+    if m := re.search(r"(?P<actors>.+)\s+-+\s+@(?P<studio>\S+)\s+-+\s+(?P<title>.+)\s+-+\s+(?P<date>[\d-]+)", filename):
+        parsed.update(m.groupdict())
+    elif m := re.search(r"(?P<actors>.+)\s+-+\s+@(?P<studio>\S+)\s+-+\s+(?P<date>[\d-]+)", filename):
+        parsed.update(m.groupdict())
+    elif m := re.search(r"(?P<actors>.+)\s+-+\s+(?P<title>.+)\s+-+\s+(?P<date>[\d-]+)", filename):
+        parsed.update(m.groupdict())
+    elif m := re.search(r"(?P<actors>.+)\s+-+\s+(?P<title>\D.+)", filename):
+        parsed.update(m.groupdict())
+    else:
+        raise ValueError("Unknown filename format")
+
+    if "actors" in parsed:
+        names = re.split(r"\s*,\s*", parsed["actors"])
+        with_aliases = []
+        for it in names:
+            with_aliases.extend(re.split(r"\s+aka\s+", it))
+        parsed["actors"] = names
+        parsed["tags"].extend([it.lower().replace(" ", ".") for it in with_aliases])
+    if "studio" in parsed:
+        parsed["tags"].append(parsed["studio"].lower() + ".com")
+    if "date" in parsed:
+        year, ym = parsed["date"][:4], parsed["date"][:7].replace("-", ".")
+        parsed["tags"].append(year)
+        parsed["tags"].append(ym)
+
+    return parsed
+
+def generate_title(parsed: ParsedFilename) -> str:
+    parts = []
+    if actors := parsed.get('actors'):
+        parts.append(', '.join(actors))
+    if studio := parsed.get('studio'):
+        parts.append(f'@{studio}')
+    if title := parsed.get('title'):
+        parts.append(title)
+    if date := parsed.get('date'):
+        parts.append(date)
+    return ' -- '.join(parts)
+
+
+def generate_post_bbcode(video_path: Path, thumbnails_path: Path) -> str:
+    try:
+        image_url = upload_image(thumbnails_path)
+        image_bbcode = f"[img]{image_url}[/img]"
+    except:
+        logging.exception("Failed to upload image")
+        image_bbcode = ""
+
+    parsed = parse_filename(video_path.stem)
+    probe = ffmpeg.ffprobe(video_path)
+
+    if 1900 <= probe.width <= 2200:
+        hd = "1080p"
+    elif 1200 <= probe.width <= 1400:
+        hd = "720p"
+    elif 3000 <= probe.width:
+        hd = "4K"
+    else:
+        hd = None
+
+    if hd:
+        parsed["tags"].append(hd.lower())
+    if probe.codec == "hevc":
+        parsed["tags"].extend(["x265", "x265.reencode", "hevc.x265"])
+
+    table = {
+        "Duration": probe.duration_human,
+        "Format": video_path.suffix.strip("."),
+        "Filesize": human_size(video_path.stat().st_size),
+        "Resolution": f"{probe.width}x{probe.height}",
+        "Codec": probe.codec,
+        "Bit rate": f"{probe.bitrate // 1000} kbit/s",
+        "FPS": probe.fps,
+    }
+    release_date = parsed.get("date", "")
+
+    if release_date:
+        table = {
+            "Release Date": release_date,
+            **table,
+        }
+
+    lines = [
+        "[table=nball]",
+        *(f"[tr][th=20]{k}[/th][td]{v}[/td][/tr]" for k, v in table.items()),
+        "[/table]",
+    ]
+    table_bbcode = "\n".join(lines)
+
+    m_performers = re.search(r"(.+?) -+\s", video_path.stem)
+    performers = m_performers.group(1) if m_performers else ""
+
+    title = generate_title(parsed)
+
+    metadata = {
+        **parsed,
+        "is_hevc": probe.codec == "hevc",
+        "hd": hd,
+    }
+    metadata_json = json.dumps(metadata)
+
+    return f"""
+{metadata_json}
+---
+[b]{title}[/b]
+
+[cast]
+{performers}
+
+[details]
+
+[info]
+{table_bbcode}
+
+[screens]
+{image_bbcode}
+""".strip()
+
+
+def add_torrent(torrent_path: Path):
+    import time
+    from functools import partial
+
+    make_id = partial(time.time_ns)
+
+    # log in
+    logging.debug("Connecting to Deluge")
+    session = httpx.Client(base_url="https://t.zzzp.win/", timeout=30)
+    res = session.post("/json", json={"method": "auth.login", "params": ["xAsametk50"], "id": make_id()})
+    res.raise_for_status()
+
+    # find first available host
+    res = session.post("/json", json={"method": "web.get_hosts", "params": [], "id": make_id()})
+    res.raise_for_status()
+    host_id = res.json()["result"][0][0]
+
+    # connect to a host
+    res = session.post("/json", json={"method": "web.connect", "params": [host_id], "id": make_id()})
+    res.raise_for_status()
+
+    # upload torrent
+    logging.debug("Uploading torrent file")
+    with torrent_path.open("rb") as f:
+        res = session.post("/upload", files={"file": f})
+    res.raise_for_status()
+    remote_path: str = res.json()["files"][0]
+
+    # add torrent
+    logging.debug("Adding torrent file")
+    res = session.post(
+        "/json",
+        json={
+            "method": "web.add_torrents",
+            "params": [
+                [
+                    {
+                        "path": remote_path,
+                        "options": {
+                            "file_priorities": [1],
+                            "add_paused": True,
+                            "sequential_download": False,
+                            "pre_allocate_storage": False,
+                            "download_location": "/dl/_new2",
+                            "move_completed": False,
+                            "move_completed_path": "/root/Downloads",
+                            "prioritize_first_last_pieces": True,
+                            "seed_mode": True,
+                            "super_seeding": False,
+                        },
+                    }
+                ]
+            ],
+            "id": make_id(),
+        },
+    )
+
+    res.raise_for_status()
+    torrent_id = res.json()["result"][0][1]
+
+    # force recheck
+    # logging.info("Triggering a forced recheck")
+    # time.sleep(5)
+    # try:
+    #     _ = session.post(
+    #         "/json", json={"method": "core.force_recheck", "params": [[torrent_id]], "id": make_id()}, timeout=0.1
+    #     )
+    # except httpx.HTTPError:
+    #     pass
+
+    logging.debug("Torrent file has been added successfully.")
+
+
+def parse_args(argv: list[str]):
+    arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    arger.add_argument("dir_or_video", type=Path, help="Directory or video file to add")
+    arger.add_argument("--name", "-n", help="Torrent name")
+    arger.add_argument("--no-metadata", action='store_true', dest='remove_metadata', help="Remove all metadata")
+    arger.add_argument("--post-only", action="store_true", help="Generate only the post bbcode")
+    arger.add_argument("--link", action="store_true", help="Link video file instead of moving")
+    arger.add_argument("--thumb-path", dest='thumbnail_path', type=Path, help="Path to thumbnail sheet")
+    if not argv:
+        arger.print_help()
+        sys.exit(0)
+    return arger.parse_args(argv)
+
+
+def validate_name(base_name: str):
+    if len(base_name) > (128 - len(".mp4")):
+        raise ValueError(f"Filename is too long: {base_name} ({len(base_name)} chars)")
+    if "?" in base_name:
+        raise ValueError(f"Filename contains invalid characters: {base_name}")
+
+
+def main():
+    args = parse_args(sys.argv[1:])
+    dir_or_video: Path = args.dir_or_video.expanduser().resolve()
+    if dir_or_video.is_dir():
+        dir_path = dir_or_video
+        if not dir_path.is_relative_to(TORRENT_DIR):
+            target_path = TORRENT_DIR / dir_path.name
+            logging.info(f"Linking to {target_path}")
+            shutil.copytree(src=dir_path, dst=target_path, copy_function=os.link, symlinks=False)
+            dir_path = target_path
+        torrent_path = make_torrent(dir_path)
+        logging.info(f"Saved torrent at {torrent_path}")
+
+        logging.info("Adding torrent file to Deluge")
+        add_torrent(torrent_path)
+        return
+
+    if not dir_or_video.is_file():
+        logging.error("Invalid file or directory")
+        return
+
+    video_path = dir_or_video
+    if video_path.suffix not in VIDEO_EXTENSIONS:
+        logging.error('not a video')
+        return
+
+    if args.remove_metadata:
+        logging.info('Removing video metadata')
+        save_path = video_path.with_stem(f'{video_path.stem}.nometadata')
+        video_path = clean_video(video_path=video_path, save_path=save_path)
+
+    video_path = move_video(video_path, new_name=args.name, link=args.link)
+    dir_path = video_path.parent
+
+    if p := args.thumbnail_path:
+        thumbnail_path = video_path.with_suffix('.jpg')
+        p.link_to(thumbnail_path)
+    else:
+        logging.info("Creating thumbnail tile")
+        old_thumb_path = video_path.with_suffix(".thumbnail.jpg")
+        thumbnail_path = video_path.with_suffix(".jpg")
+        if old_thumb_path.is_file():
+            old_thumb_path.rename(thumbnail_path)
+        ffmpeg.make_thumbnail_tile(video_path, image_path=thumbnail_path, skip_if_exists=True)
+        logging.info(f"Saved thumbnails at {thumbnail_path}")
+
+    post_bbcode = generate_post_bbcode(video_path, thumbnail_path)
+    video_path.with_name("post.txt").write_text(post_bbcode)
+    print(post_bbcode)
+
+    if args.post_only:
+        return
+
+    total_files = len(list(dir_path.rglob("*")))
+    logging.info(f"Creating torrent file from {dir_path}. Total files: {total_files}")
+    torrent_path = make_torrent(dir_path)
+    logging.info(f"Saved torrent at {torrent_path}")
+
+    logging.info("Adding torrent file to Deluge")
+    add_torrent(torrent_path)
+
+
+if __name__ == "__main__":
+    main()

torrentify_test.py

@@ -0,0 +1,60 @@
+from cmath import exp
+from torrentify import generate_title, parse_filename, ParsedFilename
+import pytest
+
+
+@pytest.mark.parametrize(
+    ["filename", "expected"],
+    [
+        [
+            "To Ki -- @Studio -- 2022-10-04",
+            {
+                "actors": ["To Ki"],
+                "date": "2022-10-04",
+                "studio": "Studio",
+                "tags": ["to.ki", "studio.com", "2022", "2022.10"],
+            },
+        ],
+        [
+            "To Ki, Ki To -- @Studio -- Title -- 2022-10-04",
+            {
+                "actors": ["To Ki", "Ki To"],
+                "date": "2022-10-04",
+                "studio": "Studio",
+                "title": "Title",
+                "tags": ["to.ki", "ki.to", "studio.com", "2022", "2022.10"],
+            },
+        ],
+    ],
+)
+def test_parse_filename(filename: str, expected: dict):
+    parsed = parse_filename(filename)
+    print(filename, parsed)
+    assert parsed == expected
+
+
+@pytest.mark.parametrize(
+    ["parsed", "expected"],
+    [
+        [
+            {
+                "actors": ["To Ki"],
+                "date": "2022-10-04",
+                "studio": "Studio",
+            },
+            'To Ki -- @Studio -- 2022-10-04',
+        ],
+        [
+            {
+                "actors": ["To Ki", "Ki To"],
+                "date": "2022-10-04",
+                "title": "Title",
+                "studio": "Studio",
+            },
+            'To Ki, Ki To -- @Studio -- Title -- 2022-10-04',
+        ],
+    ],
+)
+def test_generate_title(parsed: ParsedFilename, expected: str):
+    generated = generate_title(parsed)
+    assert generated == expected

ui_prompts.py

@@ -0,0 +1,22 @@
+import webview
+from webview import Window
+
+
+def webview_file_dialog():
+    def open_file_dialog(w: Window):
+        try:
+            return w.create_file_dialog(webview.FOLDER_DIALOG)[0]
+        except TypeError:
+            pass  # user exited file dialog without picking
+        finally:
+            w.hide()
+            w.destroy()
+
+    window = webview.create_window("", hidden=True)
+    webview.start(open_file_dialog, window)
+    # file will either be a string or None
+    return file
+
+
+if __name__ == "__main__":
+    print(webview_file_dialog())

upload_progress.py

@@ -0,0 +1,62 @@
+import dataclasses
+import functools
+import io
+import pathlib
+from datetime import timedelta, datetime
+from io import FileIO
+from typing import Callable
+
+import httpx
+
+
+class ProgressIO(io.BufferedReader):
+    @dataclasses.dataclass
+    class Progress:
+        total: int
+        processed: int
+
+        def percent(self) -> float:
+            return round(100 * self.processed / self.total, 2)
+
+    def __init__(self, file: pathlib.Path, on_progress: Callable[[Progress], None]):
+        super().__init__(raw=FileIO(str(file)))
+        self._size = file.stat().st_size
+        self._on_progress_throttled = self.throttle(1)(on_progress)
+        self._on_progress = on_progress
+        self._processed = 0
+        self._iter = None
+
+    def __next__(self) -> bytes:
+        chunk = super().__next__()
+        self._processed += len(chunk)
+        progress = ProgressIO.Progress(self._size, self._processed)
+        if self._processed == self._size:
+            self._on_progress(progress)
+        else:
+            self._on_progress_throttled(progress)
+
+        return chunk
+
+    @staticmethod
+    def throttle(seconds: float = 0):
+        period = timedelta(seconds=seconds)
+
+        def decorator(fn):
+            last_called = datetime.min
+
+            @functools.wraps(fn)
+            def wrapper(*args, **kwargs):
+                now = datetime.now()
+                nonlocal last_called
+                if now - last_called > period:
+                    last_called = now
+                    return fn(*args, **kwargs)
+
+            return wrapper
+
+        return decorator
+
+
+if __name__ == "__main__":
+    f = pathlib.Path(r"/path/to/file")
+    httpx.post("http://httpbin.org/post", files={"file": ProgressIO(f, on_progress=lambda p: print(p.percent()))})

vercel.json

@@ -0,0 +1,15 @@
+{
+    "version": 2,
+    "builds": [
+        {
+            "src": "./app.py",
+            "use": "@vercel/python"
+        }
+    ],
+    "routes": [
+        {
+            "src": "/(.*)",
+            "dest": "/app.py"
+        }
+    ]
+}

worker.py

@@ -0,0 +1,69 @@
+import flask
+import huey
+import sentry_sdk
+import sentry_sdk.integrations.flask
+from huey import signals
+
+sentry_sdk.init(
+    "https://379e370d4dee471b8ca5c7884d2137c0@o271257.ingest.sentry.io/6214795",
+    # Set traces_sample_rate to 1.0 to capture 100%
+    # of transactions for performance monitoring.
+    # We recommend adjusting this value in production.
+    traces_sample_rate=1.0,
+    with_locals=True,
+    max_breadcrumbs=100,
+    integrations=[sentry_sdk.integrations.flask.FlaskIntegration()],
+)
+
+worker = huey.SqliteHuey(filename='worker.sqlite3')
+
+
+@worker.signal(signals.SIGNAL_EXECUTING, signals.SIGNAL_LOCKED, signals.SIGNAL_CANCELED, signals.SIGNAL_REVOKED)
+def task_executing_handler(signal: str, task: huey.api.Task):
+    sentry_sdk.add_breadcrumb(
+        category='worker',
+        data=dict(
+            signal=signal,
+            task_id=task.id,
+            task_name=task.name,
+            task_retries=task.retries,
+            task_args=task.args,
+            task_kwargs=task.kwargs,
+        ),
+    )
+
+
+@worker.signal(signals.SIGNAL_ERROR)
+def task_not_executed_handler(signal: str, task: huey.api.Task, exc: Exception):
+    with sentry_sdk.push_scope():
+        sentry_sdk.capture_exception(exc)
+
+
+class SomeService:
+    @classmethod
+    @worker.task()
+    def do_something(cls, *args, **kwargs):
+        print(args, kwargs)
+
+    @classmethod
+    @worker.task()
+    def fail_something(cls, *args, **kwargs):
+        raise Exception('boom')
+
+    def do(self):
+        self.do_something(2, a=1)
+
+    def fail(self):
+        self.fail_something(2, a=1)
+
+
+def test_request():
+    flask_app = flask.Flask(__name__)
+
+    @flask_app.post('/')
+    def fail():
+        SomeService().fail()
+        return {'a': 5}
+
+    flask_app.add_url_rule('/', 'home', fail)
+    flask_app.test_client().post('/', json={'a': 10})

worker_app.py

@@ -0,0 +1,4 @@
+from worker import SomeService
+
+if __name__ == '__main__':
+    SomeService().fail()

wsgi_server.py

@@ -0,0 +1,19 @@
+from wsgiref.util import setup_testing_defaults
+from wsgiref.simple_server import make_server
+
+
+def simple_app(environ, start_response):
+    setup_testing_defaults(environ)
+
+    status = "200 OK"
+    headers = [("Content-type", "text/plain; charset=utf-8")]
+
+    start_response(status, headers)
+
+    ret = [("%s: %s\n" % (key, value)).encode("utf-8") for key, value in environ.items()]
+    return ret
+
+
+if __name__ == "__main__":
+    with make_server("", 8001, simple_app) as httpd:
+        httpd.handle_request()

xss.py

@@ -0,0 +1,10 @@
+html = '<b>i am html</b>'
+
+
+def a_func():
+    raise ValueError(html)
+
+
+if __name__ == '__main__':
+    a_func() # breakpoint here
+    ...