commit e63ab7b77e2a49ecb7a719a51c52a7888e8fae57 Author: Abdussamet Kocak Date: Sat Feb 18 07:39:08 2023 +0100 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e985853 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.vercel diff --git a/.vercelignore b/.vercelignore new file mode 100644 index 0000000..82195aa --- /dev/null +++ b/.vercelignore @@ -0,0 +1,2 @@ +venv +.idea \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..321f0d3 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,17 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Current File", + "type": "python", + "args": ["2022-12-06"], + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..bf5f1f6 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "python.testing.pytestArgs": [ + "venv" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} \ No newline at end of file diff --git a/anki.json b/anki.json new file mode 100644 index 0000000..243e99c --- /dev/null +++ b/anki.json @@ -0,0 +1 @@ +{"date": "2022-07-26T13:55:19.519504", "cookies": {"ankiweb": "eyJrIjogIkZicDI2UlFhb20xZkF4d04iLCAiYyI6IDEsICJ0IjogMTY1ODgzMjkxOX0.j3cM2VCQQZ7HW0i6cA4gJ1ysKGFl2jmQoUbEQ2CAid0"}} \ No newline at end of file diff --git a/anki.py b/anki.py new file mode 100644 index 0000000..5510e97 --- /dev/null +++ b/anki.py @@ -0,0 +1,255 @@ +import abc +import datetime +import json +import logging +import re +import sys +import typing +from abc import abstractmethod +from concurrent.futures import ThreadPoolExecutor +from typing import List + +import httpx +import typer +from typer import Typer, Option + +import sentry_sdk + +sentry_sdk.init( + "https://915a1bc84eb94d34b36cfdef13b3f3e5@o271257.ingest.sentry.io/6040098", + # Set traces_sample_rate to 1.0 to capture 100% + # of transactions for performance monitoring. + # We recommend adjusting this value in production. + traces_sample_rate=1.0, +) + +logging.basicConfig(level=logging.DEBUG) + + +class CookieStorage(abc.ABC): + @abstractmethod + def save_cookies(self, cookies: dict): + raise NotImplemented + + @abstractmethod + def load_cookies(self) -> typing.Optional[dict]: + raise NotImplemented + + +class FilesystemCookieStorage(CookieStorage): + def __init__(self, filename: str): + self.filename = filename + + def save_cookies(self, cookies: dict): + with open(self.filename, "wt") as f: + json.dump({"date": datetime.datetime.now().isoformat(), "cookies": cookies}, f) + + def load_cookies(self) -> typing.Optional[dict]: + try: + with open(self.filename) as f: + saved = json.load(f) + except FileNotFoundError: + return None + date_saved = datetime.datetime.fromisoformat(saved["date"]) + if (datetime.datetime.now() - date_saved).days > 1: + return None + return saved["cookies"] + + +class AnkiClient: + def __init__(self, cookie_storage: CookieStorage = None): + self._cookie_storage = cookie_storage or FilesystemCookieStorage("anki.json") + self._logger = logging.getLogger(self.__class__.__name__) + self._http = httpx.Client( + headers={ + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4681.0 Safari/537.36", + "X-Requested-With": "XMLHttpRequest", + }, + verify=False, + follow_redirects=False, + ) + + def _load_cookies(self): + cookies = self._cookie_storage.load_cookies() + if not cookies: + raise PermissionError("please login") + self._http.cookies = cookies + + @property + def session(self) -> dict: + """ + Returns session cookies + + :return: session cookies + """ + return self._cookie_storage.load_cookies() + + def login(self, username: str, password: str) -> dict: + """ + login and return cookies + + :return: cookies + """ + self._logger.debug("visiting login page") + res = self._http.get("https://ankiweb.net/account/login", follow_redirects=True) + res.raise_for_status() + + self._logger.debug("submitting login form") + hidden_fields = dict(re.findall(r'.*type="hidden" name="([^"]+)" value="([^"]+)".*', res.text, re.MULTILINE)) + res = self._http.post( + "https://ankiweb.net/account/login", data={**hidden_fields, "username": username, "password": password}, follow_redirects=True + ) + res.raise_for_status() + + cookies = {**self._http.cookies} + self._cookie_storage.save_cookies(cookies) + return cookies + + def get_editor_context(self): + self._load_cookies() + + add_info = self._http.get("https://ankiuser.net/edit/getAddInfo", follow_redirects=True).json() + note_types = {it["name"]: {"id": it["id"], "name": it["name"]} for it in add_info["notetypes"]} + decks = {it["name"]: {"id": it["id"], "name": it["name"]} for it in add_info["decks"]} + + with ThreadPoolExecutor(max_workers=len(note_types)) as pool: + + def fetch_note_fields(nid): + res = self._http.get( + "https://ankiuser.net/edit/getNotetypeFields", + params={"ntid": nid}, + follow_redirects=True, + ) + if res.is_error: + return [] + fields_info = res.json() + return [f["name"] for f in fields_info["fields"]] + + results = dict( + zip(note_types.keys(), pool.map(fetch_note_fields, [it["id"] for it in note_types.values()])) + ) + for t in note_types: + note_types[t]["fields"] = results[t] + + return note_types, decks + + def create_note(self, note_type: str, deck: str, fields: dict, tags: str = ""): + self._load_cookies() + + self._logger.debug("fetching note types") + add_info = self._http.get("https://ankiuser.net/edit/getAddInfo").json() + note_types = {it["name"]: it["id"] for it in add_info["notetypes"]} + decks = {it["name"]: it["id"] for it in add_info["decks"]} + + self._logger.debug("fetching note fields") + fields_info = self._http.get( + "https://ankiuser.net/edit/getNotetypeFields", params={"ntid": note_types[note_type]} + ).json() + model_fields = [f["name"] for f in fields_info["fields"]] + + editor_html = self._http.get("https://ankiuser.net/edit/").text + m = re.search(r"""new anki.Editor\('(ey.*)',.*\)""", editor_html) + csrf_token = m.group(1) + + model_id = note_types[note_type] + self._logger.debug(f"found note type: {model_id}") + self._logger.debug(f"note fields: {model_fields}") + + # unused fields must be an empty string, otherwise we get a 403 + ordered_fields = [fields.get(f, "") for f in model_fields] + payload = { + "nid": "", + "data": json.dumps([ordered_fields, tags]), + "csrf_token": csrf_token, + "mid": model_id, + "deck": decks[deck], + } + self._logger.debug("saving note") + res = self._http.post("https://ankiuser.net/edit/save", data=payload) + res.raise_for_status() + + +app = Typer(name="typer") +state = {"session": "anki.json", "creds": "anki.creds.json"} + + +@app.callback() +def callback(session: str = "anki.json", creds: str = "anki.creds.json"): + state.update(session=session, creds=creds) + + +def make_anki() -> AnkiClient: + anki = AnkiClient(FilesystemCookieStorage(state["session"])) + if not anki.session: + typer.echo("no valid session found, please login", err=True) + raise typer.Exit(1) + return anki + + +def try_login(): + try: + with open(state["creds"]) as f: + creds = json.load(f) + anki_login(username=creds["username"], password=creds["password"]) + except FileNotFoundError: + pass + + +@app.command("login") +def anki_login( + username: str = Option(..., prompt=True), + password: str = Option(..., prompt=True), + force: bool = False, +): + with open(state["creds"], "wt") as f: + json.dump({"username": username, "password": password}, f) + anki = AnkiClient(FilesystemCookieStorage(state["session"])) + if force or not anki.session: + anki.login(username=username, password=password) + + +@app.command("info") +def anki_info(): + try_login() + anki = make_anki() + note_types, decks = anki.get_editor_context() + logging.info("note types:") + for _, t in note_types.items(): + logging.info(t) + logging.info("decks:") + for d in decks.items(): + logging.info(d) + + +@app.command("note") +def anki_create_note( + deck: str = Option(...), + type: str = Option(...), + fields: List[str] = Option(..., "-f", "--field", help="Fields in k=v format"), + tags: str = "", +): + try_login() + anki = make_anki() + parsed_fields = dict([it.split("=", maxsplit=1) for it in fields]) + anki.create_note( + deck=deck, + note_type=type, + fields=parsed_fields, + tags=tags, + ) + + +if __name__ == "__main__": + logging.debug(f"Called with {sys.argv=}") + # try_login() + # anki = make_anki() + # anki.create_note( + # deck="Deutsch", + # note_type="Card with Reverse and Examples", + # fields={ + # "Front": "Es hat gerade an der Tür aufgeklingelt. Kannst du aufmachen?", + # "Back": "The doorbell just rang. Can you get it?", + # "Examples": "", + # }, + # ) + app() diff --git a/app.py b/app.py new file mode 100644 index 0000000..d438805 --- /dev/null +++ b/app.py @@ -0,0 +1,84 @@ +import typing +from typing import List, Dict + +from fastapi import FastAPI, Depends +from pydantic import BaseModel +from starlette.middleware.cors import CORSMiddleware +from starlette.requests import Request +from starlette.responses import Response, JSONResponse +import httpx + +from anki import AnkiClient, CookieStorage + +app = FastAPI( + title='anki', + docs_url='/') +app.add_middleware(CORSMiddleware, allow_origins=['*'], allow_methods=['*'], allow_headers=['*']) + + +class SessionCookieStorage(CookieStorage): + def __init__(self, req: Request, res: Response): + self.res = res + self.req = req + + def save_cookies(self, cookies: dict): + for k, v in cookies.items(): + self.res.set_cookie(k, v, max_age=24 * 60 * 60) + + def load_cookies(self) -> typing.Optional[dict]: + if 'ankiweb' not in self.req.cookies: + return None + return {**self.req.cookies} + + +def get_anki_client(req: Request, res: Response) -> AnkiClient: + return AnkiClient(SessionCookieStorage(req, res)) + + +@app.post('/login', response_model=Dict[str, str]) +async def login(username: str, password: str, anki: AnkiClient = Depends(get_anki_client)): + anki.login(username, password) + return anki.session + + +@app.get('/info', summary='Get note types and their fields', response_model=List[Dict]) +async def info(anki: AnkiClient = Depends(get_anki_client)): + return anki.get_editor_context() + + +class CreateNote(BaseModel): + deck: str + note_type: str + note_fields: Dict[str, str] + tags: str = '' + + +@app.post('/notes', summary='Create a note') +async def create_note(input: CreateNote, anki: AnkiClient = Depends(get_anki_client)): + anki.create_note(note_type=input.note_type, + deck=input.deck, + fields=input.note_fields, + tags=input.tags) + + +@app.exception_handler(Exception) +async def handle_errors(req: Request, error: Exception): + if isinstance(error, httpx.HTTPStatusError): + message = 'anki returned error' + status = error.response.status_code + elif isinstance(error, PermissionError): + message = str(error) + status = 401 + else: + message = 'oops' + status = 500 + + return JSONResponse(content={ + 'message': message + }, status_code=status) + + +if __name__ == '__main__': + import uvicorn + + uvicorn.run(app, host='0.0.0.0') diff --git a/aws_codecommit.py b/aws_codecommit.py new file mode 100644 index 0000000..3eb2a5c --- /dev/null +++ b/aws_codecommit.py @@ -0,0 +1,336 @@ +import itertools +import json +import logging +import pprint +import re +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor +from urllib.parse import quote_plus + +import boto3 +import httpx + +import urllib.parse +import subprocess +import typing + + +def list_repo_tags(*, repository_name: str, region: str, username: str, password: str) -> typing.List[dict]: + """ + Fetches the list of tags in a Codecommit repo using git ls-remote command + + :returns: List of {tag, commit} items + """ + url = "https://{username}:{password}@git-codecommit.{region}.amazonaws.com/v1/repos/{repository_name}".format( + username=quote_plus(username), + password=quote_plus(password), # passwords are base64-encoded and might contain "/" + region=region, + repository_name=repository_name, + ) + result = subprocess.run(["git", "ls-remote", "--refs", "--tags", url], text=True, stdout=subprocess.PIPE) + + tags = [] + for line in result.stdout.splitlines(keepends=False): + # lines are formatted as: + # 070161940636bab5d934af2702f4248e5a13eb29 refs/tags/ui_sandbox_acc104 + # and columns are delimited with "\t" + if "refs/tags/" in line: + commit_hash, tag_ref = line.split("\t", maxsplit=1) + tag = tag_ref.replace("refs/tags/", "") + tags.append({"commit": commit_hash, "tag": tag}) + + # natural-sort by tag name + # flora_prod_1 < ... < flora_prod_9 > flora_prod_10 + # vs lexical sort would give: + # flora_prod_1 < flora_prod_10 < ... < flora_prod_9 + tags = sorted(tags, key=lambda t: [int(it) if it.isdigit() else it for it in re.split(r"(\d+)", t["tag"])]) + + return tags + + +def chunk(it, size): + it = iter(it) + sentinel = () + return iter(lambda: tuple(itertools.islice(it, size)), sentinel) + + +def summarize_tags(tags: list[dict], take: int = 10) -> list[dict]: + re_prefix = re.compile(r"^(\D*)") + re_digits = re.compile(r"(\d+)") + grouped = defaultdict(list) + for t in tags: + tag_name = t["tag"] + if m := re_prefix.search(tag_name): + prefix = m.group(1) + grouped[prefix].append(t) + else: + print("no match", m) + flattened = [*grouped.pop("", [])] + for g, items in grouped.items(): + sorted_items = sorted( + items, key=lambda t: [int(it) if it.isdigit() else it for it in re_digits.split(t["tag"])], reverse=True + ) + flattened.extend(sorted_items[:take]) + + return flattened + + +abdus_creds = { + "username": "coder+1-at-400344683105", + "password": "4xdnGOtghKiT0M5CjulWf5raWwBSqT0Df2YkIY9Ws68=", +} + + +def get_commit_content(): + client = boto3.client("codecommit") + + res = client.get_file( + repositoryName="testing", + filePath="content.txt1", + commitSpecifier="vnext", + ) + print(res) + + +def main(): + import logging + from botocore.endpoint import Endpoint + + logging.basicConfig(level=logging.INFO) + + raw_make_request = Endpoint.make_request + + def intercepted_make_request(*args, **kwargs): + aws_response, parsed_response = raw_make_request(*args, **kwargs) + logging.info('AWS response: %s', parsed_response) + return aws_response, parsed_response + + Endpoint.make_request = intercepted_make_request + + session = boto3.Session( + aws_access_key_id="AKIAQKZTNEVLBHDG5Q3X", + aws_secret_access_key="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", + region_name="eu-central-1", + ) + cc = session.client("rds") + _ = cc.describe_db_clusters() + + +if __name__ == "__main__": + main() + exit() + # get_commit_content() + # flora b1cd47a221b84b2884b3d24198085a9f-f0dc2e747ff241649715c20d48232d8e + # oms 36363cc2f43b4cf5b1965af9f8925a4d-6deff091ce2d4c2483d8834c532a6f36 + # omnitron 36363cc2f43b4cf5b1965af9f8925a4d-97400227a37940539c55489e1354ffc2 + all_tags = list_repo_tags( + repository_name="b1cd47a221b84b2884b3d24198085a9f-f0dc2e747ff241649715c20d48232d8e", + username="sandbox-acc-api-user-at-023193265494", + password="sYPWKdhSx4JliXNI4ffRb4qWYBHLxA54nczSJ6CmYCY=", + region="eu-central-1", + ) + print(all_tags) + exit(0) + all_tags.append({"tag": "1asd", "commit": "asdf"}) + summarized = summarize_tags(all_tags, take=5) + commits = {t["commit"] for t in summarized} + session = boto3.Session( + aws_access_key_id="AKIAQKZTNEVLBHDG5Q3X", + aws_secret_access_key="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", + region_name="eu-central-1", + ) + client = session.client("codecommit") + + def get_commit_infos(commits: list[str]): + res = client.batch_get_commits( + repositoryName="36363cc2f43b4cf5b1965af9f8925a4d-97400227a37940539c55489e1354ffc2", + commitIds=commits, + ) + return res["commits"] + + with ThreadPoolExecutor() as pool: + results = list(pool.map(get_commit_infos, chunk(commits, 100))) + + clone_ssh_url = repo["repositoryMetadata"]["cloneUrlSsh"] + print(repo) + exit(0) +# exit() +from botocore.awsrequest import AWSRequest +from botocore.signers import RequestSigner + +# AWS Version 4 signing example + +# DynamoDB API (CreateTable) + +# See: http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html +# This version makes a POST request and passes request parameters +# in the body (payload) of the request. Auth information is passed in +# an Authorization header. +import sys, os, base64, datetime, hashlib, hmac +import requests # pip install requests + +# ************* REQUEST VALUES ************* +method = "POST" +service = "codecommit" +host = "codecommit.eu-central-1.amazonaws.com" +region = "eu-central-1" +endpoint = "https://codecommit.eu-central-1.amazonaws.com/" +# POST requests use a content type header. For DynamoDB, +# the content is JSON. +content_type = "application/x-amz-json-1.1" +# DynamoDB requires an x-amz-target header that has this format: +# DynamoDB_. +amz_target = "CodeCommit_20150413.GetReferences" + +# Request parameters for CreateTable--passed in a JSON block. +request_body = {"repositoryName": "36363cc2f43b4cf5b1965af9f8925a4d-6deff091ce2d4c2483d8834c532a6f36"} +request_parameters = json.dumps(request_body) + + +# Key derivation functions. See: +# http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python +def sign(key, msg): + return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest() + + +def getSignatureKey(key, date_stamp, regionName, serviceName): + kDate = sign(("AWS4" + key).encode("utf-8"), date_stamp) + kRegion = sign(kDate, regionName) + kService = sign(kRegion, serviceName) + kSigning = sign(kService, "aws4_request") + return kSigning + + +# Read AWS access key from env. variables or configuration file. Best practice is NOT +# to embed credentials in code. +access_key = os.environ.get("AWS_ACCESS_KEY_ID", "AKIAQKZTNEVLBHDG5Q3X") +secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY", "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") +if access_key is None or secret_key is None: + print("No access key is available.") + sys.exit() + +# Create a date for headers and the credential string +t = datetime.datetime.utcnow() +amz_date = t.strftime("%Y%m%dT%H%M%SZ") +date_stamp = t.strftime("%Y%m%d") # Date w/o time, used in credential scope + +# ************* TASK 1: CREATE A CANONICAL REQUEST ************* +# http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html + +# Step 1 is to define the verb (GET, POST, etc.)--already done. + +# Step 2: Create canonical URI--the part of the URI from domain to query +# string (use '/' if no path) +canonical_uri = "/" + +## Step 3: Create the canonical query string. In this example, request +# parameters are passed in the body of the request and the query string +# is blank. +canonical_querystring = "" + +# Step 4: Create the canonical headers. Header names must be trimmed +# and lowercase, and sorted in code point order from low to high. +# Note that there is a trailing \n. +canonical_headers = ( + "content-type:" + + content_type + + "\n" + + "host:" + + host + + "\n" + + "x-amz-date:" + + amz_date + + "\n" + + "x-amz-target:" + + amz_target + + "\n" +) + +# Step 5: Create the list of signed headers. This lists the headers +# in the canonical_headers list, delimited with ";" and in alpha order. +# Note: The request can include any headers; canonical_headers and +# signed_headers include those that you want to be included in the +# hash of the request. "Host" and "x-amz-date" are always required. +# For DynamoDB, content-type and x-amz-target are also required. +signed_headers = "content-type;host;x-amz-date;x-amz-target" + +# Step 6: Create payload hash. In this example, the payload (body of +# the request) contains the request parameters. +payload_hash = hashlib.sha256(request_parameters.encode("utf-8")).hexdigest() + +# Step 7: Combine elements to create canonical request +canonical_request = ( + method + + "\n" + + canonical_uri + + "\n" + + canonical_querystring + + "\n" + + canonical_headers + + "\n" + + signed_headers + + "\n" + + payload_hash +) + +# ************* TASK 2: CREATE THE STRING TO SIGN************* +# Match the algorithm to the hashing algorithm you use, either SHA-1 or +# SHA-256 (recommended) +algorithm = "AWS4-HMAC-SHA256" +credential_scope = date_stamp + "/" + region + "/" + service + "/" + "aws4_request" +string_to_sign = ( + algorithm + + "\n" + + amz_date + + "\n" + + credential_scope + + "\n" + + hashlib.sha256(canonical_request.encode("utf-8")).hexdigest() +) + +# ************* TASK 3: CALCULATE THE SIGNATURE ************* +# Create the signing key using the function defined above. +signing_key = getSignatureKey(secret_key, date_stamp, region, service) + +# Sign the string_to_sign using the signing_key +signature = hmac.new(signing_key, (string_to_sign).encode("utf-8"), hashlib.sha256).hexdigest() + +# ************* TASK 4: ADD SIGNING INFORMATION TO THE REQUEST ************* +# Put the signature information in a header named Authorization. +authorization_header = ( + algorithm + + " " + + "Credential=" + + access_key + + "/" + + credential_scope + + ", " + + "SignedHeaders=" + + signed_headers + + ", " + + "Signature=" + + signature +) + +# For DynamoDB, the request can include any headers, but MUST include "host", "x-amz-date", +# "x-amz-target", "content-type", and "Authorization". Except for the authorization +# header, the headers must be included in the canonical_headers and signed_headers values, as +# noted earlier. Order here is not significant. +# # Python note: The 'host' header is added automatically by the Python 'requests' library. +headers = { + "User-Agent": "aws-sdk-js/2.627.0 promise", + "Content-Type": content_type, + "X-Amz-Date": amz_date, + "X-Amz-Target": amz_target, + "Authorization": authorization_header, +} + +# ************* SEND THE REQUEST ************* +print("\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++") +print("Request URL = " + endpoint) + +r = httpx.post(endpoint, json=request_body, headers=headers) + +print("\nRESPONSE++++++++++++++++++++++++++++++++++++") +print("Response code: %d\n" % r.status_code) +print(r.text) diff --git a/aws_elasticache.py b/aws_elasticache.py new file mode 100644 index 0000000..39dcedf --- /dev/null +++ b/aws_elasticache.py @@ -0,0 +1,438 @@ +import dataclasses +import functools +import itertools +import logging +import re +import time +import typing +import uuid +from typing import Optional + +import boto3 +import botocore.exceptions +from botocore.exceptions import ClientError + +logger = logging.getLogger(__name__) + + +def retry(timeout: int, backoff_seconds: int = 10, bubble_errors: typing.List[typing.Type[Exception]] = None): + """ + Tries calling a function {timeout} seconds until it succeeds or gives up and throw an error + :param timeout: Timeout in seconds + :param backoff_seconds: Wait time between retries + :param bubble_errors: List of exceptions to bubble up + :return: Wrapped function + """ + bubble_errors = tuple(bubble_errors or []) + + def wrapped(func: typing.Callable): + @functools.wraps(func) + def inner(*args, **kwargs): + waited = 0 + while True: + try: + logger.debug(f"Calling {func}") + return func(*args, **kwargs) + except KeyboardInterrupt: + raise + except tuple(bubble_errors): + raise + except Exception as e: + doze = min(timeout - waited, backoff_seconds) + times = (timeout - waited) // backoff_seconds + logger.debug(f"{func} failed, will try {times} times in {doze} sec") + time.sleep(doze) + waited += backoff_seconds + if waited >= timeout: + raise TimeoutError(f"Couldn't get a successful result from {func} in {timeout} seconds") from e + + return inner + + return wrapped + + +@dataclasses.dataclass +class RedisInstance: + cluster_id: str + host: str + port: int + arn: str + user_group_ids: typing.List[str] = dataclasses.field(default_factory=list) + + +def paginate_by_marker( + func: callable, + list_field: str, +) -> typing.Iterable: + """ + Simplifies paginating over AWS results with a simpler interface than Paginators. + :param func: A function that accepts a parameter named `Marker` + :param list_field: Result field to iterate on + """ + marker = "" + while True: + result = func(Marker=marker) + + items = result.get(list_field, []) + yield from items + + if "Marker" not in result: + break + + marker = result["Marker"] + + +@dataclasses.dataclass +class RedisUser: + username: str + password: str + redis_key_prefix: str + + +class CreateUserResult(typing.TypedDict): + user_group_id: str + users: typing.List[RedisUser] + status: str + + +class RedisService: + def __init__(self, region: str): + self.elasticache = boto3.client("elasticache", region_name=region) + + def create_redis_acl_user( + self, + replication_group_id: str, + users: typing.List[RedisUser], + ) -> None: + """ + Creates a Redis ACL user. It will throw an error if the user group attached to the replication group is not ready. + This prevents us from quickly creating users. In that case, pass in a list of users instead. + + It takes about 2 minutes until the user is ready. + """ + repl_groups = self.elasticache.describe_replication_groups(ReplicationGroupId=replication_group_id) + user_group_id: str = repl_groups['ReplicationGroups'][0]['UserGroupIds'][0] + group = self.elasticache.describe_user_groups(UserGroupId=user_group_id)['UserGroups'][0] + + """ + Limits imposed by AWS: + + - User groups per replication group = 1 + - Users per user group = 100 + - Number of users = 1000 + - Number of user groups = 100 + https://docs.aws.amazon.com/AmazonElastiCache/latest/red-ug/Clusters.RBAC.html#Users-groups-to-RGs + + In short, we can't place more than 100 users on a single Redis instance. + """ + + total_users = len(group['UserIds']) + if total_users >= 100: + raise Exception('Replication group reached the limit of 100 users') + + self._update_or_create_user_group(user_group_id, users) + + def delete_redis_acl_user(self, username: str) -> None: + logger.info(f'Deleting user {username}') + self.elasticache.delete_user(UserId=username) + + # Even though we just deleted the user, it takes a couple of seconds until AWS updates user group status. + # So, we have to wait a bit before we can actually wait and check that the changes have propagated. + # Overall, it should take about 2-3 minutes to delete a user. + + def create_redis_acl( + self, + *, + replication_group_id: str, + description: str = None, + node_type: str, + cache_subnet_group_name: str, + security_group_id: str, + tags: typing.Dict[str, str] = None, + users: typing.List[RedisUser], + ) -> dict: + user_group_id = f"{replication_group_id}-ug" + # we have to wait (~1m) until the user group is ready before we can create the ACL + self._update_or_create_user_group(user_group_id, users, wait=True) + + cache_param_group = 'redis-acl-with-100-db' + self._ensure_cache_parameter_group(cache_param_group) + + logger.info(f"Creating Redis replication group {replication_group_id=}") + default_replication_group_kwargs = dict( + ReplicasPerNodeGroup=0, + Engine="redis", + EngineVersion="6.x", + TransitEncryptionEnabled=True, # must be True to use Redis ACL + MultiAZEnabled=False, + AutomaticFailoverEnabled=False, + ) + repl_result = self.elasticache.create_replication_group( + **default_replication_group_kwargs, + ReplicationGroupId=replication_group_id, + ReplicationGroupDescription=description or f'{replication_group_id} replication group', + CacheNodeType=node_type, + CacheSubnetGroupName=cache_subnet_group_name, + CacheParameterGroupName=cache_param_group, + UserGroupIds=[ + user_group_id, + ], + SecurityGroupIds=[ + security_group_id, + ], + Tags=[{"Key": k, "Value": v} for k, v in (tags or {}).items()], + ) + + # replication group takes ~8m minutes to be ready + + return { + "replication_group_id": repl_result["ReplicationGroup"]["ReplicationGroupId"], + 'arn': repl_result["ReplicationGroup"]["ARN"], + 'status': repl_result["ReplicationGroup"]["Status"], + 'users': users, + '_response': repl_result, + } + + def _update_or_create_user_group( + self, user_group_id: str, users: typing.List[RedisUser], wait: bool = False + ) -> None: + """ + Creates or updates a user group with the given users. Waits until the user group is active. + + :param user_group_id: + :param users: + """ + # we have to add the `default` user for backwards compatibility + existing_users = self.elasticache.describe_users( + Filters=[{"Name": "user-id", "Values": [u.username for u in users]}], + )["Users"] + existing_user_ids = [it["UserId"] for it in existing_users] + + created_user_ids = ["default"] + for user in users: + if user.username in existing_user_ids: + logger.info(f"User {user.username} is already exists") + created_user_ids.append(user.username) + continue + + redis_acl = f"on +@all -@dangerous ~{user.redis_key_prefix}*" + logger.info(f"Creating tenant {user.username=}") + user_result = self.elasticache.create_user( + UserId=user.username, + UserName=user.username, + Passwords=[user.password], + AccessString=redis_acl, + Engine="redis", + ) + created_user_ids.append(user_result["UserId"]) + + try: + group = self.elasticache.describe_user_groups(UserGroupId=user_group_id)["UserGroups"][0] + member_user_ids: typing.List[str] = group["UserIds"] + users_to_add = set(created_user_ids) - set(member_user_ids) + + if users_to_add: + self.elasticache.modify_user_group( + UserGroupId=user_group_id, + UserIdsToAdd=list(users_to_add), + ) + # this will take some time (~45s) until the changes propagate + except self.elasticache.exceptions.UserGroupNotFoundFault: + logger.info('User group does not exist, creating') + _ = self.elasticache.create_user_group( + UserGroupId=user_group_id, + Engine="redis", + UserIds=created_user_ids, + ) + # user group creation takes around 60s + if wait: + self._wait_user_group(user_group_id) + + @retry(timeout=60, backoff_seconds=5) + def _wait_user_group(self, user_group_id: str) -> None: + logger.info(f'Checking status of user group {user_group_id=}') + g = self.elasticache.describe_user_groups(UserGroupId=user_group_id)["UserGroups"][0] + assert g["Status"] == "active" + + def is_redis_acl_ready(self, replication_group_id: str) -> bool: + repl = self.elasticache.describe_replication_groups(ReplicationGroupId=replication_group_id)[ + 'ReplicationGroups' + ][0] + is_redis_available = repl['Status'] == 'available' + + user_group_id = repl['UserGroupIds'][0] + group = self.elasticache.describe_user_groups(UserGroupId=user_group_id)['UserGroups'][0] + is_group_available = group['Status'] == 'active' + + return is_group_available and is_redis_available + + def get_redis_acl(self, replication_group_id: str) -> dict: + redis = self.elasticache.describe_replication_groups(ReplicationGroupId=replication_group_id)[ + 'ReplicationGroups' + ][0] + return dict( + replication_group_id=redis['ReplicationGroupId'], + arn=redis['ARN'], + host=redis["NodeGroups"][0]["PrimaryEndpoint"]["Address"], + port=redis["NodeGroups"][0]["PrimaryEndpoint"]["Port"], + ) + + def delete_redis_acl(self, replication_group_id: str) -> None: + self.elasticache.delete_replication_group(ReplicationGroupId=replication_group_id) + + def _ensure_cache_parameter_group(self, parameter_group_name: str) -> None: + try: + _ = self.elasticache.describe_cache_parameter_groups(CacheParameterGroupName=parameter_group_name)[ + 'CacheParameterGroups' + ][0] + return + except self.elasticache.exceptions.CacheParameterGroupNotFoundFault: + pass + + logger.info(f"Creating cache parameter group {parameter_group_name=}") + _ = self.elasticache.create_cache_parameter_group( + CacheParameterGroupName=parameter_group_name, + CacheParameterGroupFamily="redis6.x", + Description="Redis parameter group for the Redis ACL", + ) + _ = self.elasticache.modify_cache_parameter_group( + CacheParameterGroupName=parameter_group_name, + ParameterNameValues=[ + { + "ParameterName": "databases", + "ParameterValue": "100", + }, + ], + ) + + +def akinon_create_redis( + *, + k8s_cluster_name: str, + region: str, + node_type: str, + replicas: int, + owner_arn: str, + app_name: str, + role: str, +): + subnet_group_name = f"{k8s_cluster_name}-redis-subg" + security_group_name = f"{k8s_cluster_name}-redis-sg" + + # owner arn is formatted like: + # arn:aws:iam::412344683105:user/myusername + username = owner_arn.split("/")[1] + tags = { + "CostCenter": f"{username}-{app_name}-{role}-redis", + } + + cache_cluster_id = uuid.uuid4().hex + r = RedisService(region) + return r.create_redis( + cluster_id=cache_cluster_id, + node_type=node_type, + replicas=replicas, + security_group_name=security_group_name, + subnet_group_name=subnet_group_name, + tags=tags, + ) + + +def akinon_create_redis_with_acl( + k8s_cluster_name: str, + owner_arn: str, + node_type: str, + app_name: str, + role: str, + users: typing.List[RedisUser], +) -> RedisInstance: + cache_cluster_id = uuid.uuid4().hex + subnet_group_name = f"{k8s_cluster_name}-redis-subg" + security_group_name = f"{k8s_cluster_name}-redis-sg" + + # owner arn is formatted as: + # arn:aws:iam::412344683105:user/myusername + username = owner_arn.split("/")[1] + tags = { + "CostCenter": f"{username}-{app_name}-{role}-redis", + } + + return r.create_redis_acl( + cluster_id=cache_cluster_id, + node_type=node_type, + subnet_group_name=subnet_group_name, + security_group_name=security_group_name, + tags=tags, + users=users, + ) + + +def chunk(it, size): + it = iter(it) + sentinel = () + return iter(lambda: tuple(itertools.islice(it, size)), sentinel) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format=f"%(asctime)s: {logging.BASIC_FORMAT}") + r = RedisService(region="eu-central-1") + # res = r.create_redis_acl( + # replication_group_id="zerodev3", + # description="zero redis dev", + # node_type="cache.t3.micro", + # cache_subnet_group_name="dev", + # security_group_id="sg-c11a77ac", + # tags={"CostCenter": "zero123"}, + # users=[RedisUser(username="u1", password="zeropassword1234", redis_key_prefix="zero:")], + # ) + # print(res) + + r.create_redis_acl_user( + replication_group_id='zerodev3', + users=[ + RedisUser( + username='u2', + password='testtesttesttest123', + redis_key_prefix='test:', + ) + ], + ) + # r.delete_redis_acl_user('test6') + while True: + if r.is_redis_acl_ready('zerodev3'): + logger.info('ready') + print(r.get_redis_acl('zerodev3')) + break + logger.info('not ready') + time.sleep(10) + exit() + + res = r.create_redis_acl( + replication_group_id="zerodev", + description="zero redis dev", + node_type="cache.t3.micro", + cache_subnet_group_name="dev", + security_group_id="sg-c11a77ac", + tags={"CostCenter": "zero123"}, + users=[RedisUser(username="zerouser", password="zeropassword1234", redis_key_prefix="zero:")], + ) + print(res) + # + # redis = r.get_redis_acl(cluster_arn="arn:aws:elasticache:eu-central-1:400344683105:replicationgroup:zero") + # r.add_redis_user( + # cluster_id=redis.cluster_id, + # users=[RedisUser(username="zero27", password="zero27password123", redis_key_prefix="zero27:")], + # ) + # print(r.add_redis_user(cluster_id="zero1", users=[RedisUser("zero26", "zero26password123", "zero26:")])) + # + # print( + # r.create_redis_with_acl( + # cluster_id="zero", + # cluster_description="zero redis", + # node_type="cache.t3.micro", + # subnet_group_name="testing-subnet", + # security_group_name="default", + # tags={"CostCenter": "zero123"}, + # users=[RedisUser(username="zerouser", password="zeropassword1234", redis_key_prefix="zero:")], + # ) + # ) diff --git a/aws_es.py b/aws_es.py new file mode 100644 index 0000000..0f69ca0 --- /dev/null +++ b/aws_es.py @@ -0,0 +1,238 @@ +import dataclasses +import datetime +import functools +import json +import logging +import time +import typing +import uuid + +import boto3 +import botocore +import requests +from botocore.client import BaseClient +from botocore.errorfactory import BaseClientExceptions +from botocore.exceptions import ClientError + +logger = logging.getLogger(__name__) + +# fmt: off +_response_describe_elasticsearch_domain_success = {'ResponseMetadata': {'RequestId': '38d7b555-2699-4868-9139-c47b1c2d70db', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '38d7b555-2699-4868-9139-c47b1c2d70db', 'content-type': 'application/json', 'content-length': '2314', 'date': 'Tue, 11 Jan 2022 07:52:24 GMT'}, 'RetryAttempts': 0}, 'DomainStatus': {'DomainId': '400344683105/es-d8a24850a0ae4b82af26', 'DomainName': 'es-d8a24850a0ae4b82af26', 'ARN': 'arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26', 'Created': True, 'Deleted': False, 'Endpoints': {'vpc': 'vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com'}, 'Processing': False, 'UpgradeProcessing': False, 'ElasticsearchVersion': '7.8', 'ElasticsearchClusterConfig': {'InstanceType': 't2.medium.elasticsearch', 'InstanceCount': 2, 'DedicatedMasterEnabled': False, 'ZoneAwarenessEnabled': True, 'ZoneAwarenessConfig': {'AvailabilityZoneCount': 2}, 'WarmEnabled': False, 'ColdStorageOptions': {'Enabled': False}}, 'EBSOptions': {'EBSEnabled': True, 'VolumeType': 'gp2', 'VolumeSize': 10}, 'AccessPolicies': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"es:*","Resource":"*"}]}', 'SnapshotOptions': {'AutomatedSnapshotStartHour': 0}, 'VPCOptions': {'VPCId': 'vpc-8c28b5e7', 'SubnetIds': ['subnet-004909d95a03c5fcd', 'subnet-b8f596f5'], 'AvailabilityZones': ['eu-central-1a', 'eu-central-1c'], 'SecurityGroupIds': ['sg-c11a77ac']}, 'CognitoOptions': {'Enabled': False}, 'EncryptionAtRestOptions': {'Enabled': False}, 'NodeToNodeEncryptionOptions': {'Enabled': False}, 'AdvancedOptions': {'override_main_response_version': 'false', 'rest.action.multi.allow_explicit_index': 'true'}, 'ServiceSoftwareOptions': {'CurrentVersion': 'R20211203-P2', 'NewVersion': '', 'UpdateAvailable': False, 'Cancellable': False, 'UpdateStatus': 'COMPLETED', 'Description': 'There is no software update available for this domain.', 'AutomatedUpdateDate': datetime.datetime(2021, 12, 14, 3, 38, 38), 'OptionalDeployment': False}, 'DomainEndpointOptions': {'EnforceHTTPS': True, 'TLSSecurityPolicy': 'Policy-Min-TLS-1-0-2019-07', 'CustomEndpointEnabled': False}, 'AdvancedSecurityOptions': {'Enabled': False, 'InternalUserDatabaseEnabled': False}, 'AutoTuneOptions': {'State': 'ENABLE_IN_PROGRESS'}}} +_response_list_domain_names_success = {'ResponseMetadata': {'RequestId': '25923514-a426-4223-939a-4436f1d6e243', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '25923514-a426-4223-939a-4436f1d6e243', 'content-type': 'application/json', 'content-length': '87', 'date': 'Tue, 11 Jan 2022 07:53:37 GMT'}, 'RetryAttempts': 0}, 'DomainNames': [{'DomainName': 'es-d8a24850a0ae4b82af26', 'EngineType': 'Elasticsearch'}]} +_response_describe_elasticsearch_domains_success = {'ResponseMetadata': {'RequestId': '61cfe5f4-42e7-4233-bca9-53f0ca62099d', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '61cfe5f4-42e7-4233-bca9-53f0ca62099d', 'content-type': 'application/json', 'content-length': '2320', 'date': 'Tue, 11 Jan 2022 07:55:25 GMT'}, 'RetryAttempts': 0}, 'DomainStatusList': [{'DomainId': '400344683105/es-d8a24850a0ae4b82af26', 'DomainName': 'es-d8a24850a0ae4b82af26', 'ARN': 'arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26', 'Created': True, 'Deleted': False, 'Endpoints': {'vpc': 'vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com'}, 'Processing': False, 'UpgradeProcessing': False, 'ElasticsearchVersion': '7.8', 'ElasticsearchClusterConfig': {'InstanceType': 't2.medium.elasticsearch', 'InstanceCount': 2, 'DedicatedMasterEnabled': False, 'ZoneAwarenessEnabled': True, 'ZoneAwarenessConfig': {'AvailabilityZoneCount': 2}, 'WarmEnabled': False, 'ColdStorageOptions': {'Enabled': False}}, 'EBSOptions': {'EBSEnabled': True, 'VolumeType': 'gp2', 'VolumeSize': 10}, 'AccessPolicies': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"es:*","Resource":"*"}]}', 'SnapshotOptions': {'AutomatedSnapshotStartHour': 0}, 'VPCOptions': {'VPCId': 'vpc-8c28b5e7', 'SubnetIds': ['subnet-004909d95a03c5fcd', 'subnet-b8f596f5'], 'AvailabilityZones': ['eu-central-1a', 'eu-central-1c'], 'SecurityGroupIds': ['sg-c11a77ac']}, 'CognitoOptions': {'Enabled': False}, 'EncryptionAtRestOptions': {'Enabled': False}, 'NodeToNodeEncryptionOptions': {'Enabled': False}, 'AdvancedOptions': {'override_main_response_version': 'false', 'rest.action.multi.allow_explicit_index': 'true'}, 'ServiceSoftwareOptions': {'CurrentVersion': 'R20211203-P2', 'NewVersion': '', 'UpdateAvailable': False, 'Cancellable': False, 'UpdateStatus': 'COMPLETED', 'Description': 'There is no software update available for this domain.', 'AutomatedUpdateDate': datetime.datetime(2021, 12, 14, 3, 38, 38), 'OptionalDeployment': False}, 'DomainEndpointOptions': {'EnforceHTTPS': True, 'TLSSecurityPolicy': 'Policy-Min-TLS-1-0-2019-07', 'CustomEndpointEnabled': False}, 'AdvancedSecurityOptions': {'Enabled': False, 'InternalUserDatabaseEnabled': False}, 'AutoTuneOptions': {'State': 'ENABLE_IN_PROGRESS'}}]} +_response_delete_elasticsearch_domain_not_found = ClientError({'Error': {'Message': 'Domain not found: asd', 'Code': 'ResourceNotFoundException'}, 'ResponseMetadata': {'RequestId': 'd0669560-77f6-424c-bbc6-fb5878daa494', 'HTTPStatusCode': 409, 'HTTPHeaders': {'x-amzn-requestid': 'd0669560-77f6-424c-bbc6-fb5878daa494', 'x-amzn-errortype': 'ResourceNotFoundException', 'content-type': 'application/json', 'content-length': '35', 'date': 'Tue, 11 Jan 2022 08:06:07 GMT'}, 'RetryAttempts': 0}}, 'DeleteElasticsearchDomain') +_response_delete_elasticsearch_domain_success = {'ResponseMetadata': {'RequestId': '1e9046f4-c4ee-4fda-8dfe-dd23f0161b7e', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '1e9046f4-c4ee-4fda-8dfe-dd23f0161b7e', 'content-type': 'application/json', 'content-length': '2312', 'date': 'Tue, 11 Jan 2022 08:31:33 GMT'}, 'RetryAttempts': 0}, 'DomainStatus': {'DomainId': '400344683105/es-d8a24850a0ae4b82af26', 'DomainName': 'es-d8a24850a0ae4b82af26', 'ARN': 'arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26', 'Created': True, 'Deleted': True, 'Endpoints': {'vpc': 'vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com'}, 'Processing': True, 'UpgradeProcessing': False, 'ElasticsearchVersion': '7.8', 'ElasticsearchClusterConfig': {'InstanceType': 't2.medium.elasticsearch', 'InstanceCount': 2, 'DedicatedMasterEnabled': False, 'ZoneAwarenessEnabled': True, 'ZoneAwarenessConfig': {'AvailabilityZoneCount': 2}, 'WarmEnabled': False, 'ColdStorageOptions': {'Enabled': False}}, 'EBSOptions': {'EBSEnabled': True, 'VolumeType': 'gp2', 'VolumeSize': 10}, 'AccessPolicies': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"es:*","Resource":"*"}]}', 'SnapshotOptions': {'AutomatedSnapshotStartHour': 0}, 'VPCOptions': {'VPCId': 'vpc-8c28b5e7', 'SubnetIds': ['subnet-004909d95a03c5fcd', 'subnet-b8f596f5'], 'AvailabilityZones': ['eu-central-1a', 'eu-central-1c'], 'SecurityGroupIds': ['sg-c11a77ac']}, 'CognitoOptions': {'Enabled': False}, 'EncryptionAtRestOptions': {'Enabled': False}, 'NodeToNodeEncryptionOptions': {'Enabled': False}, 'AdvancedOptions': {'override_main_response_version': 'false', 'rest.action.multi.allow_explicit_index': 'true'}, 'ServiceSoftwareOptions': {'CurrentVersion': 'R20211203-P2', 'NewVersion': '', 'UpdateAvailable': False, 'Cancellable': False, 'UpdateStatus': 'COMPLETED', 'Description': 'There is no software update available for this domain.', 'AutomatedUpdateDate': datetime.datetime(2021, 12, 14, 3, 38, 38), 'OptionalDeployment': False}, 'DomainEndpointOptions': {'EnforceHTTPS': True, 'TLSSecurityPolicy': 'Policy-Min-TLS-1-0-2019-07', 'CustomEndpointEnabled': False}, 'AdvancedSecurityOptions': {'Enabled': False, 'InternalUserDatabaseEnabled': False}, 'AutoTuneOptions': {'State': 'ENABLE_IN_PROGRESS'}}} +# fmt: on + + +def retry(timeout: int, backoff_seconds: int = 10, bubble_errors: typing.List[typing.Type[Exception]] = None): + """ + Tries calling a function {timeout} seconds until it succeeds or gives up and throw an error + :param timeout: Timeout in seconds + :param backoff_seconds: Wait time between retries + :param bubble_errors: List of exceptions to bubble up + :return: Wrapped function + """ + bubble_errors = tuple(bubble_errors or []) + + def wrapped(func: typing.Callable): + @functools.wraps(func) + def func_retrier(*args, **kwargs): + waited = 0 + started_at = datetime.datetime.now() + while True: + try: + logger.debug(f"Calling {func}") + result = func(*args, **kwargs) + logger.debug(f"{func} completed in {(datetime.datetime.now() - started_at)}") + return result + except KeyboardInterrupt: + raise + except tuple(bubble_errors): + raise + except Exception as e: + doze = min(timeout - waited, backoff_seconds) + times = (timeout - waited) // backoff_seconds + logger.debug(f"{func} failed, will try {times} times in {doze} sec", exc_info=True) + time.sleep(doze) + waited += backoff_seconds + if waited >= timeout: + raise TimeoutError(f"Couldn't get a successful result from {func} in {timeout} seconds") from e + + return func_retrier + + return wrapped + + +@dataclasses.dataclass +class ElasticsearchInstance: + domain_name: str + endpoint: str + arn: str + + +class ElasticsearchService: + domain_name_format = "es-{}" + default_version = "7.8" + valid_versions = {"7.10", "7.9", "7.8", "7.7", "7.4", "7.1", "6.8", "6.7", "6.5", "6.4", "6.3", "6.2", "6.0", "5.6", "5.5"} # fmt: skip + + def __init__(self, region: str): + self.es = boto3.client("es", region_name=region) + + def create_instance( + self, + node_type: str, + total_nodes: int, + subnet_group_ids: typing.List[str], + security_group_id: str, + version: str = None, + tags: dict = None, + volume_size_gb: int = 30, + total_availability_zones: int = 3, + ) -> ElasticsearchInstance: + if not version: + version = self.default_version + assert version in self.valid_versions, "Invalid version" + + domain_name = self.domain_name_format.format(uuid.uuid4().hex[:20]) + logger.info( + "Creating a new Elasticsearch domain", + extra=dict(version=version, domain_name=domain_name, node_type=node_type, total_nodes=total_nodes), + ) + es_result = self.es.create_elasticsearch_domain( + DomainName=domain_name, + ElasticsearchVersion=version, + ElasticsearchClusterConfig={ + "InstanceType": node_type, + "InstanceCount": total_nodes, + "ZoneAwarenessEnabled": True, + "ZoneAwarenessConfig": { + "AvailabilityZoneCount": total_availability_zones, + }, + }, + VPCOptions={ + "SubnetIds": subnet_group_ids, + "SecurityGroupIds": [security_group_id], + }, + EBSOptions={ + "EBSEnabled": True, + "VolumeType": "gp2", + "VolumeSize": volume_size_gb, + }, + AccessPolicies=json.dumps( + { + "Version": "2012-10-17", + "Statement": [{"Effect": "Allow", "Principal": "*", "Action": "es:*", "Resource": "*"}], + } + ), + DomainEndpointOptions={"EnforceHTTPS": True}, + TagList=[{"Key": k, "Value": v} for k, v in (tags or {}).items()], + ) + logger.info("Waiting until ES domain becomes available. This will take a while (~15 min)") + return self._find_instance(domain_name) + + @retry(timeout=20 * 60, backoff_seconds=30) + def _find_instance(self, domain_name: str) -> ElasticsearchInstance: + result = self.es.describe_elasticsearch_domain(DomainName=domain_name)["DomainStatus"] + return ElasticsearchInstance(endpoint=result["Endpoints"]["vpc"], arn=result["ARN"], domain_name=domain_name) + + def find_domain_name_by_arn(self, arn: str): + res = self.es.list_domain_names(EngineType='Elasticsearch') + domain_names = [it['DomainName'] for it in res['DomainNames']] + + res = self.es.describe_elasticsearch_domains(DomainNames=domain_names) + domain_name_by_arn = {it['ARN']: it['DomainName'] for it in res['DomainStatusList']} + + print(res) + + +def main(): + username, password = 'bgNNs4EWPTsuQJ24', '3asinFjikT5MgwRz!' + + +""" +GET _search +{ + "query": { + "match_all": {} + } +} + +### +PUT /my-index +PUT /my-index2 +PUT /my-index3 +PUT /my-index3-4 + +### +DELETE /my-index*?expand_wildcards=all + +### +DELETE /my-index1 + +### +GET /my-index +GET /_stats/indexing +### +GET /_cat/indices?v + +### + +GET /kibana_sample_data_ecommerce/ + +### +GET /_all/_mapping + +## delete multiple +PUT /my-index +PUT /my-index2 +DELETE /my-index,my-index2 +GET /my-index + +### +GET /_aliases + +### +GET /_stats/ + + +user: qdSmWKS8p3VrUyBC +pwd: un6f2vTrRjoa8orn! +""" + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format=f"%(asctime)s: {logging.BASIC_FORMAT}") + logging.getLogger("botocore").setLevel(logging.INFO) + + # ElasticsearchService('eu-central-1') + e = ElasticsearchService("eu-central-1") + + esi = ElasticsearchInstance( + endpoint='vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com', + arn='arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26', + domain_name='es-d8a24850a0ae4b82af26', + ) + + # e.find_instance_by_arn(esi.arn) + # exit() + try: + es = boto3.client('es') + res = es.delete_elasticsearch_domain(DomainName=esi.domain_name) + print(res) + except ClientError as e: + error_code = e.__class__.__name__ + print(error_code) + + e = ElasticsearchService("eu-central-1") + # print( + # e.create_instance( + # node_type="t2.medium.elasticsearch", + # total_nodes=2, + # volume_size_gb=10, + # total_availability_zones=2, + # security_group_id="sg-c11a77ac", + # subnet_group_ids=["subnet-b8f596f5", "subnet-004909d95a03c5fcd"], + # tags={"CostCenter": "my"}, + # ) + # ) diff --git a/aws_rds.py b/aws_rds.py new file mode 100644 index 0000000..c189258 --- /dev/null +++ b/aws_rds.py @@ -0,0 +1,401 @@ +import dataclasses +import datetime +import functools +import logging +import time +import typing +import uuid + +import boto3 + +# import mysql.connector.connection +import psycopg2 +import pymysql +from psycopg2 import sql +import psycopg2.extensions + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class Credentials: + username: str + password: str + + @classmethod + def new(cls): + username_format = "u{}" # must start with a letter + return cls( + username=username_format.format(uuid.uuid4().hex), + password=uuid.uuid4().hex, + ) # make sure it starts with a letter + + +@dataclasses.dataclass +class DbConnection: + endpoint: str + port: int + + +@dataclasses.dataclass +class DbInstance(DbConnection): + cluster_id: str + arn: str + reader_endpoint: typing.Optional[str] = None + database_name: typing.Optional[str] = None + user: typing.Optional[Credentials] = None + master_user: typing.Optional[Credentials] = None + + +def retry(timeout: int, backoff_seconds: int = 10, bubble_errors: typing.List[typing.Type[Exception]] = None): + """ + Tries calling a function {timeout} seconds until it succeeds or gives up and throw an error + :param timeout: Timeout in seconds + :param backoff_seconds: Wait time between retries + :param bubble_errors: List of exceptions to bubble up + :return: Wrapped function + """ + bubble_errors = tuple(bubble_errors or []) + + def wrapped(func: typing.Callable): + @functools.wraps(func) + def func_retrier(*args, **kwargs): + waited = 0 + started_at = datetime.datetime.now() + while True: + try: + logger.debug(f"Calling {func}") + result = func(*args, **kwargs) + logger.debug(f"{func} completed in {(datetime.datetime.now() - started_at)}") + return result + except KeyboardInterrupt: + raise + except tuple(bubble_errors): + raise + except Exception as e: + doze = min(timeout - waited, backoff_seconds) + times = (timeout - waited) // backoff_seconds + logger.debug(f"{func} failed, will try {times} times in {doze} sec", exc_info=True) + time.sleep(doze) + waited += backoff_seconds + if waited >= timeout: + raise TimeoutError(f"Couldn't get a successful result from {func} in {timeout} seconds") from e + + return func_retrier + + return wrapped + + +def paginate_by_marker( + func: callable, + list_field: str, +) -> typing.Iterable: + """ + Simplifies paginating over AWS results with a simpler interface than Paginators. + :param func: A function that accepts a parameter named `Marker` + :param list_field: Result field to iterate on + """ + marker = "" + while True: + result = func(Marker=marker) + + items = result.get(list_field, []) + yield from items + + if "Marker" not in result: + break + + marker = result["Marker"] + + +class PostgresqlService: + db_type = "Postgresql" + db_engine = "aurora-postgresql" + db_engine_version = "10.11" + default_master_database_name = "postgres" + cluster_id_format = "pg-{}" # must start with a letter + tenant_database_name_format = "db{}" # must start with a letter + + def __init__(self, region: str): + self.rds = boto3.client("rds", region_name=region) + + def create_db( + self, + total_instances: int, + db_subnet_group_name: str, + node_type: str, + security_group_id: str, + engine_version: str = None, + backup_retention_days: int = 30, + public: bool = False, + tags: dict = None, + ): + if not engine_version: + engine_version = self.db_engine_version + + cluster_id = self.cluster_id_format.format(uuid.uuid4().hex) + + master_creds = Credentials.new() + + logger.info( + f"Creating {self.db_type} cluster", + extra=dict( + cluster_id=cluster_id, + engine_version=engine_version, + db_subnet_group_name=db_subnet_group_name, + security_group_id=security_group_id, + ), + ) + + db_result = self.rds.create_db_cluster( + DBClusterIdentifier=cluster_id, + Engine=self.db_engine, + EngineVersion=engine_version, + MasterUsername=master_creds.username, + MasterUserPassword=master_creds.password, + Tags=[{"Key": k, "Value": v} for k, v in (tags or {}).items()], + DBSubnetGroupName=db_subnet_group_name, + VpcSecurityGroupIds=[security_group_id], + BackupRetentionPeriod=backup_retention_days, + )["DBCluster"] + + endpoint = db_result["Endpoint"] + reader_endpoint = db_result["ReaderEndpoint"] + port = db_result["Port"] + arn = db_result["DBClusterArn"] + + for i in range(total_instances): + instance_id = f"{cluster_id}-Instance-{i}" + logger.info( + "Creating instances on the cluster", + extra=dict( + cluster_id=cluster_id, + instance_id=instance_id, + node_type=node_type, + ), + ) + instance_result = self.rds.create_db_instance( + DBInstanceIdentifier=instance_id, + DBClusterIdentifier=cluster_id, + DBInstanceClass=node_type, + Engine=self.db_engine, + PubliclyAccessible=public, + Tags=[{"Key": k, "Value": v} for k, v in (tags or {}).items()], + ) + + tenant_database_name = self.tenant_database_name_format.format(uuid.uuid4().hex) + db = DbInstance( + cluster_id=cluster_id, + arn=arn, + endpoint=endpoint, + reader_endpoint=reader_endpoint, + port=port, + database_name=tenant_database_name, + master_user=master_creds, + ) + print(db) # TODO: remove + + logger.info("Waiting until DB instances are online. This will take a while (~5 min)") + logger.info("Connecting DB instance using master credentials", extra=dict(endpoint=db.endpoint)) + self._check_connection( + connection=db, + credentials=master_creds, + ) + + tenant_creds = Credentials.new() + logger.info( + "Connection successful. Creating a new database and user", + extra=dict( + endpoint=endpoint, + username=tenant_creds.username, + database_name=tenant_database_name, + ), + ) + self.create_tenant( + connection=db, + master_user=master_creds, + database_name=tenant_database_name, + tenant_user=tenant_creds, + ) + db.user = tenant_creds + + return db + + @retry(timeout=10 * 60) + def _check_connection(self, connection: DbConnection, credentials: Credentials, database_name: str = "postgres"): + psycopg2.connect( + host=connection.endpoint, + port=connection.port, + dbname=database_name, + user=credentials.username, + password=credentials.password, + connect_timeout=15, + ).close() + + def _find_instance(self, endpoint: str) -> typing.Optional[DbInstance]: + func = functools.partial( + self.rds.describe_db_clusters, Filters=[{"Name": "engine", "Values": [self.db_engine]}], MaxRecords=100 + ) + cluster = None + for it in paginate_by_marker(func, "DBClusters"): + if it["Endpoint"] == endpoint: + cluster = it + break + if cluster: + return DbInstance( + endpoint=cluster["Endpoint"], + port=cluster["Port"], + cluster_id=cluster["DBClusterIdentifier"], + arn=cluster["DBClusterArn"], + reader_endpoint=cluster["ReaderEndpoint"], + ) + return None + + def create_tenant( + self, + connection: DbConnection, + master_user: Credentials, + tenant_user: Credentials, + database_name: typing.Optional[str] = None, + ) -> DbInstance: + if not database_name: + database_name = self.tenant_database_name_format.format(uuid.uuid4().hex) + try: + con = psycopg2.connect( + host=connection.endpoint, + port=connection.port, + dbname=self.default_master_database_name, + user=master_user.username, + password=master_user.password, + connect_timeout=30, + ) + con.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + cur = con.cursor() + + cur.execute( + sql.SQL("CREATE DATABASE {}").format( + sql.Identifier(database_name), + ), + ) + cur.execute( + sql.SQL("CREATE USER {} WITH ENCRYPTED PASSWORD {}").format( + sql.Identifier(tenant_user.username), + sql.Placeholder(), + ), + [tenant_user.password], + ) + cur.execute( + sql.SQL("GRANT ALL PRIVILEGES ON DATABASE {} TO {}").format( + sql.Identifier(database_name), + sql.Identifier(tenant_user.username), + ) + ) + + # `with` block would create an implicit transaction + con.close() + + db = self._find_instance(connection.endpoint) + db.database_name = database_name + db.user = tenant_user + db.master_user = master_user + return db + except psycopg2.Error: + logging.error( + f"Failed to create database and user", + exc_info=True, + extra=dict(endpoint=connection.endpoint, database_name=database_name), + ) + raise + + +class MysqlService(PostgresqlService): + db_type = "MySQL" + db_engine = "aurora-mysql" + db_engine_version = "5.7.12" + default_master_database_name = "mysql" + cluster_id_format = "mysql-{}" # must start with a letter + + @retry(timeout=10 * 60) + def _check_connection(self, connection: DbConnection, credentials: Credentials, database_name: str = "postgres"): + pymysql.connect( + host=connection.endpoint, + port=connection.port, + user=credentials.username, + password=credentials.password, + connect_timeout=15, + ).close() + + def create_tenant( + self, + connection: DbConnection, + master_user: Credentials, + tenant_user: Credentials, + database_name: typing.Optional[str] = None, + ) -> DbInstance: + if not database_name: + database_name = self.tenant_database_name_format.format(uuid.uuid4().hex) + + try: + with pymysql.connect( + host=connection.endpoint, + port=connection.port, + user=master_user.username, + password=master_user.password, + connect_timeout=15, + ) as con: + cur = con.cursor() + cur.execute(f"CREATE DATABASE {database_name}") + cur.execute(f"CREATE USER {tenant_user.username} IDENTIFIED BY '{tenant_user.password}'") + cur.execute( + f"GRANT ALL PRIVILEGES ON {database_name}.* TO {tenant_user.username}@'%' IDENTIFIED BY '{tenant_user.password}'" + ) + + db = self._find_instance(endpoint=connection.endpoint) + db.database_name = database_name + db.master_user = master_user + db.user = tenant_user + return db + except pymysql.Error: + logger.error( + "Failed to create database and user", + exc_info=True, + extra=dict(endpoint=connection.endpoint, database_name=database_name), + ) + raise + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format=f"%(asctime)s: {logging.BASIC_FORMAT}") + logging.getLogger("botocore").setLevel(logging.INFO) + + # m = MysqlService("eu-central-1") + # db = m.create_db( + # total_instances=1, + # db_subnet_group_name="mydbsubnet", + # security_group_id="sg-c11a77ac", + # node_type="db.t3.medium", + # tags={"CostCenter": "hello"}, + # public=True, + # ) + # print(db) + + # p = PostgresqlService("eu-central-1") + # db = p.create_db( + # total_instances=1, + # db_subnet_group_name="mydbsubnet", + # security_group_id="sg-c11a77ac", + # node_type="db.t3.medium", + # tags={"CostCenter": "hello"}, + # public=True, + # ) + # p.create_tenant( + # connection=DbConnection( + # endpoint="pg0a085614d5eb4f34affd2a24b1fb18c4.cluster-c8v5rp0ouaey.eu-central-1.rds.amazonaws.com", + # port=5432, + # ), + # master_user=Credentials( + # username="u7d0f9621f5a5419288a4ef7ce9fd5fe", password="ca9473879b794dd3a0b0f0d2a8d4aebb" + # ), + # tenant_user=Credentials.new(), + # database_name="mydb", + # ) diff --git a/aws_redis_acl.py b/aws_redis_acl.py new file mode 100644 index 0000000..b56d096 --- /dev/null +++ b/aws_redis_acl.py @@ -0,0 +1,107 @@ +import logging +import typing + +import boto3 +import botocore.client + + +def paginate_by_marker( + func: callable, + list_field: str, +) -> typing.Iterable: + """ + Simplifies paginating over AWS results with a simpler interface than Paginators. + :param func: A function that accepts a parameter named `Marker` + :param list_field: Result field to iterate on + """ + marker = "" + while True: + result = func(Marker=marker) + + items = result.get(list_field, []) + yield from items + + if "Marker" not in result: + break + + marker = result["Marker"] + + +logger = logging.getLogger(__name__) + + +class Rediser: + @classmethod + def new(cls): + return cls(boto3.client("elasticache", region_name='eu-central-1')) + + def __init__(self, elasticache: botocore.client.BaseClient): + self.elasticache = elasticache + + def _get_redis_acl( + self, + cluster_id: str = None, + status: str = "available", + ) -> typing.Optional[dict]: + """ + Finds a Redis replication group by its ID + + :param cluster_id: Replication group ID + :param status: Replication status. One of available/starting/modifying/deleting. + :return: RedisInstance or throws LookupError error + """ + try: + redis = self.elasticache.describe_replication_groups(ReplicationGroupId=cluster_id)["ReplicationGroups"][0] + if redis["Status"] != status: + raise Exception(f"Found redis with {cluster_id=} but its status={redis['Status']}") + except self.elasticache.exceptions.ReplicationGroupNotFoundFault: + return None + + return dict( + cluster_id=redis["ReplicationGroupId"], + host=redis["NodeGroups"][0]["PrimaryEndpoint"]["Address"], + port=redis["NodeGroups"][0]["PrimaryEndpoint"]["Port"], + arn=redis["ARN"], + user_group_ids=redis["UserGroupIds"], + ) + + def create_redis_acl( + self, + *, + cluster_id: str, + cluster_description: typing.Optional[str] = None, + node_type: str, + subnet_group_name: str, + security_group_id: str, + ): + redis = self._get_redis_acl(cluster_id=cluster_id) + + if not redis: + res = self.elasticache.create_replication_group( + ReplicationGroupId=cluster_id, + ReplicationGroupDescription=cluster_description or cluster_id, + ReplicasPerNodeGroup=0, + Engine="redis", + EngineVersion="6.x", + CacheNodeType=node_type, + CacheSubnetGroupName=subnet_group_name, + TransitEncryptionEnabled=True, # must be True to use Redis ACL + UserGroupIds=[ + user_group_id, + ], + SecurityGroupIds=[ + security_group_id, + ], + Tags=[{"Key": k, "Value": v} for k, v in tags.items()], + MultiAZEnabled=False, + AutomaticFailoverEnabled=False, + ) + + +def main(): + r = Rediser.new() + res = r.create_redis_acl(cluster_id='hey', node_type='cache.t2.micro') + + +if __name__ == '__main__': + main() diff --git a/aws_ses.py b/aws_ses.py new file mode 100644 index 0000000..e8c8cd8 --- /dev/null +++ b/aws_ses.py @@ -0,0 +1,202 @@ +import concurrent.futures +import functools +import logging +import pprint +import time +import typing + +import boto3 +import botocore.client + +DnsRecord = typing.TypedDict( + "DnsRecord", + { + "purpose": str, + "type": str, + "name": str, + "value": str, + "priority": typing.Optional[str], + }, +) +VerificationStatus = typing.Literal["Pending", "Success", "Failed", "TemporaryFailure", "NotStarted"] + +logger = logging.getLogger(__name__) + + +def retry(tries: int = 5, delay: int = 1.5): + def wrapper(fn): + @functools.wraps(fn) + def wrapped(*args, **kwargs): + nonlocal tries + while tries: + try: + return fn(*args, **kwargs) + except Exception: + logger.exception(f'{fn} returned error, retrying {tries} more times') + time.sleep(delay) + tries -= 1 + + return wrapped + + return wrapper + + +class AwsSesService: + def __init__(self, client: typing.Optional[botocore.client.BaseClient] = None, timeout: int = 5): + self.ses = client or boto3.client("ses") + self.region = self.ses._client_config.region_name + self.timeout = timeout + + # @retry() + def configure_domain_identity( + self, domain: str, mail_subdomain: typing.Optional[str] = None + ) -> typing.List[DnsRecord]: + """ + Creates up mail identities for sending email through a domain. + + DKIM-signed messages help receiving mail servers validate that a message was not forged or altered in transit. + + :param domain: domain name to set up email identity + :return: DNS records that need to be set + """ + + res_id = self.ses.verify_domain_identity(Domain=domain) + id_token = res_id["VerificationToken"] + + res_dkim = self.ses.verify_domain_dkim(Domain=domain) + dns_records = [ + { + "purpose": "identity", + "type": "TXT", + "name": domain, + "value": id_token, + }, + *[ + { + "purpose": "dkim", + "type": "CNAME", + "name": f"{dkim_token}._domainkey.{domain}", + "value": f"{dkim_token}.dkim.amazonses.com", + } + for dkim_token in res_dkim["DkimTokens"] + ], + ] + if mail_subdomain: + more_dns = self.configure_from_domain(domain, mail_subdomain) + dns_records.extend(more_dns) + return dns_records + + @retry() + def check_dkim_verification_status(self, domain: str) -> VerificationStatus: + """ + Returns the DKIM verification status for an email identity. + + :param domain: domain used to set up the mail identity + :return: one of {Pending, Success, Failed, TemporaryFailure, NotStarted} + """ + res = self.ses.get_identity_dkim_attributes(Identities=[domain]) + return res["DkimAttributes"][domain]["DkimVerificationStatus"] + + @retry() + def check_id_verification_status(self, domain: str) -> VerificationStatus: + """ + Returns the id verification status of an email identity. + + :param domain: domain used to set up the mail identity + :return: one of {Pending, Success, Failed, TemporaryFailure, NotStarted} + """ + res = self.ses.get_identity_verification_attributes(Identities=[domain]) + return res["VerificationAttributes"][domain]["VerificationStatus"] + + def configure_from_domain(self, domain: str, mail_subdomain: str) -> typing.List[DnsRecord]: + """ + Allows using `mail_subdomain` as FROM address when sending emails. + Messages sent through Amazon SES will be marked as originating from your domain instead of a subdomain of amazon.com. + + :param domain: mail identity + :param mail_subdomain: a subdomain + :return: DNS records that need to be set + """ + _ = self.ses.set_identity_mail_from_domain( + Identity=domain, + MailFromDomain=mail_subdomain, + BehaviorOnMXFailure="UseDefaultValue", # uses $region.amazonses.com if MX records not present on the mail subdomain + ) + return [ + { + "purpose": "mail_from_domain", + "type": "MX", + "name": mail_subdomain, + "value": f"feedback-smtp.{self.region}.amazonses.com", + "priority": "10", + }, + { + "purpose": "mail_from_domain", + "type": "TXT", + "name": mail_subdomain, + "value": f'"v=spf1 include:amazonses.com ~all"', + }, + ] + + @retry() + def check_custom_from_domain_status(self, domain: str) -> VerificationStatus: + """ + Returns the status of custom email FROM request verification. + + :param domain: domain used to set up email identity + :return: one of {Pending, Success, Failed, TemporaryFailure} + """ + res = self.ses.get_identity_mail_from_domain_attributes(Identities=[domain]) + return res["MailFromDomainAttributes"][domain].get("MailFromDomainStatus", 'NotStarted') + + def bulk_check_domain_verification(self, identity_domain: str, timeout: int = 5) -> dict: + tasks = { + "status_id": functools.partial(self.check_id_verification_status, identity_domain), + "status_dkim": functools.partial(self.check_dkim_verification_status, identity_domain), + "status_custom": functools.partial(self.check_custom_from_domain_status, identity_domain), + } + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool: + futures = [pool.submit(t) for _, t in tasks.items()] + results = dict(zip(tasks.keys(), [f.result(timeout=timeout) for f in futures])) + + return { + **results, + "identity_domain": identity_domain, + } + + @retry() + def bulk_configure_email_domains( + self, + identity_domain: str, + mail_subdomain: typing.Optional[str] = None, + timeout: int = 10, + ) -> dict: + tasks = { + "dns": functools.partial(self.configure_domain_identity, identity_domain, mail_subdomain), + } + + # send all requests in parallel + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool: + futures = [pool.submit(t) for _, t in tasks.items()] + results = dict(zip(tasks.keys(), [f.result(timeout=timeout) for f in futures])) + return { + **results, + "identity_domain": identity_domain, + "mail_subdomain": mail_subdomain, + } + + +if __name__ == "__main__": + ses = AwsSesService() + logging.basicConfig(level=logging.INFO) + # print(ses.bulk_configure_email_domains('abdus100.dev', 'mail.abdus100.dev')) + # print(ses.check_custom_from_domain_status('abdus101.dev')) + + tasks = [ + functools.partial(ses.bulk_check_domain_verification, d) for d in ['abdus.dev', 'abdus1.dev', 'abdus2.dev'] + ] * 3 + + with concurrent.futures.ThreadPoolExecutor(max_workers=20) as pool: + futures = [pool.submit(t) for t in tasks] + results = [f.result(timeout=5) for f in futures] + pprint.pp(results) diff --git a/aws_ses_test.py b/aws_ses_test.py new file mode 100644 index 0000000..b598cce --- /dev/null +++ b/aws_ses_test.py @@ -0,0 +1,133 @@ +from unittest import mock + +import pytest + +from aws_ses import AwsSesService + +# fmt: off +_response_verify_domain_dkim = {'DkimTokens': ['token1', 'token2', 'token3'], 'ResponseMetadata': {'RequestId': '6e4da72e-3063-41da-9434-dae7ef19c78e', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:27:40 GMT', 'content-type': 'text/xml', 'content-length': '469', 'connection': 'keep-alive', 'x-amzn-requestid': '6e4da72e-3063-41da-9434-dae7ef19c78e'}, 'RetryAttempts': 0}} +_response_verify_domain_identity = {'VerificationToken': 'token', 'ResponseMetadata': {'RequestId': 'b733069c-115f-4457-b6cb-14e44f29d17f', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:27:12 GMT', 'content-type': 'text/xml', 'content-length': '370', 'connection': 'keep-alive', 'x-amzn-requestid': 'b733069c-115f-4457-b6cb-14e44f29d17f'}, 'RetryAttempts': 0}} +_response_get_identity_dkim_attributes = {'DkimAttributes': {'example.com': {'DkimEnabled': True, 'DkimVerificationStatus': 'Success', 'DkimTokens': ['bgobmdqs2vbxr6ygwbmg753knfu6tos7', 'm6o6w6twpmwzwe3igpxxo7qsvwp3hlgj', 'zqfz4ekdelp7m2u2phsyad3h5dsfle4v']}}, 'ResponseMetadata': {'RequestId': 'ba466be1-79bc-499b-9766-ca1acd7d1c08', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 16:49:53 GMT', 'content-type': 'text/xml', 'content-length': '833', 'connection': 'keep-alive', 'x-amzn-requestid': 'ba466be1-79bc-499b-9766-ca1acd7d1c08'}, 'RetryAttempts': 0}} +_response_get_identity_dkim_attributes_pending = {'DkimAttributes': {'example.com': {'DkimEnabled': True, 'DkimVerificationStatus': 'Pending', 'DkimTokens': ['bgobmdqs2vbxr6ygwbmg753knfu6tos7', 'm6o6w6twpmwzwe3igpxxo7qsvwp3hlgj', 'zqfz4ekdelp7m2u2phsyad3h5dsfle4v']}}, 'ResponseMetadata': {'RequestId': 'b24732d9-01e1-4266-8b34-550a9ecf634d', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:30:03 GMT', 'content-type': 'text/xml', 'content-length': '833', 'connection': 'keep-alive', 'x-amzn-requestid': 'b24732d9-01e1-4266-8b34-550a9ecf634d'}, 'RetryAttempts': 0}} +_response_get_identity_verification_attributes = {'VerificationAttributes': {'example.com': {'VerificationStatus': 'Success', 'VerificationToken': 'XzMvLQqOPdC0iJ02YTdjqWBJr7FTqhGtAsEcfpUwfoA='}}, 'ResponseMetadata': {'RequestId': '022c3e6e-a6e8-426f-bf4d-149f71ca7913', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 16:56:19 GMT', 'content-type': 'text/xml', 'content-length': '637', 'connection': 'keep-alive', 'x-amzn-requestid': '022c3e6e-a6e8-426f-bf4d-149f71ca7913'}, 'RetryAttempts': 0}} +_response_get_identity_verification_attributes_pending = {'VerificationAttributes': {'example.com': {'VerificationStatus': 'Pending', 'VerificationToken': 'XzMvLQqOPdC0iJ02YTdjqWBJr7FTqhGtAsEcfpUwfoA='}}, 'ResponseMetadata': {'RequestId': '022c3e6e-a6e8-426f-bf4d-149f71ca7913', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 16:56:19 GMT', 'content-type': 'text/xml', 'content-length': '637', 'connection': 'keep-alive', 'x-amzn-requestid': '022c3e6e-a6e8-426f-bf4d-149f71ca7913'}, 'RetryAttempts': 0}} +_response_set_identity_mail_from_domain = {'ResponseMetadata': {'RequestId': 'c04acd33-40be-46cd-9960-c9e51397dc48', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:03:50 GMT', 'content-type': 'text/xml', 'content-length': '266', 'connection': 'keep-alive', 'x-amzn-requestid': 'c04acd33-40be-46cd-9960-c9e51397dc48'}, 'RetryAttempts': 0}} +_response_get_identity_mail_from_domain_attributes = {'MailFromDomainAttributes': {'example.com': {'MailFromDomain': 'email.abdus.dev', 'MailFromDomainStatus': 'Success', 'BehaviorOnMXFailure': 'UseDefaultValue'}}, 'ResponseMetadata': {'RequestId': '63049966-c062-47f4-8238-145bd533d880', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:37:06 GMT', 'content-type': 'text/xml', 'content-length': '687', 'connection': 'keep-alive', 'x-amzn-requestid': '63049966-c062-47f4-8238-145bd533d880'}, 'RetryAttempts': 0}} +_response_get_identity_mail_from_domain_attributes_pending = {'MailFromDomainAttributes': {'example.com': {'MailFromDomain': 'email.abdus.dev', 'MailFromDomainStatus': 'Pending', 'BehaviorOnMXFailure': 'UseDefaultValue'}}, 'ResponseMetadata': {'RequestId': '63049966-c062-47f4-8238-145bd533d880', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:37:06 GMT', 'content-type': 'text/xml', 'content-length': '687', 'connection': 'keep-alive', 'x-amzn-requestid': '63049966-c062-47f4-8238-145bd533d880'}, 'RetryAttempts': 0}} +# fmt: on + + +@pytest.fixture() +def mock_ses() -> mock.MagicMock: + with mock.patch("boto3.client") as m: + m_ses = m.return_value + yield m_ses + + +def test_init_domain_verification(mock_ses): + m_verify_id = mock_ses.verify_domain_identity + m_verify_dkim = mock_ses.verify_domain_dkim + m_verify_id.return_value = _response_verify_domain_identity + m_verify_dkim.return_value = _response_verify_domain_dkim + domain = "example.com" + + dns = AwsSesService(mock_ses).configure_domain_identity(domain) + + m_verify_id.assert_called_once_with(Domain=domain) + m_verify_dkim.assert_called_once_with(Domain=domain) + assert dns == [ + { + "type": "TXT", + "name": domain, + "value": "token", + }, + { + "type": "CNAME", + "name": f"token1._domainkey.{domain}", + "value": f"token1.dkim.amazonses.com", + }, + { + "type": "CNAME", + "name": f"token2._domainkey.{domain}", + "value": f"token2.dkim.amazonses.com", + }, + { + "type": "CNAME", + "name": f"token3._domainkey.{domain}", + "value": f"token3.dkim.amazonses.com", + }, + ] + + +def test_get_dkim_status(mock_ses): + m = mock_ses.get_identity_dkim_attributes + m.return_value = _response_get_identity_dkim_attributes_pending + domain = "example.com" + + status = AwsSesService(mock_ses).check_dkim_verification_status(domain) + + assert status == "Pending" + m.assert_called_once_with(Identities=[domain]) + + # === + + m.reset_mock() + m.return_value = _response_get_identity_dkim_attributes + + status = AwsSesService(mock_ses).check_dkim_verification_status(domain) + assert status == "Success" + + +def test_get_id_status(mock_ses): + m = mock_ses.get_identity_verification_attributes + m.return_value = _response_get_identity_verification_attributes + + domain = "example.com" + status = AwsSesService(mock_ses).check_id_verification_status(domain) + + assert status == "Success" + m.assert_called_once_with(Identities=[domain]) + + +def test_set_custom_from_domain(mock_ses): + mock_ses._client_config.region_name = "fake-region" + m = mock_ses.set_identity_mail_from_domain + m.return_value = _response_set_identity_mail_from_domain + + dns = AwsSesService(mock_ses).configure_from_domain("example.com", "mail.example.com") + + assert dns == [ + { + "type": "MX", + "name": "mail.example.com", + "value": f"feedback-smtp.fake-region.amazonses.com", + "priority": "10", + }, + {"type": "TXT", "name": "mail.example.com", "value": f'"v=spf1 include:amazonses.com ~all"'}, + ] + m.assert_called_once_with( + Identity="example.com", + MailFromDomain="mail.example.com", + BehaviorOnMXFailure="UseDefaultValue", + ) + + +def test_get_mail_from_status(mock_ses): + m = mock_ses.get_identity_mail_from_domain_attributes + m.return_value = _response_get_identity_mail_from_domain_attributes_pending + + domain = "example.com" + status = AwsSesService(mock_ses).check_custom_from_domain_status(domain) + + assert status == "Pending" + m.assert_called_once_with(Identities=[domain]) + + # === + + m.reset_mock() + m.return_value = _response_get_identity_mail_from_domain_attributes + + status = AwsSesService(mock_ses).check_custom_from_domain_status(domain) + + assert status == "Success" + m.assert_called_once_with(Identities=[domain]) diff --git a/berlin.py b/berlin.py new file mode 100644 index 0000000..3cb5ecb --- /dev/null +++ b/berlin.py @@ -0,0 +1,48 @@ +import contextlib + +from playwright.sync_api import sync_playwright, Page +from time import sleep + + +@contextlib.contextmanager +def launch_browser() -> Page: + with sync_playwright() as playwright: + with playwright.chromium.launch(headless=False) as browser: + with browser.new_context() as context: + with context.new_page() as page: + yield page + + +def book_slot(): + with launch_browser() as page: + page.goto("https://service.berlin.de/dienstleistung/120686/") + page.get_by_role("complementary").get_by_role("link", name="Termin berlinweit suchen").click() + + with page.expect_navigation(url='https://service.berlin.de/terminvereinbarung/termin/time/*', timeout=0): + pass + + with page.expect_navigation(url='https://service.berlin.de/terminvereinbarung/termin/register/*', timeout=0): + pass + + page.get_by_label("Ihr Vor- und Nachname *").fill("Paulina Gutkowska") + page.get_by_label("Ihre E-Mail Adresse *").fill("gutpaula11@gmail.com") + if page.is_visible("Ihre Telefonnummer oder Mobilfunknummer *"): + page.get_by_label("Ihre Telefonnummer oder Mobilfunknummer *").fill("+48501099723") + page.get_by_role("combobox").select_option("0") + page.get_by_label( + "Ich erkläre mich mit den Nutzungsbedingungen einverstanden und akzeptiere diese. (Pflichtangabe. Termineintrag nur bei Einverständnis möglich) *" + ).check() + + with page.expect_navigation(): + page.get_by_role("button", name="Termin eintragen").click() + + sleep(10) + input() + + +def main(): + book_slot() + + +if __name__ == '__main__': + main() diff --git a/berlin_anmeldung.py b/berlin_anmeldung.py new file mode 100644 index 0000000..5fa4307 --- /dev/null +++ b/berlin_anmeldung.py @@ -0,0 +1,75 @@ +import contextlib +import datetime +import locale + +import httpx +from bs4 import BeautifulSoup + +import berlin + +http = httpx.Client( + headers={ + 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36' + }, + follow_redirects=True, +) + + +@contextlib.contextmanager +def override_locale(category: int, val: str) -> None: + prev = locale.getlocale(category) + locale.setlocale(category, val) + yield + locale.setlocale(category, prev) + + +def get_available_slots(): + res = http.get('https://service.berlin.de/dienstleistung/120686/') + res.raise_for_status() + + soup = BeautifulSoup(res.text, 'html.parser') + link = soup.select_one('[role="complementary"] .zmstermin-multi a') + calendar_url = link.attrs['href'] + + try: + # redirect fails, we just need the cookies + _ = http.get(calendar_url) + except: + pass + res = http.get('https://service.berlin.de/terminvereinbarung/termin/day/') + res.raise_for_status() + + return extract_days(res.text) + + +def extract_days(html: str) -> list[datetime.date]: + soup = BeautifulSoup(html, 'html.parser') + + found = [] + for it in soup.select('.calendar-month-table .buchbar'): + month_year = it.find_parent(attrs={'class': 'calendar-month-table'}).select_one('thead .month').text.strip() + day = it.text.strip() + found.append(f'{day} {month_year}') + + with override_locale(locale.LC_TIME, 'de_DE'): + days = [datetime.datetime.strptime(it, '%d %B %Y').date() for it in found] + + return days + + +def main(): + days = get_available_slots() + good_days = [it for it in days if it < datetime.date(2023, 1, 27)] + + if not good_days: + print('no slots') + return + + print('found slots') + berlin.book_slot() + for it in days: + print(it) + + +if __name__ == '__main__': + main() diff --git a/berlin_immigration.py b/berlin_immigration.py new file mode 100644 index 0000000..253a0ba --- /dev/null +++ b/berlin_immigration.py @@ -0,0 +1,139 @@ +import argparse +import asyncio +import contextlib +import datetime +import logging +import random +import re +import subprocess +import time +import urllib.parse +from pathlib import Path + +from playwright.async_api import async_playwright, Browser, BrowserContext, Page + + +@contextlib.asynccontextmanager +async def launch_browser(headless: bool = False) -> Page: + async with async_playwright() as playwright: + browser = await playwright.chromium.launch(headless=headless) + browser: Browser + async with browser: + ctx = await browser.new_context() + ctx: BrowserContext + async with ctx: + page = await ctx.new_page() + async with page: + yield page + + +class Error(Exception): + pass + + +def alert(text: str): + subprocess.run(['open', f'alfred://runtrigger/dev.abdus.integrations/remind/?argument={urllib.parse.quote(text)}']) + + +dump_dir = Path('~/Desktop').expanduser() / 'immigration' +dump_dir.mkdir(parents=True, exist_ok=True) + +async def check_slots(page: Page, retry_count: int = 1) -> list[datetime.datetime]: + await page.add_init_script('''Object.defineProperty(navigator, 'webdriver', { get: () => false })''') + window_id = random.randint(1000, 9999) + req_id = random.randint(0, 999) + await page.goto( + f'https://otv.verwalt-berlin.de/ams/TerminBuchen/wizardng?dswid={window_id}&dsrid={req_id}', + wait_until='networkidle', + ) + await page.check('[name="gelesen"]') + async with page.expect_navigation(url=re.compile('st=2')): + await page.click('[name="applicationForm:managedForm:proceed"]') + + await page.get_by_role("combobox", name="Staatsangehörigkeit *").select_option(label="Türkei") + await asyncio.sleep(0.3) + + await page.get_by_role( + "combobox", + name="Anzahl der Personen, die einen Aufenthaltstitel beantragen (auch ausländische Ehepartner und Kinder) *", + ).select_option(label="eine Person") + await asyncio.sleep(0.3) + + await page.get_by_role( + "combobox", name="Leben Sie in Berlin zusammen mit einem Familienangehörigen (z.B. Ehepartner, Kind) *" + ).select_option(label="nein") + await asyncio.sleep(0.3) + + await page.get_by_text("Aufenthaltstitel - beantragen").click() + await page.locator("label").filter(has_text="Erwerbstätigkeit").click() + await page.get_by_text("Blaue Karte EU (§ 18b Abs. 2)").click() + + stop_at = datetime.datetime.now() + datetime.timedelta(minutes=28) + + async def go_forward(): + async with page.expect_navigation(url=re.compile(r'st='), wait_until='networkidle', timeout=60_000): + await page.get_by_role("button", name="Weiter").click() + if 'st=2' in page.url: + error_message = await page.inner_text('.errorMessage') + if 'keine Termine frei' in error_message: + return False + raise Error(error_message) + return 'st=3' in page.url + + while 'st=2' in page.url: + retry_count -= 1 + + async with page.expect_navigation(url=re.compile(r'st='), wait_until='networkidle', timeout=60_000): + await page.get_by_role("button", name="Weiter").click() + + if datetime.datetime.now() > stop_at: + raise TimeoutError('could not find a slot') + + if not retry_count: + return [] + + await asyncio.sleep(random.randint(4, 10)) + continue + + if 'st=3' in page.url: + alert('empty slot found') + now = int(datetime.datetime.now().timestamp()) + html_path = dump_dir / f'immigration_{now}.html' + html_path.write_text(await page.content()) + await page.screenshot(path=dump_dir/f'immigration_{now}.png') + + await page.pause() + + input() + + +def parse_args(): + arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + arger.add_argument( + '--headless', '--silent', '--quiet', action='store_true', dest='headless', help='Run in headless mode' + ) + arger.add_argument( + '--retry-count', '--retry', type=int, default=1, help='Run in headless mode' + ) + + args, _ = arger.parse_known_args() + return args + + +async def main(): + args = parse_args() + async with launch_browser(headless=args.headless) as page: + try: + slots = await check_slots(page, retry_count=args.retry_count) + if not slots: + print('no slots yet') + return + except Exception as e: + logging.exception('got an error') + await page.pause() + + +if __name__ == '__main__': + asyncio.run(main()) + logging.basicConfig(level=logging.INFO) + main() diff --git a/deluge_import.py b/deluge_import.py new file mode 100644 index 0000000..c9aaef3 --- /dev/null +++ b/deluge_import.py @@ -0,0 +1,121 @@ +import logging +import random +import subprocess +import httpx +from pathlib import Path +import torf +import time + +class Deluge: + def __init__(self, session: httpx.Client) -> None: + self.session = session + + @classmethod + def new(cls) -> 'Deluge': + logging.debug("Connecting to Deluge") + session = httpx.Client(base_url="https://t.zzzp.win/", timeout=10, auth=httpx.BasicAuth('abdus', 'xAsametk50')) + res = session.post("/json", json={"method": "auth.login", "params": ["xAsametk50"], "id": cls.make_id()}) + res.raise_for_status() + + # find first available host + res = session.post("/json", json={"method": "web.get_hosts", "params": [], "id": cls.make_id()}) + res.raise_for_status() + host_id = res.json()["result"][0][0] + + # connect to a host + res = session.post("/json", json={"method": "web.connect", "params": [host_id], "id": cls.make_id()}) + res.raise_for_status() + + return cls(session) + + @classmethod + def make_id(cls) -> int: + return time.time_ns() + + def add(self, torrent_path: Path, download_path: Path, paused: bool = True, skip_check: bool = True): + # upload torrent + logging.debug("Uploading torrent file") + with torrent_path.open("rb") as f: + res = self.session.post("/upload", files={"file": f}) + res.raise_for_status() + remote_path: str = res.json()["files"][0] + + # add torrent + logging.debug("Adding torrent file") + res = self.session.post( + "/json", + timeout=1, + json={ + "method": "web.add_torrents", + "params": [ + [ + { + "path": remote_path, + "options": { + "file_priorities": [1], + "add_paused": paused, + "sequential_download": False, + "pre_allocate_storage": False, + "download_location": str(download_path), + # "move_completed": False, + # "move_completed_path": "/root/Downloads", + # "prioritize_first_last_pieces": True, + "seed_mode": skip_check, + # "super_seeding": False, + }, + } + ] + ], + "id": self.make_id(), + }, + ) + + res.raise_for_status() + + +def find_torrents() -> list[Path]: + files = list(Path('/tmp/deluge/config/state').glob('*.torrent')) + random.shuffle(files) + return files + +def find_existing_torrents() -> dict[str, Path]: + lines = subprocess.check_output(['ssh', 'klein', '--', 'find', '/mnt/box/files/_torrents', '-maxdepth', '1']).decode() + existing = {} + lines = lines.strip().splitlines(keepends=False) + for line in lines: + if line.startswith('.'): + continue + p = Path(line) + existing[p.name] = p + return existing + + +def main(): + existing = find_existing_torrents() + d = Deluge.new() + done = set() + for torrent_path in find_torrents(): + if torrent_path in done: + continue + if len(done) == len(existing): + return + with torrent_path.open('rb') as f: + t = torf.Torrent.read_stream(f) + if '.PRT' in t.name: + continue + if et := existing.get(t.name): + logging.debug(f'adding {t.name}') + download_path = Path("/dl") + try: + d.add(torrent_path, download_path=download_path) + torrent_path.unlink() + done.add(torrent_path) + logging.info(f'added: {t.name}') + except Exception as e: + pass + + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format=f'%(asctime)s {logging.BASIC_FORMAT}') + main() \ No newline at end of file diff --git a/duration_parser.py b/duration_parser.py new file mode 100644 index 0000000..5a8a08a --- /dev/null +++ b/duration_parser.py @@ -0,0 +1,40 @@ +import sys +import re +import math +import datetime + + +def parse_duration(text: str) -> datetime.timedelta: + re_duration = re.compile(r'(\d+(?:\.\d+)?(?:sec|min|hour|hr|h|m|s))') + re_digits = re.compile(r'([\d.]+)') + time_scales = { + 's': 1, + 'sec': 1, + 'm': 60, + 'min': 60, + 'h': 3600, + 'hr': 3600, + } + + parts = re_duration.findall(text) + if parts: + total_secs = 0 + for it in parts: + if m := re_digits.search(it): + _, end = m.span() + scale = it[end:] + total_secs += math.ceil(float(m.group(1)) * time_scales[scale]) + return datetime.timedelta(seconds=total_secs) + + parts = [float(it) for it in text.split(':')] + if len(parts) == 1: + return datetime.timedelta(seconds=parts[0]) + if len(parts) == 2: + return datetime.timedelta(minutes=parts[0], seconds=parts[1]) + if len(parts) == 3: + return datetime.timedelta(hours=parts[0], minutes=parts[1], seconds=parts[2]) + + +if __name__ == '__main__': + parsed = parse_duration(sys.argv[1]) + seconds = int(parsed.total_seconds()) diff --git a/emp.py b/emp.py new file mode 100755 index 0000000..6b7a4ba --- /dev/null +++ b/emp.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3.9 +from dataclasses import dataclass +import json +from pathlib import Path +import subprocess +import typing +from playwright.sync_api import Playwright, sync_playwright, Browser +from typer import Option, Typer + + +class Storage(typing.Protocol): + def get(self, key: str) -> typing.Optional[typing.Any]: + ... + + def set(self, key: str, value) -> None: + ... + + +@dataclass +class FileStorage: + path: Path + + def get(self, key: str): + try: + return json.loads(self.path.read_text()).get(key) + except FileNotFoundError: + return None + + def set(self, key: str, value) -> None: + try: + data = json.loads(self.path.read_text()) + except FileNotFoundError: + data = {} + data[key] = value + self.path.write_text(json.dumps(data)) + + +class Emp: + def __init__(self, browser: Browser, storage: Storage) -> None: + self.browser = browser + self.storage = storage + + def ensure_session(self, username: str, password: str) -> None: + if self.storage.get("cookies") is None: + self.login(username, password) + + def login(self, username: str, password: str) -> typing.Dict[str, str]: + """ + logins and returns session cookies + """ + context = self.browser.new_context() + + # Open new page + page = context.new_page() + + # Go to https://www.empornium.is/ + page.goto("https://www.empornium.is/login") + + page.locator('[placeholder="Username"]').fill(username) + page.locator('[placeholder="Password"]').fill(password) + # Click text=Stay logged in + page.locator("text=Stay logged in").click() + + # Click input:has-text("login") + page.locator('input:has-text("login")').click() + + cookies = {c["name"]: c["value"] for c in context.cookies()} + self.storage.set("cookies", {"sid": cookies["sid"]}) + + return { + "sid": cookies["sid"], + } + + def prepare_post( + self, + torrent_path: Path, + title: str, + tags: str, + description: str, + cover_image_url: str, + category: typing.Optional[str] = None, + ) -> None: + cookies = self.storage.get("cookies") + assert cookies, "You must login first" + + context = self.browser.new_context() + context.add_cookies( + [{"name": k, "value": v, "domain": "www.empornium.is", "path": "/"} for k, v in cookies.items()] + ) + + page = context.new_page() + + page.goto("https://www.empornium.is/upload.php") + + if torrent_path.is_file(): + page.locator('input[name="file_input"]').set_input_files(torrent_path.expanduser().resolve()) + page.locator('text="check for dupes"').click() + + # Select category + if category: + page.locator('select[name="category"]').select_option(label=category) + + page.locator('input[name="title"]').fill(title) + page.locator('textarea[name="taglist"]').fill(tags) + page.locator('input[name="image"]').fill(cover_image_url) + page.locator('textarea[name="desc"]').fill(description) + + # Click text=Preview + page.locator("text=Preview").click() + + +def submit_post(): + storage = FileStorage(Path("emp.json")) + + with sync_playwright() as playwright: + browser = playwright.chromium.launch(headless=False) + emp = Emp(browser=browser, storage=storage) + emp.ensure_session(username="zzzp", password="9arjs9za2o") + + emp.prepare_post( + torrent_path=Path("~/Downloads/v.torrent"), + title="A Title", + tags="tag.1 tag.2", + description="Some description", + cover_image_url="https://images.com/image.jpg", + category="Anal", + ) + input("Press Enter to continue...") + browser.close() + + +cli = Typer(name="emp") + + +torrent_cli = Typer(name="torrent") +cli.add_typer(torrent_cli) + + +@torrent_cli.callback("torrent") +def make_torrent(paths: typing.List[Path], announce_url: str = Option(..., envvar="ANNOUNCE_URL")): + if len(paths) == 1 and paths[0].is_dir(): + dir_path = paths[0] + + dir_path = dir_path.expanduser().resolve() + proc = subprocess.run( + [ + "torrentify", + f"-announce={announce_url}", + f"-comment=created by zzzp", + f"-created-by=zzzp", + f"-name={dir_path.name}", + str(dir_path), + ], + check=True, + capture_output=True, + ) + torrent_path = dir_path / f"{dir_path.name}.torrent" + torrent_path.write_bytes(proc.stdout) + + +@torrent_cli.command() +def clone(): + pass + + +if __name__ == "__main__": + cli() diff --git a/enc.py b/enc.py new file mode 100755 index 0000000..bfcd495 --- /dev/null +++ b/enc.py @@ -0,0 +1,479 @@ +#!/usr/bin/env python3.9 +import argparse +import dataclasses +import datetime +import json +import logging +import re +import subprocess +import sys +import threading +import time +import typing +from pathlib import Path + +import math +import rich.progress + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class FfprobeResult: + duration_sec: float + duration_human: str + codec: str + fps: float + size_bytes: int + width: int + height: int + bitrate: int + container: str + ar: float + sample_ar: float + + +def ffprobe(video: Path) -> FfprobeResult: + logger.debug("executing ffprobe") + proc = subprocess.run( + args=[ + "ffprobe", + "-v", + "error", + "-show_streams", + "-show_format", + "-print_format", + "json", + str(video), + ], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + proc.check_returncode() + except subprocess.CalledProcessError: + logger.error("failed to run ffprobe", exc_info=True) + raise + output: dict = json.loads(proc.stdout) + video_stream: dict = [s for s in output.get("streams", []) if s.get("codec_type") == "video"][0] + video_format: dict = output.get("format", {}) + width, height = video_stream.get("width"), video_stream.get("height") + codec = video_stream.get("codec_name") + duration = float(video_format.get("duration")) + fps_long = eval(video_stream.get("r_frame_rate", "")) + fps = float(f"{fps_long:.3f}") + size = int(video_format.get("size", 0)) + duration_time = datetime.timedelta(seconds=math.ceil(duration)) + bitrate = int(video_stream.get("bit_rate", video_format.get("bit_rate", 0))) + + try: + sample_w, sample_h = list(map(int, video_stream["sample_aspect_ratio"].split(":"))) + sample_ar = (sample_w / sample_h) or 1 + except: + sample_ar = 1 + + logger.debug(f"dimensions={width}x{height} duration={duration_time}, fps={fps}, size={size // 1_048_576}MB") + # total_frames = video_stream.get("nb_frames") + + return FfprobeResult( + duration_sec=duration, + duration_human=str(duration_time), + codec=codec, + fps=fps, + size_bytes=size, + width=width, + bitrate=bitrate, + height=height, + container=video.suffix.lstrip(".").lower(), + ar=width / height, + sample_ar=sample_ar, + # total_frames=total_frames, + ) + + +def make_thumbnail_tile( + video: Path, + image_path: Path = None, + columns: int = 3, + interval_seconds: int = 60, + tile_width: int = 540, + skip_first_sec: int = 10, + skip_if_exists: bool = False, +) -> Path: + if not image_path: + image_path = video.parent / f"{video.stem}.thumbnail.jpg" + if image_path.is_file() and skip_if_exists: + logger.debug("thumbnail already exists") + return image_path + + info = ffprobe(video) + + min_frames = columns * 3 + sec_per_frame = min(interval_seconds, math.ceil(info.duration_sec / min_frames)) + rows = math.ceil(info.duration_sec // sec_per_frame / columns) + tile = f"{columns}x{rows}" + + scaled_width, scaled_height = tile_width, math.ceil(tile_width / info.ar / info.sample_ar) + scale = f"{scaled_width}:{scaled_height}" + + proc = subprocess.run( + # fmt: off + args=[ + "ffmpeg", + "-v", "error", + "-skip_frame", "nokey", + "-ss", f"{skip_first_sec}", + "-i", str(video.absolute()), + "-vf", f"fps=1/{sec_per_frame},scale={scale},tile={tile}", + "-frames", "1", + "-y", str(image_path.absolute()), + ], + # fmt: on + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=2000, + ) + proc.check_returncode() + return image_path + + +@dataclasses.dataclass +class EncodeProgress: + percent: float + fps_avg: int + eta: str + current_size: int + + @property + def encoded_mb(self) -> int: + return math.ceil(self.current_size / 1_048_576) + + @property + def estimated_mb(self) -> int: + return math.ceil(self.encoded_mb / (self.percent / 100)) + + +@dataclasses.dataclass +class UploadProgress: + uploaded_mb: int + speed: str + percent: float + + +def noop(*args): + pass + + +def upload_file( + file_path: Path, + destination: str, + watch: bool = False, + on_progress: typing.Callable[[UploadProgress], None] = noop, + on_resync: typing.Callable[[], None] = noop, +): + size = file_path.stat().st_size + while True: + proc = subprocess.Popen( + ["rsync", "--bwlimit", "1100", "-rvPa", str(file_path), destination], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + def watch_progress(p: subprocess.Popen): + # 5233864 0% 1.93MB/s 0:21:14 + # 12,450,960 30% 2.49MB/s 0:00:11 + pattern = re.compile(r"\b(?P[\d,]+)\s+(?P[\d]+)%\s+(?P\d+\.\d+[kmg]B/s)\b", re.I) + for line in p.stdout: + if m := pattern.search(line): + parsed = m.groupdict() + progress = UploadProgress( + uploaded_mb=int(parsed["uploaded"].replace(",", "")) // 1_048_576, + speed=parsed["speed"], + percent=float(parsed["percent"]), + ) + try: + on_progress(progress) + except: + pass + + t = threading.Thread(target=watch_progress, args=(proc,)) + t.start() + + if proc.wait() != 0: + code = proc.poll() + err = proc.stderr.read() + logger.warning(f"rsync failed with {code=}. error: {err}") + return + + if not watch: + return + + time.sleep(5) + current_size = file_path.stat().st_size + diff = current_size - size + size = current_size + + if not diff: + return + + logger.info(f"filesize changed {diff} bytes, re-syncing...") + on_resync() + + +def encode_video_handbrake( + video_path: Path, + save_path: Path, + quality: int = 30, + from_time: typing.Optional[int] = None, + duration: typing.Optional[int] = None, + on_progress: typing.Callable[[EncodeProgress], None] = noop, + extra_args: typing.List[str] = None, + denoise: bool = False, + is_10bit: bool = True, +): + # fmt: off + args = [ + 'HandbrakeCLI', + '--format', 'av_mp4', + '--input', str(video_path), + *(['--start-at', f'duration:{from_time}'] if from_time else []), + *(['--stop-at', f'duration:{duration}'] if duration else []), + '--output', str(save_path), + '--optimize', + '--encoder', *['vt_h265_10bit' if is_10bit else 'vt_h265'], + '--quality', str(quality), + '--vfr', + '--aencoder', 'ac3', + '--ab', '160', + '--non-anamorphic', + *(['--hqdn3d', 'light'] if denoise else []), + *(extra_args or []), + # '--json', + '--verbose', '0' + ] + # fmt: on + # subprocess.run(args, check=True) + # return + logger.debug("executing handbrake with args: %s", args) + p = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + + last_progress: typing.Optional[EncodeProgress] = None + + def parse_progress(p: subprocess.Popen): + # Encoding: task 1 of 1, 11.96 % (210.12 fps, avg 206.32 fps, ETA 00h06m14s) + re_progress = re.compile(r"(?P[\d.]+) % \(.+ avg (?P[\d.]+) fps, ETA (?P[^)]+)") + for line in p.stdout: + if m := re_progress.search(line.strip()): + parsed = m.groupdict() + percent = round(float(parsed["percent"]), 1) + fps_avg = math.floor(float(parsed["fps_avg"])) + eta = parsed["eta"] + + nonlocal last_progress + last_progress = EncodeProgress(percent, fps_avg, eta, current_size=save_path.stat().st_size) + + try: + on_progress(last_progress) + except: + pass + + t = threading.Thread(target=parse_progress, args=(p,)) + t.start() + + try: + if p.wait() != 0: + logger.error(f"handbrake failed: {p.stderr.read()}") + except KeyboardInterrupt: + p.kill() + if last_progress: + logger.info(f"cancelled at {last_progress.percent}%") + logger.info(f"encoded file is {last_progress.encoded_mb}MB, estimated: {last_progress.estimated_mb}MB") + raise + + +def encode_video_ffmpeg( + video_path: Path, + save_path: Path, + quality: int = 30, + from_time: typing.Optional[int] = None, + duration: typing.Optional[int] = None, + on_progress: typing.Callable[[EncodeProgress], None] = noop, + extra_args: typing.List[str] = None, + is_10bit: bool = True, + **kwargs, +): + # fmt: off + args = [ + 'ffmpeg', + '-y', + # '-v', 'quiet', + '-progress', 'pipe:1', + '-stats_period', '3', + *(['-ss', str(from_time)] if from_time else []), + *(['-t', str(duration)] if duration else []), + '-i', str(video_path), + '-c:v', 'hevc_videotoolbox', + '-q:v', f'{quality}', + '-profile:v', *['main10' if is_10bit else 'main'], + '-map_metadata', '0', + '-metadata', f'title={video_path.stem}', + *(extra_args or []), + str(save_path), + ] + # fmt: on + + logger.info(f"calling ffmpeg with {args=}") + + proc = subprocess.run(args, check=True) + + +def parse_args(): + arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + arger.add_argument("video_path", type=Path, help="path to video file") + arger.add_argument("-e", "--encoder", choices=["handbrake", "ffmpeg"], default="handbrake", help="encoder engine") + arger.add_argument("-q", "--quality", type=float, default=25, help="x265 quality factor") + arger.add_argument("--output-dir", "-o", dest="output_dir", type=Path, help="Dir to save encoded files") + arger.add_argument("--rsync", dest="upload_target", help="rsync encoded file to a host") + arger.add_argument("--validate", action="store_true", default=False, help="perform validations before starting") + arger.add_argument("--denoise", action="store_true", default=False, help="Enable denoise filter (Handbrake only)") + arger.add_argument("--10bit", action="store_true", dest="is_10bit", help="Encode using 10-bit profile") + arger.add_argument("--8bit", action="store_false", dest="is_10bit", help="Encode using 8-bit profile") + arger.set_defaults(is_10bit=True) + + def parse_time(val: str) -> int: + parts = val.split(":") + if len(parts) == 1: + return int(val) + elif len(parts) == 2: + return int(parts[0]) * 60 + int(parts[1]) + return int(parts[0]) * 60 + int(parts[1]) * 60 + int(parts[2]) + + arger.add_argument( + "--from", + dest="from_time", + default=0, + type=parse_time, + help="Start encoding from this time. Example 05:00 or 300", + ) + arger.add_argument("--duration", type=parse_time, help="Stop encoding at this time. Example 07:00 or 420") + + if len(sys.argv[1:]) < 1: + arger.print_help() + exit(1) + + args, extra_args = arger.parse_known_args() + if extra_args and extra_args[0] == "--": + extra_args = extra_args[1:] + + return args, extra_args + + +def generate_filename(video_path: Path) -> str: + probe = ffprobe(video_path) + if 1900 <= probe.width <= 2000: + hd = "1080p" + elif 1200 <= probe.width <= 1400: + hd = "720p" + elif 3000 <= probe.width: + hd = "4K" + else: + hd = None + new_stem = re.sub(r"(\[\d+[pk]])", "", video_path.stem) + new_stem = f"{new_stem.strip()} [{hd}, x265]" + return new_stem + + +def main(): + args, extra_args = parse_args() + + video_path: Path = args.video_path + video_path = video_path.expanduser().resolve() + if not video_path.is_file(): + logger.error("No such file") + exit(1) + + if args.validate and len(video_path.name) >= 128: + logger.error("Filename is too long") + exit(1) + + save_path = video_path.with_stem(generate_filename(video_path)).with_suffix(".mp4") + if video_path == save_path: + save_path = save_path.with_suffix(".reencoded" + video_path.suffix) + + output_dir: Path = args.output_dir + if not output_dir: + output_dir = save_path.parent / "_reenc" + + output_dir = output_dir.expanduser().resolve() + output_dir.mkdir(exist_ok=True, parents=True) + save_path = output_dir / save_path.name + + bar = rich.progress.Progress(refresh_per_second=2) + task_encode = bar.add_task("encoding", visible=False) + up_task = bar.add_task("uploading", visible=False, start=False) + + total_encoded_mb = 0 + + def sync_later(): + logger.info("syncing in 10s") + time.sleep(10) + bar.update(up_task, visible=True) + bar.start_task(up_task) + upload_file( + save_path, + args.upload_target, + watch=True, + on_progress=lambda p: bar.update( + up_task, + description=f"upload: {p.speed}, {p.uploaded_mb:3}MB/{total_encoded_mb:3}MB", + completed=p.uploaded_mb / total_encoded_mb * 100, + ), + on_resync=lambda: bar.reset(up_task, description="upload: re-syncing"), + ) + + t = threading.Thread(target=sync_later) + if args.upload_target: + t.start() + + encoder = encode_video_ffmpeg if args.encoder == "ffmpeg" else encode_video_handbrake + + with bar: + + def _on_encode_progress(p: EncodeProgress): + nonlocal total_encoded_mb + total_encoded_mb = p.encoded_mb + bar.update( + task_encode, + completed=p.percent, + description=f"encode: {p.fps_avg:3}fps, {p.encoded_mb:3}MB/{p.estimated_mb:3}MB", + ) + + try: + bar.update(task_encode, visible=True) + encoder( + video_path, + quality=args.quality, + save_path=save_path, + extra_args=extra_args, + on_progress=_on_encode_progress, + from_time=args.from_time, + duration=args.duration, + denoise=args.denoise, + is_10bit=args.is_10bit, + ) + except KeyboardInterrupt: + logger.info("cancelled") + exit(1) + + logger.info(f"finished encoding {video_path}") + if args.upload_target: + t.join() + image_path = make_thumbnail_tile(save_path, image_path=save_path.with_suffix(".jpg"), skip_if_exists=True) + upload_file(image_path, args.upload_target) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + main() diff --git a/ffmpeg.py b/ffmpeg.py new file mode 100755 index 0000000..b8b765d --- /dev/null +++ b/ffmpeg.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3.9 + +import dataclasses +import json +import logging +import subprocess +import typing +from datetime import timedelta +from math import ceil +from pathlib import Path + + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class FfprobeResult: + duration_sec: float + duration_human: str + codec: str + fps: float + size_bytes: int + width: int + height: int + bitrate: int + container: str + ar: float + sample_ar: float + tags: typing.Dict[str, str] + + +def ffprobe(video: Path) -> FfprobeResult: + logger.debug("executing ffprobe") + proc = subprocess.run( + args=[ + "ffprobe", + "-v", + "error", + "-show_streams", + "-show_format", + "-print_format", + "json", + str(video), + ], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + proc.check_returncode() + except subprocess.CalledProcessError: + logger.error("failed to run ffprobe", exc_info=True) + raise + output: dict = json.loads(proc.stdout) + video_stream: dict = [s for s in output.get("streams", []) if s.get("codec_type") == "video"][0] + video_format: dict = output.get("format", {}) + width, height = video_stream.get("width"), video_stream.get("height") + profile = video_stream.get("profile") + codec = video_stream.get("codec_name") + duration = float(video_format.get("duration")) + fps_long = eval(video_stream.get("r_frame_rate", "")) + fps = float(f"{fps_long:.3f}") + size = int(video_format.get("size", 0)) + duration_time = timedelta(seconds=ceil(duration)) + bitrate = int(video_stream.get("bit_rate", video_format.get("bit_rate", 0))) + tags = video_format.get("tags", {}) + + try: + sample_w, sample_h = list(map(int, video_stream["sample_aspect_ratio"].split(":"))) + sample_ar = (sample_w / sample_h) or 1 + except: + sample_ar = 1 + + logger.debug(f"dimensions={width}x{height} duration={duration_time}") + # total_frames = video_stream.get("nb_frames") + + return FfprobeResult( + duration_sec=duration, + duration_human=str(duration_time), + codec=codec, + fps=fps, + size_bytes=size, + width=width, + bitrate=bitrate, + height=height, + container=video.suffix.lstrip(".").lower(), + ar=width / height, + sample_ar=sample_ar, + tags=tags, + # total_frames=total_frames, + ) + + +def strip_metadata(video_path: Path, save_path: Path) -> Path: + media_info = ffprobe(video_path) + # check if the first video stream has hevc codec + if media_info["streams"][0]["codec_name"] == "hevc": + logger.info(f"Video codec is hevc, adding hvc1 tag") + extra_args = ["-tag:v", "hvc1"] + else: + extra_args = [] + + # fmt: off + args = [ + 'ffmpeg', + '-i', + str(video_path), + '-c:v', 'copy', + '-movflags', '+faststart', + '-map_metadata', '-1', + *extra_args, + '-c:a', 'copy', + '-y', + str(save_path), + ] + # fmt: on + logger.info(f"calling ffmpeg with {args=}") + subprocess.run(args, check=True) + return save_path + + +def make_thumbnail_tile( + video: Path, + image_path: Path = None, + columns: int = 3, + interval_seconds: int = 60, + tile_width: int = 540, + skip_first_sec: int = 10, + skip_if_exists: bool = False, +) -> Path: + if not image_path: + image_path = video.parent / f"{video.stem}.thumbnail.jpg" + if image_path.is_file() and skip_if_exists: + logger.debug("thumbnail already exists") + return image_path + + info = ffprobe(video) + + min_frames = columns * 3 + + sec_per_frame = min(interval_seconds, ceil(info.duration_sec / min_frames)) + + rows = ceil(info.duration_sec // sec_per_frame / columns) + tile = f"{columns}x{rows}" + + scaled_width, scaled_height = tile_width, ceil(tile_width / info.ar / info.sample_ar) + scale = f"{scaled_width}:{scaled_height}" + + # font_path = Path(__file__).parent / 'iosevka.ttf' + # timestamp_filter = rf"drawtext=r=1:timecode='00\:00\:00\:00':fontsize=16:fontcolor=white:x=10:y=10:box=1:boxcolor=black@0.5" + # timestamp_filter = fr"drawtext=text='%{{(pts\\+{skip_first_sec})\:hms}}':fontsize=16:fontcolor=white:x=10:y=10:box=1:boxcolor=black@0.5" + + proc = subprocess.run( + # fmt: off + args=[ + "ffmpeg", + "-v", "error", + "-skip_frame", "nokey", + "-ss", f"{skip_first_sec}", + "-i", str(video.absolute()), + # "-vf", f"fps=1/{sec_per_frame},scale={scale},{timestamp_filter},tile={tile}", + "-vf", f"fps=1/{sec_per_frame},scale={scale},tile={tile}", + "-frames", "1", + "-y", str(image_path.absolute()), + ], + # fmt: on + stdout=subprocess.DEVNULL, + timeout=2000, + ) + proc.check_returncode() + return image_path + + +if __name__ == "__main__": + import sys + + logging.basicConfig(level=logging.DEBUG) + for it in sys.argv[1:]: + video = Path(it) + + if not video.exists(): + continue + + logger.info("creating thumbnail for %s", video) + make_thumbnail_tile(video) diff --git a/ffmpeg_concat.py b/ffmpeg_concat.py new file mode 100755 index 0000000..b132310 --- /dev/null +++ b/ffmpeg_concat.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3.9 +import argparse +import logging +import subprocess +import sys +from pathlib import Path + + +def parse_args(argv: list[str]): + arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + arger.add_argument("files", nargs="+", type=Path, help="Video paths") + arger.add_argument("-o", dest="output_path", type=Path, help="Output path", required=True) + if len(argv) == 0: + arger.print_help() + exit(1) + + return arger.parse_args(argv) + + +def combine_videos(videos: list[Path], save_path: Path) -> Path: + """ + Combine multiple videos into one. + :param videos: List of video paths. + :param save_path: Path to save the combined video. + :return: Path to the combined video. + """ + + # file '/path/to/file1' + # file '/path/to/file2' + # file '/path/to/file3' + # + # $ ffmpeg -f concat -safe 0 -i mylist.txt -c copy output.mp4 + + # ffmpeg -f concat -safe 0 -i mylist.txt -c copy output.mp4 + stdin = "\n".join(f"file '{video.absolute()}'" for video in videos if video.is_file()) + # fmt: off + args = [ + 'ffmpeg', + '-protocol_whitelist', 'file,pipe', + '-f', 'concat', + '-safe', '0', + '-i', '-', + '-c', 'copy', + str(save_path), + ] + # fmt: on + logging.info(f"calling ffmpeg with {args=} and {stdin=}") + subprocess.run(args, input=stdin.encode(), check=True) + return save_path + + +def main(): + logging.basicConfig(level=logging.INFO) + args = parse_args(sys.argv[1:]) + paths = [Path(f) for f in args.files] + combine_videos(videos=paths, save_path=args.output_path) + + +if __name__ == "__main__": + main() diff --git a/ffmpeg_gif.py b/ffmpeg_gif.py new file mode 100755 index 0000000..4fe23d2 --- /dev/null +++ b/ffmpeg_gif.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3.9 +import argparse +import datetime +import hashlib +import math +import re + +import subprocess +import sys +import typing +from pathlib import Path + +import ffmpeg + + +def create_gif( + video_path: Path, + from_time: str, + to_time: typing.Optional[str] = None, + save_path: typing.Optional[Path] = None, + duration: typing.Optional[int] = 10, + fps: int = 15, + width: int = 320, +) -> Path: + if not save_path: + # strip extra decimals + time_clean = re.sub(r"\.(\d)\d+", r".\1", from_time).replace(":", "") + save_path = video_path.with_name(f"{video_path.stem}_{time_clean}.gif") + + info = ffmpeg.ffprobe(video_path) + scaled_width, scaled_height = width, math.ceil(width / info.ar / info.sample_ar) + + subprocess.run( + [ + "ffmpeg", + "-v", + "warning", + "-y", + "-ss", + from_time, + *(["-to", to_time] if to_time else ["-t", str(duration)]), + "-ignore_chapters", + "1", + "-i", + str(video_path), + "-filter_complex", + # f"fps={fps},scale={scale}:-1:flags=lanczos[x];[x]split[x1][x2];[x1]palettegen[p];[x2][p]paletteuse", + f"[0:v] fps={fps},scale={scaled_width}:{scaled_height},split [a][b];[a] palettegen=stats_mode=full [p];[b][p] paletteuse=new=1", + str(save_path), + ], + check=True, + ) + + return save_path + + +def optimize_gif(gif_path: Path) -> Path: + subprocess.run( + [ + "gifsicle", + "--lossy=98", + "--optimize=3", + "--batch", + "-i", + str(gif_path), + ], + check=True, + ) + return gif_path + + +def parse_args(argv: typing.List[str] = None) -> argparse.Namespace: + arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + arger.add_argument("video_path", type=str, help="Path to video file") + arger.add_argument( + "--start", "-s", dest="from_time", type=str, help="Time to start from, e.g. 02:59", required=True + ) + arger.add_argument("--duration", "-d", default=3, type=int, help="Duration of gif in seconds") + arger.add_argument("--fps", default=15, type=int, help="Frames per second") + arger.add_argument("--width", default=400, type=int, help="Image width") + arger.add_argument("--optimize", default=True, action="store_true", help="Reduce GIF filesize") + + if len(argv) == 0: + argv = ["--help"] + + return arger.parse_args(argv) + + +def md5(text: str) -> str: + return hashlib.md5(text.encode("utf-8")).hexdigest() + + +def main(): + args = parse_args(sys.argv[1:]) + video_path = Path(args.video_path) + + from_time = args.from_time + try: + time = datetime.timedelta(seconds=round(float(from_time), 1)) + from_time = str(time) + if time.total_seconds() < 3600: + from_time = from_time[2:] + except ValueError: + pass + + save_path = None + if str(args.video_path).startswith("http"): + save_path = Path("~/Downloads").expanduser() / f"{md5(str(video_path))}__{from_time.replace(':', '')}.gif" + + gif_path = create_gif( + video_path=video_path, + save_path=save_path, + from_time=from_time, + duration=args.duration, + fps=args.fps, + width=args.width, + ) + if args.optimize: + try: + gif_path = optimize_gif(gif_path) + except: + pass + print(gif_path.resolve()) + + +if __name__ == "__main__": + main() diff --git a/ffmpeg_gif_summary.py b/ffmpeg_gif_summary.py new file mode 100755 index 0000000..e4f06cf --- /dev/null +++ b/ffmpeg_gif_summary.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3.9 +import argparse +import datetime +import math + +import subprocess +import sys +import typing +from pathlib import Path +import re + +import ffmpeg_gif + +def parse_duration(text: str) -> datetime.timedelta: + re_duration = re.compile(r"(\d+(?:\.\d+)?(?:sec|min|hour|hr|h|m|s))") + re_digits = re.compile(r"([\d.]+)") + time_scales = { + "s": 1, + "sec": 1, + "m": 60, + "min": 60, + "h": 3600, + "hr": 3600, + } + + parts = re_duration.findall(text) + if parts: + total_secs = 0 + for it in parts: + if m := re_digits.search(it): + _, end = m.span() + scale = it[end:] + total_secs += math.ceil(float(m.group(1)) * time_scales[scale]) + return datetime.timedelta(seconds=total_secs) + + parts = [float(it) for it in text.split(":")] + if len(parts) == 1: + return datetime.timedelta(seconds=parts[0]) + if len(parts) == 2: + return datetime.timedelta(minutes=parts[0], seconds=parts[1]) + if len(parts) == 3: + return datetime.timedelta(hours=parts[0], minutes=parts[1], seconds=parts[2]) + + + +def parse_args(argv: typing.List[str] = None) -> argparse.Namespace: + arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + arger.add_argument("video_path", type=Path, help="Path to video file") + arger.add_argument("--interval", "-i", default=60, type=int, help="Interval between GIFs in seconds") + arger.add_argument("--duration", "-d", default=3, type=int, help="GIF duration in seconds") + arger.add_argument("--fps", default=18, type=float, help="Frames per second") + arger.add_argument("--width", default=400, type=int, help="Image width") + arger.add_argument("--optimize", default=True, action="store_true", help="Reduce GIF filesize") + arger.add_argument("--from", dest='from_time', type=parse_duration, help="Start from timestamp") + arger.add_argument("--to", dest='to_time', type=parse_duration, help="Stop at timestamp") + + if len(argv) == 0: + argv = ["--help"] + + return arger.parse_args(argv) + + +def get_video_duration(video_path: Path) -> datetime.timedelta: + # fmt: off + args = [ + 'ffprobe', + '-v', 'error', + '-show_entries', 'format=duration', + '-of', 'default=noprint_wrappers=1:nokey=1', + video_path, + ] + # fmt: on + p = subprocess.run(args, stdout=subprocess.PIPE, check=True) + return datetime.timedelta(seconds=round(float(p.stdout), 1)) + + +def summarize(video_path: Path, width: int = 400, gif_duration: int = 3, fps: int = 15, interval_seconds: int = 60, optimize: bool = True, +from_time: typing.Optional[datetime.timedelta] = None, to_time: typing.Optional[datetime.timedelta] = None) -> None: + duration = get_video_duration(video_path) + + if not from_time: + from_time = datetime.timedelta(minutes=2) + if not to_time: + to_time = duration + + count = math.ceil((to_time - from_time).total_seconds() // interval_seconds) + interval = (to_time - from_time) / count + for i in range(count): + start = from_time + datetime.timedelta(seconds=round(interval.total_seconds() * i, 1)) + end = start + datetime.timedelta(seconds=gif_duration) + if end >= to_time: + break + gif_path = ffmpeg_gif.create_gif( + video_path=video_path, + from_time=str(start), + to_time=str(end), + width=width, + fps=fps, + duration=duration, + ) + if optimize: + gif_path = ffmpeg_gif.optimize_gif(gif_path) + print(str(gif_path.absolute())) + +def main(): + args = parse_args(sys.argv[1:]) + video_path: Path = args.video_path + summarize( + video_path=video_path, + width=args.width, + gif_duration=args.duration, + fps=args.fps, + optimize=args.optimize, + interval_seconds=args.interval, + from_time=args.from_time, + to_time=args.to_time, + ) + + +if __name__ == "__main__": + main() diff --git a/ffmpeg_resolution.py b/ffmpeg_resolution.py new file mode 100755 index 0000000..4681525 --- /dev/null +++ b/ffmpeg_resolution.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3.9 + +from pathlib import Path +import subprocess +import sys +from ffmpeg import ffprobe +import logging + + +def get_resolution(video_path: Path) -> str: + try: + info = ffprobe(video_path) + except subprocess.CalledProcessError: + return None + + if 700 <= info.height <= 800: + return "720p" + elif 1000 <= info.height <= 1200: + return "1080p" + elif info.height > 2000: + return "4K" + return None + + +def main(): + for it in sys.argv[1:]: + video_path = Path(it) + if not video_path.is_file(): + continue + resolution = get_resolution(video_path) + if resolution is None: + logging.info("unknown resolution: %s", video_path) + continue + if resolution in video_path.stem: + continue + + logging.info("%s: %s", video_path.stem, resolution) + new_path = video_path.with_stem(f"{video_path.stem} [{resolution}]") + video_path.rename(new_path) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() diff --git a/ffmpeg_strip.py b/ffmpeg_strip.py new file mode 100755 index 0000000..67e2dba --- /dev/null +++ b/ffmpeg_strip.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3.9 +import argparse +import json +import logging +import subprocess +import sys +from pathlib import Path +import typing + + +def parse_args(argv: list[str]): + arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + arger.add_argument("video_path", type=Path, help="Video path") + arger.add_argument("save_path", nargs="?", type=Path, help="Output path") + if len(argv) == 0: + arger.print_help() + exit(1) + + return arger.parse_args(argv) + + +def ffprobe(video_path: Path) -> dict: + proc = subprocess.run( + [ + "ffprobe", + "-v", + "quiet", + "-print_format", + "json", + "-show_format", + "-show_streams", + str(video_path), + ], + check=True, + capture_output=True, + ) + return json.loads(proc.stdout) + + +def clean_video(video_path: Path, save_path: Path) -> Path: + media_info = ffprobe(video_path) + # check if the first video stream has hevc codec + if media_info["streams"][0]["codec_name"] == "hevc": + logging.info(f"Video codec is hevc, adding hvc1 tag") + extra_args = ["-tag:v", "hvc1"] + else: + extra_args = [] + + # fmt: off + args = [ + 'ffmpeg', + '-i', + str(video_path), + '-c:v', 'copy', + # '-movflags', '+faststart', + '-map_metadata', '-1', + *extra_args, + '-c:a', 'copy', + '-y', + str(save_path), + ] + # fmt: on + logging.info(f"calling ffmpeg with {args=}") + subprocess.run(args, check=True) + return save_path + + +def main(): + logging.basicConfig(level=logging.INFO) + args = parse_args(sys.argv[1:]) + save_path: typing.Optional[Path] = args.save_path + if save_path and save_path.resolve().is_dir(): + save_path = save_path / args.video_path.name + if save_path is None: + save_path = args.video_path.with_suffix(".clean.mp4") + + clean_video(video_path=args.video_path, save_path=save_path) + + +if __name__ == "__main__": + main() diff --git a/file_renamer.py b/file_renamer.py new file mode 100755 index 0000000..64887a2 --- /dev/null +++ b/file_renamer.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3.10 +import argparse +import dataclasses +import datetime +import json +import logging +import re +import subprocess +import typing +from pathlib import Path + +script_path = Path(__file__) +db_filename = script_path.with_name(f'{script_path.stem}.known.txt') + + +def save_actors(actors: typing.Iterable[str]): + existing = load_actors() + all_actors = sorted({*existing, *actors}) + with db_filename.open('w', encoding='utf-8', newline='\n') as f: + f.writelines(all_actors) + + +def load_actors() -> set[str]: + try: + with db_filename.open('w', encoding='utf-8', newline='\n') as f: + return {line.strip() for line in f} + except FileNotFoundError: + return set() + + +re_date_us = re.compile(r"\b(\d{2})\D(\d{2})\D(\d{2})\b") # 12/31/21 +re_date_iso = re.compile(r"\b(\d{4})\D?(\d{2})\D?(\d{2})\b") # 2020-12-31 +re_date_iso_short = re.compile(r"(\d{2})\D?(\d{2})\D?(\d{2})") # 20-12-31"' +re_date_iso_rev = re.compile(r"\b(\d{2})\D?(\d{2})\D?(\d{4})\b") # 31-12-2020 +re_date_iso_rev_short = re.compile(r"\b(\d{2})\D(\d{2})\D(\d{2})\b") # 31.12.21 + + +class Pattern: + def __init__(self, pattern: re.Pattern): + self.re = pattern if isinstance(pattern, re.Pattern) else re.compile(pattern) + + def __eq__(self, other): + if isinstance(other, str): + return self.re.search(other) + raise ValueError + + +@dataclasses.dataclass +class ParsedDate: + date: datetime.date + start: int + end: int + + +def parse_date(filename: str) -> ParsedDate | None: + def from_us_format(): + match = re_date_us.search(filename) + m, d, y = map(int, match.groups()) + return ParsedDate(datetime.date(y + 2000, m, d), start=match.start(), end=match.end()) + + def from_iso(): + match = re_date_iso.search(filename) + y, m, d = map(int, match.groups()) + return ParsedDate(datetime.date(y, m, d), start=match.start(), end=match.end()) + + def from_iso_short(): + match = re_date_iso_short.search(filename) + y, m, d = map(int, match.groups()) + return ParsedDate(datetime.date(y + 2000, m, d), start=match.start(), end=match.end()) + + def from_iso_reversed(): + match = re_date_iso_rev.search(filename) + d, m, y = map(int, match.groups()) + return ParsedDate(datetime.date(y, m, d), start=match.start(), end=match.end()) + + def from_iso_reversed_short(): + match = re_date_iso_rev_short.search(filename) + d, m, y = map(int, match.groups()) + return ParsedDate(datetime.date(y + 2000, m, d), start=match.start(), end=match.end()) + + candidates = [] + for fn in [ + from_us_format, + from_iso, + from_iso_short, + from_iso_reversed, + from_iso_reversed_short, + ]: + try: + candidates.append(fn()) + except (ValueError, AttributeError): + pass + + today = datetime.date.today() + future_threshold = today + datetime.timedelta(days=60) + past_threshold = datetime.date(2010, 1, 1) + candidates = [it for it in candidates if past_threshold < it.date < future_threshold] + + if not candidates: + return + + return candidates[0] + + +@dataclasses.dataclass +class Release: + actors: list[str] + studio: str | None = None + title: str | None = None + released_at: datetime.date | None = None + + def to_filename(self): + parts = [] + if self.actors: + parts.append(', '.join(self.actors)) + if self.studio: + parts.append(f'@{self.studio}') + if self.title: + parts.append(self.title) + if self.released_at: + parts.append(self.released_at.isoformat()) + return ' -- '.join(parts) + + +def trash(path: Path): + filename = f'"{path}"' + cmd = ['osascript', '-e', f'tell app "Finder" to move (POSIX file {filename}) to trash'] + subprocess.run(cmd).check_returncode() + + +def filenames_to_actors(): + sources = [ + Path(r'/Users/abdus/Downloads/temp/'), + Path(r'/Volumes/BANDAID/_temp/__reenc/'), + Path(r'/Volumes/BANDAID/_temp/'), + ] + actors = set() + for it in sources: + for f in it.glob('*.mp4'): + if not f.is_file(): + continue + if r := parse_release(f.name): + actors.update(r.actors) + return actors + + +def parse_release(filename: str, known_actors: set[str] | None = None): + if not known_actors: + known_actors = set() + filename = re.sub(r'\.\w{3,4}$', '', filename) + + def from_own(): + remaining = re.sub(r'\s+\[[^]]+]$', '', filename) + remaining = re.sub(r'\s+\[([^]]+|\d+\w)(,\s*[^]]+)?]', '', remaining) + remaining = re.sub(r'\s*--\s*', ' -- ', remaining) + + match remaining.split(' -- '): + case [actors, studio, title, date] if studio.startswith('@'): + return Release( + actors=sorted(actors.split(', ')), + studio=studio.removeprefix('@'), + title=title, + released_at=datetime.date.fromisoformat(date), + ) + case [actors, studio, date] if studio.startswith('@') and (parsed := parse_date(date)): + return Release( + actors=sorted(actors.split(', ')), + studio=studio.removeprefix('@'), + title=None, + released_at=parsed.date, + ) + case [actors, studio, title] if studio.startswith('@'): + return Release( + actors=sorted(actors.split(', ')), + studio=studio.removeprefix('@'), + title=title, + released_at=None, + ) + case [actors, title, date] if (parsed := parse_date(date)): + return Release( + actors=sorted(actors.split(', ')), + studio=None, + title=title, + released_at=parsed.date, + ) + case [actors, studio] if studio.startswith('@'): + return Release( + actors=sorted(actors.split(', ')), + studio=studio, + ) + case [actors, title]: + return Release( + actors=sorted(actors.split(', ')), + title=title, + ) + + def from_prt(): + assert '.PRT' in filename + remaining = re.sub(r'\.(720p|1080p|HEVC|x265|PRT|XXX)', ' ', filename) + studio = remaining[:remaining.index('.')] + parsed_date = parse_date(remaining) + remaining = remaining[parsed_date.end:] + remaining = re.sub(r'[. ]+', ' ', remaining).strip() + + actors = [] + title = None + + match remaining.split(' '): + case [a_first, a_last, 'And', b_first, b_last]: + actors = [f'{a_first} {a_last}', f'{b_first} {b_last}'] + case [a_first, a_last, 'And', b_first, b_last, *rest]: + actors = [f'{a_first} {a_last}', f'{b_first} {b_last}'] + title = ' '.join(rest) + case [first, last]: + actors = [f'{first} {last}'] + case [first, last, *rest]: + actors = [f'{first} {last}'] + title = ' '.join(rest) + case [first]: + actors = [first] + + return Release( + studio=studio, + actors=sorted(actors), + title=title, + released_at=parsed_date.date, + ) + + def from_galaxxxy(): + pass + + for fn in [ + from_own, + from_prt, + from_galaxxxy, + ]: + try: + if res := fn(): + return res + except (ValueError, AttributeError, AssertionError): + pass + + +def parse_args(): + arger = argparse.ArgumentParser() + arger.add_argument('filenames', nargs='+', type=lambda v: Path(v).resolve(), help='Filenames') + return arger.parse_args() + + +@dataclasses.dataclass +class MoveOp: + source: Path + target: Path + + def to_dict(self) -> dict: + return { + 'source': str(self.source), + 'target': str(self.target), + } + + +def write_undo_script(save_dir: Path, ops: list[MoveOp]): + now = datetime.datetime.now().isoformat().replace(':', '') + save_path = save_dir / f'undo_{now}.json' + with save_path.open('w', encoding='utf-8') as f: + for it in ops: + f.write(json.dumps(it.to_dict()) + '\n') + +def fix_filename(): + pass + + +def main(): + actors = load_actors() + + args = parse_args() + paths: list[Path] = args.filenames + + jobs = [] + for it in paths: + if parsed := parse_release(it.name, known_actors=actors): + logging.debug(f'filename={it} release={parsed}') + suggested = parsed.to_filename() + target = it.with_stem(suggested) + if target.is_file(): + logging.error('target already exists') + continue + jobs.append(MoveOp(source=it, target=target)) + + if not jobs: + return + + write_undo_script(paths[0].parent, jobs) + + for it in jobs: + it.target.hardlink_to(it.source) + + for it in jobs: + trash(it.source) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + main() diff --git a/file_renamer_test.py b/file_renamer_test.py new file mode 100644 index 0000000..f4acedb --- /dev/null +++ b/file_renamer_test.py @@ -0,0 +1,120 @@ +import datetime + +import pytest + +import file_renamer +from file_renamer import Release + + +@pytest.mark.parametrize( + ["filename", "date"], + [ + [ + "Tushy.21.09.13.Jia.Lissa.XXX.720p.WEB.x264-GalaXXXy", + datetime.date(2021, 9, 13), + ], + [ + "[AdultTime] Aidra Fox (Lady Gonzo - 28.02.2019) [720p HEVC].mkv", + datetime.date(2019, 2, 28), + ], + [ + "15.01.10_-_Aspen_Ora_-_Cock_Crazy_-_x265.mp4", + datetime.date(2015, 1, 10), + ], + [ + "Karma Rx, Valerica Steele - Valerica & Karma’s Anal Slutfest -- AllAnal 20210912", + datetime.date(2021, 9, 12), + ], + [ + "Scarlett Hampton - Taking Scarlett’s Ass For a Ride 09_15_21.mp4", + datetime.date(2021, 9, 15), + ], + [ + "Sia_Siberia & LittleReislin - Best Valentine's Day gift ever Release Date _ 12_10_20.mp4", + datetime.date(2020, 12, 10), + ], + [ + "Kenna James - Succubus 09_09_21.mp4", + datetime.date(2021, 9, 9), + ], + [ + "Hot Bitch Dreaming about being Hard Fucked by her Stepbrother - Diana Daniels_Diana Daniels_720p.mp4", + None, + ], + [ + "DadCrush.21.10.09.Ailee.Anne.My.Stepdaughters.Hot.XXX.1080p.HEVC.x265.PRT.mp4", + datetime.date(2021, 10, 9) + ] + ], +) +def test_parse_date(filename: str, date: datetime.date | None): + parsed = file_renamer.parse_date(filename) + + if not date: + assert parsed is None + return + + assert parsed.date == date + + +@pytest.mark.parametrize(['filename', 'expected'], [ + [ + 'Rebecca Volpetti -- @RealityKings -- Driving Him Crazy [1080p, x265] -- 2019-04-10', + Release(actors=['Rebecca Volpetti'], studio='RealityKings', title='Driving Him Crazy', released_at=datetime.date(2019, 4, 10)), + ], + [ + 'Liz Jordan -- @Lubed -- Sopping Oil -- 2023-02-07 [1080p, x265]', + Release(actors=['Liz Jordan'], studio='Lubed', title='Sopping Oil', released_at=datetime.date(2023, 2, 7)), + ], + [ + 'Liz Jordan--@Lubed--Sopping Oil--2023-02-07 [1080p, x265]', + Release(actors=['Liz Jordan'], studio='Lubed', title='Sopping Oil', released_at=datetime.date(2023, 2, 7)), + ], + [ + 'Liz Jordan -- @Lubed -- 2023-02-07 [1080p, x265]', + Release(actors=['Liz Jordan'], studio='Lubed', title=None, released_at=datetime.date(2023, 2, 7)), + ], + [ + 'Liz Jordan -- Sopping Oil -- 2023-02-07 [1080p, x265]', + Release(actors=['Liz Jordan'], studio=None, title='Sopping Oil', released_at=datetime.date(2023, 2, 7)), + ], + [ + 'Liz Jordan -- Sopping Oil', + Release(actors=['Liz Jordan'], studio=None, title='Sopping Oil', released_at=None), + ], + [ + 'DadCrush.21.10.09.Ailee.Anne.My.Stepdaughters.Hot.XXX.1080p.HEVC.x265.PRT.mp4', + Release(studio='DadCrush', actors=['Ailee Anne'], title='My Stepdaughters Hot', released_at=datetime.date(2021, 10, 9)) + ], + [ + 'Deeper.21.11.11.Kenzie.Anne.XXX.1080p.HEVC.x265.PRT.mkv', + Release(studio='Deeper', actors=['Kenzie Anne'], title=None, released_at=datetime.date(2021, 11, 11)), + ], + [ + 'DevilsFilm.22.03.19.Kira.Noir.Wife.Swap.Schemes.2.XXX.1080p.HEVC.x265.PRT.mkv', + Release(studio='DevilsFilm', actors=['Kira Noir'], title='Wife Swap Schemes 2', released_at=datetime.date(2022, 3, 19)), + ], + [ + 'DogHouseDigital.22.04.06.Maddy.May.Full.Service.Massage.XXX.1080p.HEVC.x265.PRT.mkv', + Release(studio='DogHouseDigital', actors=['Maddy May'], title='Full Service Massage', released_at=datetime.date(2022, 4, 6)), + ], + [ + 'EvilAngel.22.04.05.Diana.Grace.XXX.1080p.HEVC.x265.PRT.mkv', + Release(studio='EvilAngel', actors=['Diana Grace'], title=None, released_at=datetime.date(2022, 4, 5)), + ], + [ + 'Slayed.21.10.07.Izzy.Lush.And.Aidra.Fox.XXX.1080p.HEVC.x265.PRT.mkv', + Release(studio='Slayed', actors=['Aidra Fox', 'Izzy Lush'], title=None, released_at=datetime.date(2021, 10, 7)), + ], + [ + 'ExploitedCollegeGirls.22.06.30.Gaby.XXX.1080p.HEVC.x265.PRT.mkv', + Release(studio='ExploitedCollegeGirls', actors=['Gaby'], title=None, released_at=datetime.date(2022, 6, 30)), + ], + [ + 'DoctorAdventures.21.05.26.Jamie.Michelle.Nurse.Jamie.Knows.Best.PRT.mp4', + Release(studio='DoctorAdventures', actors=['Jamie Michelle'], title='Nurse Jamie Knows Best', released_at=datetime.date(2021, 5, 26)), + ], +]) +def test_parse_release(filename: str, expected): + parsed = file_renamer.parse_release(filename) + assert parsed == expected diff --git a/fpup.py b/fpup.py new file mode 100755 index 0000000..d755ce3 --- /dev/null +++ b/fpup.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3.9 +import argparse +import concurrent.futures +import json +import re +import sys +import typing +from pathlib import Path +from typing import Optional +import logging + + +import httpx + + +def upload_image(image_path: Path) -> dict: + session = httpx.Client(timeout=20) + logging.info(f"uploading {image_path.name}") + + with image_path.open("rb") as f: + res = session.post( + "https://fapping.empornium.sx/upload.php", + files={ + "ImageUp": f, + }, + ) + + res = session.get("https://fapping.empornium.sx/uploaded/", follow_redirects=True) + if m := re.search(r"var ImagesUp = (\[.+\]);", res.text, re.MULTILINE): + data: list[dict] = json.loads(m.group(1)) + logging.info(f"upload result on the page={data}") + + return dict( + image_url=data[0]["image_url"], + thumbnail_url=data[0]["image_thumb_url"], + ) + + logging.error(f"failed to find upload result on the page. html={res.text}") + + raise ValueError(f"failed to parse upload result for {image_path.name}") + + +def parse_args(argv: Optional[typing.Sequence[str]] = None) -> argparse.Namespace: + arger = argparse.ArgumentParser() + arger.add_argument("image_paths", nargs="+", action="store", type=Path, help="Path to image to upload") + if not argv or len(argv) == 0: + arger.print_help() + sys.exit(1) + + return arger.parse_args(argv) + + +def main(): + args = parse_args(sys.argv[1:]) + image_paths = args.image_paths + + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(upload_image, image_paths)) + for it in results: + print(it["image_url"]) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() diff --git a/gardener.py b/gardener.py new file mode 100644 index 0000000..7865c84 --- /dev/null +++ b/gardener.py @@ -0,0 +1,207 @@ +import dataclasses +import logging +import os +import webbrowser + + +def force_import(module: str): + import importlib + import subprocess + import sys + + try: + return importlib.import_module(module) + except ModuleNotFoundError: + subprocess.run([sys.executable, "-m", "pip", "install", module]) + importlib.invalidate_caches() + return importlib.import_module(module) + + +try: + import httpx +except ImportError: + torf = force_import("httpx") + +try: + import typer +except ImportError: + typer = force_import("typer") +try: + import rich +except ImportError: + rich = force_import("rich") +try: + import inquirer +except ImportError: + inquirer = force_import("inquirer") + +from rich import print +import rich.table + +JIRA_EMAIL = os.getenv('JIRA_EMAIL', 'abdussamet.kocak@akinon.com') +JIRA_TOKEN = os.getenv('JIRA_TOKEN', 'iAZhlJ1AdGwckbVllhlx48C4') +JIRA_URL = os.getenv('JIRA_URL', 'https://omniplatform.atlassian.net/') + +if not JIRA_TOKEN: + logging.info('') + webbrowser.open('https://id.atlassian.com/manage-profile/security/api-tokens') + + +@dataclasses.dataclass +class Issue: + id: str + key: str + summary: str + + +@dataclasses.dataclass +class PullRequest: + id: str + name: str + repository: str + branch: str + + +class Jira: + def __init__(self, email: str, token: str): + self.client = httpx.Client( + base_url=JIRA_URL, + auth=httpx.BasicAuth(email, token), + ) + + def list_issues_for_release(self) -> list[Issue]: + mergeable_status = 'In Review' + res = self.client.post( + '/rest/api/2/search', + json={ + 'jql': f'project = COM AND status = "{mergeable_status}" AND "Team[Dropdown]" = Backend ORDER BY created DESC' + }, + ) + + res.raise_for_status() + data = res.json() + development_type = ['Story', 'Bug', 'Task'] + return [ + Issue(id=it['id'], key=it['key'], summary=it['fields']['summary']) + for it in data['issues'] + if it['fields']['issuetype']['name'] in development_type + ] + + def get_approved_prs(self, issue_id: str): + res = self.client.post( + f'/jsw/graphql', + params={'operation': 'DevDetailsDialog'}, + json={ + "operationName": "DevDetailsDialog", + "query": """ + query DevDetailsDialog ($issueId: ID!) { + developmentInformation(issueId: $issueId){ + details { + instanceTypes { + repository { + name + branches { + name + pullRequests { + name + status + lastUpdate + } + reviews { + state + id + } + } + pullRequests { + id + name + branchName + status + reviewers{ + name + isApproved + } + } + } + danglingPullRequests { + id + name + branchName + status + reviewers{ + name + isApproved + } + } + } + } + } + } + """, + "variables": {"issueId": issue_id}, + }, + ) + res.raise_for_status() + data = res.json()['data'] + if not data['developmentInformation']['details']['instanceTypes']: + return [] + repo_name = data['developmentInformation']['details']['instanceTypes'][0]['repository'][0]['name'] + prs = data['developmentInformation']['details']['instanceTypes'][0]['danglingPullRequests'] + if not prs: + repo_name = data['developmentInformation']['details']['instanceTypes'][0]['repository'][0]['name'] + prs = data['developmentInformation']['details']['instanceTypes'][0]['repository'][0]['pullRequests'] + + return [ + PullRequest( + id=it['id'], + name=it['name'], + repository=repo_name, + branch=it['branchName'], + ) + for it in prs + if it['status'] != 'MERGED' + ] + + +console = rich.console.Console() + + +def render_issue(issue: Issue, prs: list[PullRequest]): + t = rich.table.Table( + title=f'{issue.key} -- {issue.summary}', + caption_justify='left', + ) + t.add_column("Repository", no_wrap=True) + t.add_column("Branch", no_wrap=True) + t.add_column("Pull Request") + + for it in prs: + t.add_row(it.repository, it.branch, it.name) + + console.print(t) + + +def main(): + logging.basicConfig(level=logging.INFO) + if not JIRA_EMAIL: + logging.error('JIRA_EMAIL is not set') + exit(1) + if not JIRA_TOKEN: + logging.error('JIRA_TOKEN is not set') + exit(1) + + j = Jira(email=JIRA_EMAIL, token=JIRA_TOKEN) + + logging.info('fetching issues in "ready to release" status') + issues = j.list_issues_for_release() + for it in issues: + logging.info(f'fetching pull requests for issue {it.key}') + prs = j.get_approved_prs(it.id) + if not prs: + logging.info(f'no non-merged pull requests found for issue {it.key}') + continue + render_issue(it, prs) + + +if __name__ == '__main__': + main() diff --git a/hetzner_kube.py b/hetzner_kube.py new file mode 100755 index 0000000..1b0f77b --- /dev/null +++ b/hetzner_kube.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3.9 +import base64 +import json +import logging +import pprint +import subprocess +import time +import webbrowser +from pathlib import Path + +import httpx +import typer + +hetzner_token = 'zdlkmAF9EswvOIEGvWgutULYTmUpa3c44YcKZOB0jymQfgvra7kRVJxuLiFTouM9' +client = httpx.Client(base_url='https://api.hetzner.cloud/v1', headers={'Authorization': f'Bearer {hetzner_token}'}) + + +datacenters = [ + {'id': 2, 'name': 'nbg1-dc3', 'description': 'Nuremberg 1 DC 3'}, + {'id': 3, 'name': 'hel1-dc2', 'description': 'Helsinki 1 DC 2'}, + {'id': 4, 'name': 'fsn1-dc14', 'description': 'Falkenstein 1 DC14'}, + {'id': 5, 'name': 'ash-dc1', 'description': 'Ashburn DC1'}, +] + +server_types = { + 'cx11': {'cores': 1, 'memory': 2.0, 'disk': 20}, + 'cx21': {'cores': 2, 'memory': 4.0, 'disk': 40}, + 'cx31': {'cores': 2, 'memory': 8.0, 'disk': 80}, + 'cx41': {'cores': 4, 'memory': 16.0, 'disk': 160}, + 'cx51': {'cores': 8, 'memory': 32.0, 'disk': 240}, + 'ccx11': {'cores': 2, 'memory': 8.0, 'disk': 80}, + 'ccx21': {'cores': 4, 'memory': 16.0, 'disk': 160}, + 'ccx31': {'cores': 8, 'memory': 32.0, 'disk': 240}, + 'ccx41': {'cores': 16, 'memory': 64.0, 'disk': 360}, + 'ccx51': {'cores': 32, 'memory': 128.0, 'disk': 600}, + 'cpx11': {'cores': 2, 'memory': 2.0, 'disk': 40}, + 'cpx21': {'cores': 3, 'memory': 4.0, 'disk': 80}, + 'cpx31': {'cores': 4, 'memory': 8.0, 'disk': 160}, + 'cpx41': {'cores': 8, 'memory': 16.0, 'disk': 240}, + 'cpx51': {'cores': 16, 'memory': 32.0, 'disk': 360}, + 'ccx12': {'cores': 2, 'memory': 8.0, 'disk': 80}, + 'ccx22': {'cores': 4, 'memory': 16.0, 'disk': 160}, + 'ccx32': {'cores': 8, 'memory': 32.0, 'disk': 240}, + 'ccx42': {'cores': 16, 'memory': 64.0, 'disk': 360}, + 'ccx52': {'cores': 32, 'memory': 128.0, 'disk': 600}, + 'ccx62': {'cores': 48, 'memory': 192.0, 'disk': 960}, +} + + +def create_server(name: str, datacenter: str): + payload = { + "datacenter": datacenter, + "image": "ubuntu-20.04", + "name": name, + "server_type": "cpx21", + "ssh_keys": ["entropy"], + } + + res = client.post('/servers', json=payload) + res.raise_for_status() + + data = res.json() + server = data['server'] + pprint.pprint(server) + + return { + 'id': server['id'], + 'name': server['name'], + 'ip': server['public_net']['ipv4']['ip'], + } + + +def find_server(name: str) -> dict: + res = client.get('/servers') + res.raise_for_status() + + data = res.json() + + server = [it for it in data['servers'] if it['name'] == name][0] + return { + 'id': server['id'], + 'name': server['name'], + 'ip': server['public_net']['ipv4']['ip'], + } + + +def delete_server(name: str) -> None: + server = find_server(name) + logging.info(f'deleting server={name} with id={server["id"]}') + + res = client.delete(f'/servers/{server["id"]}') + res.raise_for_status() + + logging.info(f'deleted server={name}') + + +def copy_to_clipboard(text: str) -> None: + subprocess.Popen(['pbcopy'], stdin=subprocess.PIPE).communicate(input=text.encode()) + + +class Server: + ssh_options = ['-o', 'StrictHostKeyChecking=no'] + + def __init__(self, ip_address: str, name: str = None): + self.ip = ip_address + self.ssh_host = f'root@{self.ip}' + self.name = name + + @classmethod + def from_hetzner(cls, server: dict): + return cls(server['ip'], name=server['name']) + + def run_command(self, cmd: str, return_output: bool = False) -> str: + + logging.debug(f'running command={cmd} on {self.ssh_host}') + extra_args = {} + if return_output: + extra_args['stdout'] = subprocess.PIPE + p = subprocess.run(['ssh', *self.ssh_options, self.ssh_host, '--', cmd], text=True, **extra_args) + p.check_returncode() + return p.stdout + + def install_package(self, packages: str) -> str: + return self.run_command(f'apt-get install -y {packages}') + + def install_shell(self): + self.install_package('fish') + self.run_command('chsh -s /usr/bin/fish') + + def install_microk8s(self): + self.install_package('snapd') + self.run_command('snap install microk8s --classic') + self.run_command('ufw allow in on cni0 && ufw allow out on cni0 && ufw default allow routed') + self.run_command('microk8s enable dashboard dns rbac ingress storage prometheus registry') + + def get_microk8s_token(self): + raw_json = self.run_command('microk8s kubectl -n kube-system get secret -o json', return_output=True) + secrets = json.loads(raw_json)['items'] + + default_token_secret = [it for it in secrets if it['metadata']['name'].startswith('default-token')][0] + token_base64 = default_token_secret['data']['token'] + + token = base64.decodebytes(token_base64.encode()).decode() + return token + + def show_microk8s_access_info(self): + logging.info('getting default service account token') + token = self.get_microk8s_token() + print(token) + logging.info('copying token to clipboard') + copy_to_clipboard(token) + + logging.info('getting dashboard ip') + proxy = self.get_microk8s_dashboard_proxy() + print(proxy['ssh']) + copy_to_clipboard(proxy['ssh']) + + logging.info('opening url in the browser') + print(proxy['url']) + webbrowser.open_new_tab(proxy['url']) + + def get_microk8s_dashboard_proxy(self): + cluster_ip = self.run_command( + 'microk8s kubectl get -n kube-system -o json service/kubernetes-dashboard -o jsonpath="{.spec.clusterIP}"', + return_output=True, + ).strip() + port, local_port = 443, 18433 + + return { + 'ssh': f'ssh -vNL {local_port}:{cluster_ip}:{port} {self.ssh_host}', + 'url': f'https://localhost:{local_port}', + } + + def write_kubeconfig(self, filename: str): + logging.info(f'writing kubeconfig to {filename}') + kubeconfig = self.run_command('microk8s config', return_output=True) + file_path = Path(filename).expanduser() + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.write_text(kubeconfig) + + def install_micro_editor(self): + logging.info('installing micro editor') + self.run_command('curl https://getmic.ro | bash && mv ./micro /usr/bin/') + + def setup(self): + self.run_command('apt-get update') + + self.install_shell() + self.install_micro_editor() + self.install_microk8s() + + logging.info('waiting until pods are online') + time.sleep(10) + + self.show_microk8s_access_info() + self.write_kubeconfig(f'~/Desktop/kubeconfig-{self.name}.yml') + + +cli = typer.Typer(invoke_without_command=False) + + +@cli.command() +def create( + name: str, + datacenter: str = typer.Option('nbg1-dc3'), +): + create_server(name=name, datacenter=datacenter) + + +@cli.command() +def destroy( + name: str, +): + delete_server(name=name) + + +@cli.command() +def setup( + name: str, +): + server = find_server(name) + server = Server.from_hetzner(server) + server.setup() + + +@cli.command() +def show_k8s( + name: str, +): + server = Server.from_hetzner(find_server(name)) + server.show_microk8s_access_info() + server.write_kubeconfig(f'~/Desktop/kubeconfig-{server.name}.yml') + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + cli() diff --git a/hooksy/app.py b/hooksy/app.py new file mode 100644 index 0000000..f969217 --- /dev/null +++ b/hooksy/app.py @@ -0,0 +1,80 @@ +import datetime +import importlib +import json +import logging +import typing +import uuid +from typing import Any + +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel +from starlette import status +import sentry_sdk +from sentry_sdk.integrations.asgi import SentryAsgiMiddleware +import seqlog + +app = FastAPI() + + +class JsonDateEncoder(json.JSONEncoder): + def default(self, o: Any) -> Any: + if isinstance(o, datetime.datetime): + return o.isoformat() + return super().default(o) + + +seqlog.log_to_seq( + server_url="http://log.abdus.dev:5341/", + api_key="ePewcJUxe8dST5EiC8Pw", + level=logging.INFO, + batch_size=10, + auto_flush_timeout=10, # seconds + override_root_logger=True, + json_encoder_class=JsonDateEncoder, # Optional; only specify this if you want to use a custom JSON encoder +) +sentry_sdk.init( + "https://ccb81025166c449d87e0646cede48448@o271257.ingest.sentry.io/6040020", + # Set traces_sample_rate to 1.0 to capture 100% + # of transactions for performance monitoring. + # We recommend adjusting this value in production. + traces_sample_rate=1.0, +) + +logger = logging.getLogger(__name__) + + +class HookResponse(BaseModel): + id: str + result: typing.Optional[dict] = None + error: typing.Optional[str] = None + + +@app.post("/{event}", response_model=HookResponse) +def receive_hook(event: str, payload: dict): + logger.info(f"Received hook request for {event=} with {payload=}") + hook_id = uuid.uuid4().hex + + try: + module = importlib.import_module(f"scripts.{event}") + func = getattr(module, "main") + except ModuleNotFoundError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No such handler") + + if not callable(func): + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Invalid script interface") + + try: + logger.info(f"Running hook handler {func=}") + result = func(payload) + return HookResponse(id=hook_id, result=result, error=None) + except Exception as e: + logger.error(f"Hook handler threw error", exc_info=True) + return HookResponse(id=hook_id, result=None, error=str(e)) + + +app = SentryAsgiMiddleware(app) + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app) diff --git a/hooksy/scripts/handle_aria2_completed.py b/hooksy/scripts/handle_aria2_completed.py new file mode 100644 index 0000000..eaa27f0 --- /dev/null +++ b/hooksy/scripts/handle_aria2_completed.py @@ -0,0 +1,21 @@ +import logging +import typing +from pathlib import Path + +logger = logging.getLogger(__name__) + + +Payload = typing.TypedDict( + "Payload", + { + "gid": str, + "num_files": int, + "path": str, + "files": list[typing.TypedDict("File", {"path": str, "size": int})], + }, +) + + +def main(payload: Payload): + logging.info("got {torrent}", torrent=payload) + path = Path(payload["path"]) diff --git a/hooksy/scripts/handle_downloads.py b/hooksy/scripts/handle_downloads.py new file mode 100644 index 0000000..56f252b --- /dev/null +++ b/hooksy/scripts/handle_downloads.py @@ -0,0 +1,26 @@ +import logging +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def main(payload: dict = None): + if not payload: + payload = {} + + links: list[dict] = payload.get("links", []) + if not links: + logger.info("No links, exiting") + return + + logger.info("Got %d links", len(links)) + + for it in links: + download_path = Path(it["download_path"]) + url: str = it.get("url", it.get("content_url")) + logger.info("Processing downloaded link {url} at {path}", path=download_path, url=url) + + if download_path.suffix.lower() in {".zip", ".rar"}: + logger.info("Got an imageset!") + + # raise Exception("oops!") diff --git a/hooksy/worker.py b/hooksy/worker.py new file mode 100644 index 0000000..e69de29 diff --git a/imgsheet.py b/imgsheet.py new file mode 100755 index 0000000..88883b2 --- /dev/null +++ b/imgsheet.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3.9 +import argparse +import dataclasses +import io +import itertools +import logging +import math +import statistics +import sys +import typing +from pathlib import Path + +from PIL import Image +from PIL import ImageDraw +from PIL import ImageFont + +T = typing.TypeVar("T") + +logger = logging.getLogger("imgsheet") + + +def chunk(it: typing.Iterable[T], size: int) -> typing.Iterable[typing.Iterable[T]]: + it = iter(it) + sentinel = () + return iter(lambda: tuple(itertools.islice(it, size)), sentinel) + + +@dataclasses.dataclass +class PositionedImage: + path: Path + image_size: typing.Tuple[int, int] + size: typing.Tuple[int, int] = dataclasses.field(default_factory=lambda: (0, 0)) + position: typing.Tuple[int, int] = dataclasses.field(default_factory=lambda: (0, 0)) + + @property + def top(self) -> int: + return self.position[1] + + @property + def left(self) -> int: + return self.position[0] + + @property + def bottom(self) -> int: + return self.height + self.top + + @property + def right(self) -> int: + return self.width + self.left + + @property + def width(self) -> int: + return self.size[0] + + @property + def height(self) -> int: + return self.size[1] + + def __post_init__(self): + self.size = self.image_size + + def scale(self, scale: float) -> None: + self.size = (math.ceil(self.width * scale), math.ceil(self.height * scale)) + + +class ImageSheet: + def __init__( + self, + width: int, + columns: int = 6, + font_path: typing.Optional[Path] = None, + label_format: typing.Optional[str] = None, + label_font_size: typing.Optional[int] = 10, + ): + self.width = width + self.padding = 20 + self.columns = columns + self.font_path = font_path + self.label_format = label_format + self.label_font_size = label_font_size + + def create(self, image_paths: typing.Iterable[Path]) -> io.BytesIO: + images = [] + for it in image_paths: + with Image.open(it) as img: + images.append(PositionedImage(it, img.size)) + + if self.font_path and self.font_path.is_file(): + font = ImageFont.truetype(str(self.font_path), self.label_font_size) + else: + font = None + + positioned = self._calculate_positions(images) + image_sheet = Image.new("RGB", (self.width, 0)) + for row_images in positioned: + height = row_images[0].top + row_images[0].size[1] + + extended_sheet = Image.new("RGB", (self.width, height + self.padding)) + extended_sheet.paste(image_sheet, (0, 0)) + image_sheet = extended_sheet + + draw = ImageDraw.Draw(image_sheet) + for it in row_images: + img = Image.open(it.path) + img.thumbnail(it.size, Image.LANCZOS) + image_sheet.paste(img, it.position) + + if font: + # write filename on image + text_x, text_y = it.left + 2, it.bottom + 2 + filesize = it.path.stat().st_size / 1_048_576 + + text = self._render_text( + width=it.image_size[0], + height=it.image_size[1], + size_mb=filesize, + filename=it.path.name, + ) + if text: + draw.text((text_x, text_y), text, (255, 255, 255), font=font, align="left", anchor="la") + + print(f"{it.path.name}\t{it.position}\t{it.size}") + + f = io.BytesIO() + image_sheet.save(f, format="JPEG") + return f + + def _render_text(self, width: int, height: int, size_mb: float, filename: str) -> typing.Optional[str]: + if not self.label_format: + return None + + return self.label_format.format(width=width, height=height, size_mb=size_mb, filename=filename) + + def _calculate_positions(self, images: list[PositionedImage]) -> list[list[PositionedImage]]: + cols = self.columns + width = self.width + padding = self.padding + + positioned = [] + + rows = list(chunk(images, cols)) + x, y = 0, 0 + for row, row_images in enumerate(rows): + row_images = list(row_images) + + usable_width = width - (len(row_images) - 1) * padding + max_height = max(image.height for image in row_images) + # check if there are enough images to fill the row + if len(row_images) < cols: + mean_height = statistics.median(prow[0].height for prow in positioned) + max_height = mean_height + + for img in row_images: + y_scaling = max_height / img.height + img.scale(y_scaling) + + if len(row_images) >= cols: + total_width = sum(image.width for image in row_images) + x_scaling = usable_width / total_width + max_height = math.ceil(max_height * x_scaling) + for img in row_images: + img.scale(x_scaling) + + is_last_row = row == len(rows) - 1 + x = 0 + for i, img in enumerate(row_images): + img.position = (x, y) + is_last_col = i == len(row_images) - 1 + x = x + img.width + (0 if is_last_col else padding) + y = y + max_height + (0 if is_last_row else padding) + + positioned.append(row_images) + + return positioned + + +def parse_args(argv: typing.List[str]) -> argparse.Namespace: + arger = argparse.ArgumentParser( + description="Create a sheet of images", formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + arger.add_argument("--width", "-w", type=int, default=2000, help="Width of the sheet") + arger.add_argument("--columns", "-c", type=int, default=4, help="Number of columns") + arger.add_argument("--output", "-o", type=Path, default=Path("image_sheet.jpg"), help="Output file") + arger.add_argument("--font", dest="font_path", type=Path, help="Label font to use") + arger.add_argument("--font-size", dest="font_size", default=10, type=int, help="Label font size") + arger.add_argument( + "--label", + dest="label_format", + default="{width}x{height} {size_mb:.1f}MB {filename}", + help="Label format. Available tokens: width, height, size_mb, filename", + ) + arger.add_argument("images", nargs="+", type=Path, help="Images or image directory to include in the sheet") + if len(argv) == 0: + arger.print_help() + sys.exit(1) + return arger.parse_args(argv) + + +def main(): + args = parse_args(sys.argv[1:]) + + images: list[Path] = args.images + if len(images) == 1 and images[0].expanduser().is_dir(): + images = sorted(it for it in images[0].resolve().glob("*.jpg")) + + sheet = ImageSheet( + width=args.width, + columns=args.columns, + font_path=args.font_path, + label_font_size=args.font_size, + label_format=args.label_format, + ) + + sheet_img = sheet.create(images) + with args.output.open("wb") as f: + f.write(sheet_img.getvalue()) + + +if __name__ == "__main__": + main() diff --git a/imgup.py b/imgup.py new file mode 100755 index 0000000..ccc75d8 --- /dev/null +++ b/imgup.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3.9 +import argparse +import concurrent.futures +import dataclasses +import functools +import logging +import os +import random +import re +import sys +import time +import typing +import uuid +from pathlib import Path +from typing import Optional + +import httpx + + +IMGBB_TOKEN = os.getenv('IMGBB_TOKEN', '391f0559cc209b11f37eda0084bb3281') +client = httpx.Client(base_url='https://api.imgbb.com/1/', params={'key': IMGBB_TOKEN}) + + +def upload_image(image_path: Path) -> str: + filename = f'i{image_path.suffix}' + with image_path.open('rb') as f: + res = client.post('/upload', files={'image': (filename, f)}) + res.raise_for_status() + data = res.json()['data'] + return data['url'] + + +def parse_args(argv: Optional[typing.Sequence[str]] = None) -> argparse.Namespace: + arger = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter) + arger.add_argument('image_paths', nargs='+', action='store', type=Path, help='Path to image to upload') + if not argv or len(argv) == 0: + arger.print_help() + sys.exit(1) + + return arger.parse_args(argv) + + +def main(): + args = parse_args(sys.argv[1:]) + image_paths = args.image_paths + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool: + results = list(pool.map(upload_image, image_paths)) + for url in results: + print(url) + + +if __name__ == '__main__': + main() diff --git a/input.ttf b/input.ttf new file mode 100644 index 0000000..4883b6c Binary files /dev/null and b/input.ttf differ diff --git a/iosevka.ttf b/iosevka.ttf new file mode 100644 index 0000000..bd82e49 Binary files /dev/null and b/iosevka.ttf differ diff --git a/jd2_hook.py b/jd2_hook.py new file mode 100644 index 0000000..c98c607 --- /dev/null +++ b/jd2_hook.py @@ -0,0 +1,35 @@ +import json +import logging as logger +import sys +from pathlib import Path + +import file_renamer + +logger.basicConfig( + format=f"%(asctime)s {logger.BASIC_FORMAT}", + level=logger.DEBUG, + handlers=[ + logger.FileHandler(Path(__file__).with_name("jd2_hook.log"), mode="a"), + logger.StreamHandler(), + ], +) + + +def main(): + try: + payload = json.loads(sys.argv[1]) + except (IndexError, ValueError): + logger.error("Expected JSON as first argument") + return 1 + + for link in payload["links"]: + path = link["download_path"] + f = Path(path) + logger.info("") + if f.is_file(): + file_renamer.fix_dates([f]) + + +if __name__ == "__main__": + logger.debug(f"Got called with: {sys.argv}") + exit(main()) diff --git a/jkup.py b/jkup.py new file mode 100755 index 0000000..294cbed --- /dev/null +++ b/jkup.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3.9 +import argparse +import concurrent.futures +import dataclasses +import re +import sys +import time +import typing +from pathlib import Path +from typing import Optional +import logging +from functools import cache + + +import httpx + + +@dataclasses.dataclass +class UploadResult: + image_url: str + thumbnail_url: str + +session = httpx.Client(timeout=20) + +@cache +def get_auth_token(): + logging.info('extracting auth token') + res = session.get("https://jerking.empornium.ph/?agree-consent", follow_redirects=True) + try: + auth_token = re.search(r'name="auth_token" value="([^"]+)"', res.text).group(1) + except AttributeError: + auth_token = re.search(r'auth_token = "([^"]+)"', res.text).group(1) + + logging.info(f'got {auth_token=}') + + return auth_token + +def upload_image(image_path: Path) -> UploadResult: + auth_token = get_auth_token() + + logging.info(f'uploading {image_path.name}') + + with image_path.open("rb") as f: + res = session.post( + "https://jerking.empornium.ph/json", + headers={"accept": "application/json"}, + data={ + "thumb_width": "400", + "thumb_height": "400", + "thumb_crop": "false", + "medium_width": "600", + "medium_crop": "false", + "type": "file", + "action": "upload", + "timestamp": str(round(time.time() * 1000)), + "auth_token": auth_token, + "nsfw": "1", + }, + files={ + "source": f, + }, + ) + + data = res.json() + logging.info(f'got response={data}') + image_url = data["image"]["url"] + thumbnail_url = data["image"]["display_url"] + + logging.info(f'uploaded {image_path.name}. url={image_url}') + + return UploadResult(image_url, thumbnail_url) + + +def parse_args(argv: Optional[typing.Sequence[str]] = None) -> argparse.Namespace: + arger = argparse.ArgumentParser() + arger.add_argument('image_paths', nargs='+', action='store', type=Path, help='Path to image to upload') + if not argv or len(argv) == 0: + arger.print_help() + sys.exit(1) + + return arger.parse_args(argv) + + +def main(): + args = parse_args(sys.argv[1:]) + image_paths = args.image_paths + + get_auth_token() + + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(upload_image, image_paths)) + for it in results: + print(it.image_url) + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + main() diff --git a/kube.py b/kube.py new file mode 100644 index 0000000..a42e7cf --- /dev/null +++ b/kube.py @@ -0,0 +1,122 @@ +import base64 +import logging + +import kubernetes.client +import urllib3.response +import yaml +from kubernetes.client import ApiException +from kubernetes.client.rest import RESTResponse + +_result_deleted = { + 'api_version': 'v1', + 'code': None, + 'details': None, + 'kind': 'Namespace', + 'message': None, + 'metadata': { + '_continue': None, + 'remaining_item_count': None, + 'resource_version': '4424', + 'self_link': '/api/v1/namespaces/mydep3', + }, + 'reason': None, + 'status': "{'phase': 'Terminating'}", +} +_result_not_found = ApiException( + RESTResponse( + urllib3.response.HTTPResponse( + body=b'{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \\"mydep3\\" not found","reason":"NotFound","details":{"name":"mydep3","kind":"namespaces"},"code":404}', + status=404, + ) + ) +) + +kubeconfig = yaml.load( + ''' +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUREekNDQWZlZ0F3SUJBZ0lVVjhlN3pIa0Q5SitzcnJuN1Ywb0Zka2ZTaFQ0d0RRWUpLb1pJaHZjTkFRRUwKQlFBd0Z6RVZNQk1HQTFVRUF3d01NVEF1TVRVeUxqRTRNeTR4TUI0WERUSXlNREl3TnpBME1qY3dNVm9YRFRNeQpNREl3TlRBME1qY3dNVm93RnpFVk1CTUdBMVVFQXd3TU1UQXVNVFV5TGpFNE15NHhNSUlCSWpBTkJna3Foa2lHCjl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUE0UDNFSkozUXlxbzF4c2ducTh1dXY5Yk9yVks1QmQzOTBwWGcKUVlpMTBnZmd5UzBrWnZ6RFRFYXYzTmNKRnk0N2puYmdkVFF3em1YcHo5SFpKcFRZQUM0M0M1T2hEcHNFZzZEQQo2cVZYcmcrVjhObUlXelRMRk5wT2o2eUFvNnN0V3VqSkpyNDBkR0s1ZE5zT3QwMk9uMElOTkV1MHMraU5uc21tCkhNSFlhc3FXL0YvekFyR0psTTkwUjNtL05wZHdhMWxXcCtKY0lHYnQ3TmRyY21SV05HL3RyU05XQVZZTStrenQKVm02YUIweWZZdTRiUHB3bEtCVU43SjhqdCtJWWQwSmdiMmliUXd6V2VnTVdIU050b0ZtVEN1WVNqYTVTKzVheQpMeGE1R3BaY3B3UUo1dGcvRjFZNTU5QTZkS1gycjFtRzBzS2NIZDk0ZE1IT1FRYVRqd0lEQVFBQm8xTXdVVEFkCkJnTlZIUTRFRmdRVVM2T3l6MkZyNlBVR01Hdk9TSFhVckcvVitBb3dId1lEVlIwakJCZ3dGb0FVUzZPeXoyRnIKNlBVR01Hdk9TSFhVckcvVitBb3dEd1lEVlIwVEFRSC9CQVV3QXdFQi96QU5CZ2txaGtpRzl3MEJBUXNGQUFPQwpBUUVBU0d2bnY3NHNJb2hvcC9DWWpJSHpLMTJGWS9PUmZqUUNNR0tMRGhjN3dEMENXMFhjaHRaQmsrc3VLZVFuCnNJTTlub0k2TStJZGxWM2xCUlBpU1p6ZWpPV2FiZmpHak5MOWw1RGRyVjByNnBtN3p0ckNQU0lXdVY5N29LMTMKVnIwNlVpM2l3ZTM5M00rTlZmYjlxd2wzaGlMUGZ1c0hKM3ZJRGdKWjMyeTI0S2VMekc4QWJTcFJXYjVjWUcvRApQSmpaZGh4bzVjanF0SnZONXBCZ1JlYWdoZTRKdVZKRkh0VWgyYTQrR1F1QjQyVGNIZnFnaEF5TmtkblpIVWRjCmRjOWordFM2YW9xZTRHWEY2bkplcGhSalNaRXEwb1J4MStnR2tvUENkdkdoTGV2ZDRvaUV6eWprYzM1eWNYanQKOTNGOWQzaENxckhQRDBPaW4vQkN0Rk0vclE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + server: https://142.132.167.243:16443 + name: microk8s-cluster +contexts: +- context: + cluster: microk8s-cluster + user: admin + name: microk8s +current-context: microk8s +kind: Config +preferences: {} +users: +- name: admin + user: + token: VU96c211SXRDNlg0WUZOTUh0dHpYd2luNy84Ulh0Y2Nka09wWGlmOTQ1ST0K +''' +) + +logger = logging.getLogger(__name__) + + +class KubernetesService: + def __init__(self, client: kubernetes.client.CoreV1Api): + self.client = client + + @classmethod + def new(cls, kubeconfig: dict): + api_client = kubernetes.config.new_client_from_config_dict(kubeconfig) + v1_client = kubernetes.client.CoreV1Api(api_client) + return cls(v1_client) + + def delete_namespace(self, namespace_name: str) -> None: + # just in case + assert ( + namespace_name + not in [ + 'kube-system', + 'kube-public', + 'kube-node-lease', + 'default', + ] + or not namespace_name.startswith('kube-') + ), 'protected namespaces cannot be deleted' + + try: + _ = self.client.delete_namespace(namespace_name) + except ApiException as e: + if e.status != 404: # nocover + raise + logger.error( + 'Namespace not found', + extra={ + 'namespace_name': namespace_name, + 'response_body': e.body, + }, + ) + + +def main(): + kubernetes.config.load_kube_config_from_dict(kubeconfig) + client = kubernetes.client.CoreV1Api() + env = { + 'AWS_ACCESS_KEY': 'aws', + 'AWS_ACCESS_SECRET_KEY': 'secret', + } + secret = client.read_namespaced_secret('env', 'default') + secret_names = list(secret.data.keys()) + # ['AWS_ACCESS_KEY', 'AWS_ACCESS_SECRET_KEY'] + secret = kubernetes.client.V1Secret( + api_version='v1', + metadata={'name': 'env'}, + data={k: base64.b64encode(v.encode()).decode() for k, v in env.items()}, + immutable=True, + kind='Secret', + type='Opaque', + ) + res = client.create_namespaced_secret('default', secret) + print(res) + + +# + +if __name__ == '__main__': + main() diff --git a/kube_test.py b/kube_test.py new file mode 100644 index 0000000..7b4d3e9 --- /dev/null +++ b/kube_test.py @@ -0,0 +1,54 @@ +from unittest import mock + +import kubernetes.client.api.core_v1_api +import pytest + +import urllib3.response +from kubernetes.client import ApiException +from kubernetes.client.rest import RESTResponse +from kube import KubernetesService + +# fmt: off +_result_deleted = {'api_version': 'v1', 'code': None, 'details': None, 'kind': 'Namespace', 'message': None, 'metadata': {'_continue': None, 'remaining_item_count': None, 'resource_version': '4424', 'self_link': '/api/v1/namespaces/mydep3',}, 'reason': None, 'status': "{'phase': 'Terminating'}",} +_result_not_found = ApiException(status=404, http_resp=RESTResponse(urllib3.response.HTTPResponse(body=b'{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \\"mydep3\\" not found","reason":"NotFound","details":{"name":"mydep3","kind":"namespaces"},"code":404}'))) +# fmt: on + + +@pytest.fixture() +def mock_k8s(): + yield mock.MagicMock(autospec=kubernetes.client.api.core_v1_api.CoreV1Api) + + +def test_delete_forbidden_namespace(mock_k8s): + service = KubernetesService(mock_k8s) + + with pytest.raises(AssertionError): + service.delete_namespace('default') + + +def test_delete_namespace(mock_k8s): + mock_k8s.delete_namespace.return_value = _result_deleted + service = KubernetesService(mock_k8s) + + service.delete_namespace('app-uuid') + + mock_k8s.delete_namespace.assert_called_once_with('app-uuid') + + +def test_deleting_nonexistent_namespace_is_ok(mock_k8s): + mock_k8s.delete_namespace.side_effect = _result_not_found + service = KubernetesService(mock_k8s) + + with mock.patch('logging.Logger.error') as m_log: + service.delete_namespace('app-uuid') + + m_log.assert_called() + + +def test_any_other_error_bubbles_up(mock_k8s): + mock_k8s.delete_namespace.side_effect = ApiException(status=500, http_resp=urllib3.response.HTTPResponse(body=b'')) + + service = KubernetesService(mock_k8s) + + with pytest.raises(ApiException): + service.delete_namespace('app-uuid') diff --git a/launchd_plist.py b/launchd_plist.py new file mode 100755 index 0000000..e08a3f6 --- /dev/null +++ b/launchd_plist.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +import argparse +import logging +import plistlib +import sys +import typing +from pathlib import Path + + +def parse_args(argv: list[str]) -> typing.Tuple[argparse.Namespace, list[str]]: + def parse_env(env: str) -> typing.Tuple[str, str]: + return env.split("=", maxsplit=1) + + def parse_path(path: str) -> Path: + return Path(path).expanduser().resolve() + + arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + arger.add_argument( + "-e", "--env", dest="env", metavar="ENVVAR", type=parse_env, nargs="+", help="Environment variables as K=V list" + ) + arger.add_argument("-w", "--cwd", dest="cwd", default=Path.cwd(), type=parse_path, help="Working directory") + arger.add_argument("-l", "--label", required=True, help="Label") + arger.add_argument( + "--install", + action="store_true", + help="Install the plist to ~/Library/LaunchAgents/", + ) + + if not argv: + arger.print_help() + sys.exit(1) + + args, unknown = arger.parse_known_args() + if unknown and unknown[0] == "--": + unknown = unknown[1:] + + if not unknown: + arger.print_help() + sys.exit(1) + + return args, unknown + + +def generate_plist(label: str, args: list[str], env: dict, cwd: Path) -> str: + doc = { + "Label": label, + "ProgramArguments": args, + "EnvironmentVariables": env, + "WorkingDirectory": str(cwd), + "RunAtLoad": True, + "StandardOutPath": str(cwd / f"{label}.stdout.log"), + "StandardErrorPath": str(cwd / f"{label}.stderr.log"), + } + return plistlib.dumps( + doc, + fmt=plistlib.FMT_XML, + sort_keys=True, + ).decode() + + +def main(): + args, unknown = parse_args(sys.argv[1:]) + print(args, unknown) + plist_doc = generate_plist( + label=args.label, + args=unknown, + env=dict(args.env), + cwd=args.cwd, + ) + print(plist_doc) + + if args.install: + install_dir: Path = Path("~/Library/LaunchAgents/").expanduser() + logging.info(f"Installing plist to {install_dir}") + + install_path = install_dir / f"{args.label}.plist" + install_path.write_text(plist_doc) + + logging.info(f'Run "launchctl load -w {install_path}" to start the service') + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() diff --git a/macos_wallpaper.py b/macos_wallpaper.py new file mode 100644 index 0000000..72c9925 --- /dev/null +++ b/macos_wallpaper.py @@ -0,0 +1,72 @@ +import contextlib +import dataclasses +import plistlib +import sqlite3 +import subprocess +from pathlib import Path +from typing import Optional + +DB_PATH = Path("~/Library/Application Support/Dock/desktoppicture.db").expanduser() + + +@dataclasses.dataclass +class SpaceInfo: + uuid: str + image_path: Optional[Path] + data_id: Optional[Path] + + +@contextlib.contextmanager +def get_connection() -> sqlite3.Connection: + con = sqlite3.connect(str(DB_PATH)) + con.row_factory = sqlite3.Row + yield con + con.close() + + +def get_current_wallpapers() -> list[SpaceInfo]: + sql = """ + select distinct s.space_uuid, pr.key, d.rowid, d.value from preferences pr + left join payload d on d.rowid = pr.data_id + left join pictures p on p.rowid = pr.picture_id + inner join spaces s on s.rowid = p.space_id + where key = 1 + + order by space_uuid, key + """ + spaces = list_spaces() + + with get_connection() as con: + cur = con.execute(sql) + + space_props = {} + for i, row in enumerate(cur): + space_uuid = row["space_uuid"] + data_id = row["rowid"] + image_path = row["value"] + + if image_path: + image_path = Path(image_path).expanduser() + space_props[space_uuid] = SpaceInfo(uuid=space_uuid, image_path=image_path, data_id=data_id) + return [space_props.get(id, SpaceInfo(uuid="", image_path=None, data_id=None)) for id in spaces] + + +def read_plist(plist_path): + with plist_path.open("rb") as f: + return plistlib.load(f) + + +def list_spaces(): + spaces_plist_path = Path("~/Library/Preferences/com.apple.spaces.plist").expanduser() + data = read_plist(spaces_plist_path) + spaces = data["SpacesDisplayConfiguration"]["Management Data"]["Monitors"][0]["Spaces"] + return [s["uuid"] for s in spaces] + + +def set_wallpaper(image_path: Path): + script = f'tell application "Finder" to set desktop picture to "{image_path.expanduser().resolve()}" as POSIX file' + subprocess.run(["osascript", "-e", script], check=True) + + +if __name__ == "__main__": + pass diff --git a/make_kubeconfig.sh b/make_kubeconfig.sh new file mode 100755 index 0000000..85f487c --- /dev/null +++ b/make_kubeconfig.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +cluster_name=sandbox +namespace=default +service_account=demo-readonly +server=http://0.0.0.0:58253 + +secret_name=$(kubectl --namespace $namespace get serviceaccount $service_account -o jsonpath='{.secrets[0].name}') +ca=$(kubectl --namespace $namespace get secret/$secret_name -o jsonpath='{.data.ca\.crt}') +token=$(kubectl --namespace $namespace get secret/$secret_name -o jsonpath='{.data.token}' | base64 --decode) + +echo "apiVersion: v1 +kind: Config +clusters: + - name: ${cluster_name} + cluster: + certificate-authority-data: ${ca} + server: ${server} +contexts: + - name: ${service_account}@${cluster_name} + context: + cluster: ${cluster_name} + namespace: ${namespace} + user: ${service_account} +users: + - name: ${service_account} + user: + token: ${token} +current-context: ${service_account}@${cluster_name} +" diff --git a/personio_attendance.py b/personio_attendance.py new file mode 100644 index 0000000..6356674 --- /dev/null +++ b/personio_attendance.py @@ -0,0 +1,71 @@ +import datetime +from pathlib import Path +import sys +import time + +from playwright.sync_api import BrowserContext, sync_playwright + + +class creds: + employee_id: str = "9374380" + email: str = "abduessamet.kocak@refurbed.com" + password: str = "^6LMq$LX&2#c4y" + + +def main(): + try: + day = datetime.date.today().fromisoformat(sys.argv[1]).isoformat() + except: + day = "today" + + with sync_playwright() as playwright: + data_dir = Path("/tmp/personio") + data_dir.mkdir(exist_ok=True, parents=True) + context = playwright.chromium.launch_persistent_context(headless=False, user_data_dir=data_dir) + with context: + fill_attendance( + context=context, + day=day, + work=("08:00", "17:00"), + pause=("12:00", "13:00"), + ) + + +def fill_attendance(context: BrowserContext, day: str, work: tuple[str, str], pause=[str, str]): + # Open new page + page = context.new_page() + + url = f"https://refurbed.personio.de/attendance/employee/{creds.employee_id}/" + page.goto(url) + page.wait_for_load_state() + + if "/login/index" in page.url: + page.locator('[placeholder="Email"]').fill(creds.email) + page.locator('[placeholder="Password"]').fill(creds.password) + page.locator('button:has-text("Login")').click() + + card_selector = f'[data-test-id="day_{day}"] button' + if day == "today": + card_selector = '[data-test-id="today-cell"] button' + + while True: + page.locator(card_selector).click() + if page.locator('[data-test-id="day-entry-dialog"]').is_visible(): + break + time.sleep(0.05) + + work_start, work_end = work + page.locator('[data-test-id="timerange-start"]').first.fill(work_start) + page.locator('[data-test-id="work-entry"] [data-test-id="timerange-end"]').fill(work_end) + + pause_start, pause_end = pause + page.locator('[data-test-id="break-entry"] [data-test-id="timerange-start"]').fill(pause_start) + page.locator('[data-test-id="break-entry"] [data-test-id="timerange-end"]').fill(pause_end) + + page.locator('[data-test-id="day-entry-save"]').click() + # wait until network req is sent + time.sleep(1) + + +if __name__ == "__main__": + main() diff --git a/plab_search.py b/plab_search.py new file mode 100644 index 0000000..9d182fb --- /dev/null +++ b/plab_search.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import argparse +import dataclasses +import datetime +import json +import typing +from pathlib import Path +from pprint import pprint +from urllib.parse import urljoin + +import bs4 +import httpx + + +@dataclasses.dataclass() +class SearchResult: + posted_at: datetime.datetime + title: str + size_mb: float + url: str + + +class Auth(typing.Protocol): + def authenticate(self) -> dict[str, str]: + ... + + +@dataclasses.dataclass() +class CookieFileAuth: + file_path: Path + + def authenticate(self) -> dict[str, str]: + with self.file_path.open() as f: + return json.load(f) + + +@dataclasses.dataclass() +class CredentialAuth: + username: str + password: str + + def authenticate(self) -> dict[str, str]: + client = httpx.Client(base_url="https://pornolab.net/") + res = client.post( + "/forum/login.php", + data={ + "login_username": self.username, + "login_password": self.password, + "login": "Вход", + }, + follow_redirects=True, + ) + res.raise_for_status() + soup = bs4.BeautifulSoup(res.text, "html.parser") + username_el = soup.select_one(".topmenu .med") + if self.username not in username_el.text.strip(): + raise ValueError("failed to login") + return dict(client.cookies) + + +@dataclasses.dataclass() +class PersistedCookieAuth: + cookie_path: Path + auth: Auth + + def authenticate(self) -> dict[str, str]: + cookie_auth = CookieFileAuth(self.cookie_path) + try: + return cookie_auth.authenticate() + except (ValueError, FileNotFoundError): + pass + + cookies = self.auth.authenticate() + with self.cookie_path.open("w") as f: + json.dump(cookies, f) + return cookies + + +class PlabAuthMiddleware(httpx.Auth): + def auth_flow(self, request: httpx.Request) -> typing.Generator[httpx.Request, httpx.Response, None]: + # if no cookies in req + # fill cookie + # check response + # reauth + # fill cookie + + res = yield request + return super().auth_flow(request) + + +class Plab: + def __init__(self, auth: Auth): + self.auth = auth + self.client = httpx.Client(base_url="https://pornolab.net/") + + def search(self, term: str) -> list[SearchResult]: + self._ensure_session() + res = self.client.post( + "/forum/tracker.php", + data={ + "max": "1", + "to": "1", + "nm": f'{term}', + }, + follow_redirects=True, + ) + res.raise_for_status() + + if 'Введите ваше имя и пароль' in res.text: + raise + + soup = bs4.BeautifulSoup(res.text, "html.parser") + rows = soup.select("#tor-tbl tbody tr") + items = [] + for row in rows: + title_link = row.select_one(f'[href*="viewtopic.php?t="]') + if not title_link: + continue + size_mb = ( + int(row.select_one('[href*="dl.php?t="]').find_previous_sibling(name='u').text.strip()) / 1_048_576 + ) + date = datetime.datetime.fromtimestamp(int(row.select_one('[title="Добавлен"] u').text.strip())) + items.append( + SearchResult( + url=urljoin("https://pornolab.net/forum/", title_link["href"]), + title=title_link.text.strip(), + size_mb=size_mb, + posted_at=date, + ) + ) + return items + + def _ensure_session(self): + if self.client.cookies: + return + self.client.cookies = self.auth.authenticate() + + +def dump_for_alfred(items: list[SearchResult]): + alfreds = [] + for it in items: + alfreds.append( + { + "title": it.title, + "subtitle": f"{it.size_mb:.1f}MB -- {it.posted_at.isoformat()}", + "arg": it.url, + "text": { + "copy": it.url, + "largetype": f"{it.title}\nPosted at: {it.posted_at.isoformat()}\nSize: {it.size_mb:.1f}MB", + }, + "match": f"{it.title} {it.posted_at.isoformat()}", + } + ) + print(json.dumps({"items": alfreds}, indent=2)) + + +def parse_args(): + arger = argparse.ArgumentParser() + arger.add_argument('term', help='Term to search for') + arger.add_argument('--alfred', action='store_true', help='Dump for Alfred') + return arger.parse_args() + + +def main(): + args = parse_args() + + p = Plab( + auth=PersistedCookieAuth( + cookie_path=Path(__file__).parent / "plab.json", + auth=CredentialAuth( + username="zzzp", + password="3c45HzLMgSB7M5hJ", + ), + ) + ) + + results = p.search(args.term) + + if args.alfred: + dump_for_alfred(results) + else: + for it in results: + pprint(it) + + +if __name__ == "__main__": + main() diff --git a/playwright_playground.py b/playwright_playground.py new file mode 100644 index 0000000..657c583 --- /dev/null +++ b/playwright_playground.py @@ -0,0 +1,55 @@ +from os import PathLike +from pathlib import Path +import re +from playwright.sync_api import sync_playwright, Browser +from contextlib import contextmanager + + +@contextmanager +def run_browser(data_dir: PathLike, extension_path: PathLike) -> Browser: + with sync_playwright() as pw: + with pw.chromium.launch_persistent_context( + user_data_dir=data_dir, + headless=False, + args=[ + f"--disable-extensions-except={extension_path}", + f"--load-extension={extension_path}", + ], + ) as browser: + yield browser + + +def find_sales_performance(browser: Browser, url: str) -> dict: + with browser.new_page() as page: + page.goto(url) + text = page.wait_for_selector("#h10-product-score").inner_text() + + amazon_id = re.search(r'/dp/([^?]+)', url).group(1) + score = float(re.search(r"(\d+\.?\d*)", text).group(1)) + try: + sales = int(re.search(r"Sales\s*(\d+[,.]?\d*)", text, re.MULTILINE).group(1).replace(',', "")) + except: + sales = -1 + + return { + 'amazon_id': amazon_id, + 'sales': sales, + 'score': score, + } + + +extension_path = Path(r"~/Desktop/amazon").expanduser().resolve() +data_dir = Path(r'~/Desktop/amazondemo').expanduser() + +def main(): + urls = [ + "https://www.amazon.co.uk/dp/B07B9G7V3P?psc=1", + ] + with run_browser(data_dir=data_dir, extension_path=extension_path) as browser: + for url in urls: + perf = find_sales_performance(browser=browser, url=url) + print(perf) + + +if __name__ == "__main__": + main() diff --git a/realdebrid.py b/realdebrid.py new file mode 100755 index 0000000..31333d4 --- /dev/null +++ b/realdebrid.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +import httpx +import subprocess +import logging +import typer +from pathlib import Path + +logging.basicConfig( + level=logging.DEBUG, + format="%(levelname)s %(asctime)s: %(message)s", + datefmt=logging.Formatter.default_time_format, +) +logging.getLogger("httpx").setLevel(logging.WARNING) + +# get api token from https://real-debrid.com/apitoken +key = "P4SUGS4VSXLDIRXSOAB46DXMU2HVFJTOWYCACNULX4V7TOZRW2BA" + +cli = typer.Typer() + +http = httpx.Client( + base_url="https://api.real-debrid.com/rest/1.0/", + headers={"Authorization": f"Bearer {key}"}, +) + + +def get_download_url(url: str) -> str: + res = http.post("/unrestrict/link", data={"link": url}) + res.raise_for_status() + return res.json()["download"] + + +def download_with_aria2( + url: str, cwd: Path = Path.cwd(), extra_args: list[str] = None +) -> None: + if not extra_args: + extra_args = [] + args = ["aria2c", "-x", "5", url, *extra_args] + logging.debug("calling aria2 with args=%r", args) + subprocess.run(args, text=True, cwd=str(cwd.resolve())) + + +@cli.command( + context_settings={ + "ignore_unknown_options": True, + "allow_extra_args": True, + } +) +def main( + ctx: typer.Context, + url: str, + cwd: Path = typer.Option(Path("."), "-o", "--out", "--cwd", writable=True), +): + try: + logging.debug("generating premium link") + download_url = get_download_url(url) + except (httpx.HTTPStatusError, KeyError): + logging.error("cannot generate premium link") + raise typer.Exit(1) + + logging.info("got url=%s", download_url) + + logging.debug("passing url to aria2") + try: + download_with_aria2(download_url, cwd=cwd, extra_args=ctx.args) + except KeyboardInterrupt: + logging.error("download interrupted. exiting") + raise typer.Exit(2) + + +if __name__ == "__main__": + cli() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d62553e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +beautifulsoup4==4.11.2 +boto3==1.19.1 +fastapi==0.86.0 +httpx==0.23.3 +pytest==6.2.5 +typer==0.6.1 diff --git a/rsync_it.py b/rsync_it.py new file mode 100755 index 0000000..c6318f4 --- /dev/null +++ b/rsync_it.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +import logging +import subprocess +import sys +import time +from pathlib import Path + + +def find_file_arg(argv: list[str]) -> Path: + for it in argv: + if it.startswith('-'): + continue + if Path(it).is_file(): + return Path(it) + + +def main(): + argv = sys.argv[1:] + file_path = find_file_arg(argv) + if not file_path: + logging.error('file argument not passed') + return + + size = file_path.stat().st_size + while True: + args = ['rsync', '--recursive', '--human-readable', '--progress', '--partial', *argv] + logging.debug(f'running rsync with {args=}') + p = subprocess.run(args) + p.check_returncode() + + logging.debug('will check the file size in 20s') + time.sleep(20) + subprocess.run(['touch', str(file_path)]) + + new_size = file_path.stat().st_size + + if new_size == size: + logging.info('file size has not changed, stopping') + break + logging.info(f'file size has changed {new_size - size} bytes, re-syncing') + size = new_size + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + main() diff --git a/tekton_api.py b/tekton_api.py new file mode 100644 index 0000000..97e1fa5 --- /dev/null +++ b/tekton_api.py @@ -0,0 +1,190 @@ +import functools +import hashlib +import json +import textwrap +import typing +import uuid +from pathlib import Path + +import kubernetes.config +import yaml + + +def make_client() -> kubernetes.client.CoreV1Api: + kubernetes.config.load_kube_config('/Users/abdus/Desktop/kubeconfig-kube.yml') + return kubernetes.client.CoreV1Api() + + +class Kubernetes: + def __init__(self, client: kubernetes.client.CoreV1Api, namespace='default'): + self.client = client + self.custom_objects = kubernetes.client.CustomObjectsApi(self.client.api_client) + self.namespace = namespace + + @classmethod + def from_kubeconfig(cls, kubeconfig_path: str, namespace: str) -> 'Kubernetes': + kubernetes.config.load_kube_config(kubeconfig_path) + return cls(kubernetes.client.CoreV1Api(), namespace=namespace) + + def read_logs(self, pod_name: str, container_name: typing.Optional[str] = None, timestamps: bool = False) -> str: + return self.client.read_namespaced_pod_log( + pod_name, + self.namespace, + container=container_name, + timestamps=timestamps, + pretty=True, + ) + + def get_tekton_object(self, plural: str, name: str = None): + getter = functools.partial( + self.custom_objects.get_namespaced_custom_object, + group='tekton.dev', + version='v1beta1', + namespace=self.namespace, + ) + return getter(plural=plural, name=name) + + def list_tekton_object(self, plural: str): + getter = functools.partial( + self.custom_objects.list_namespaced_custom_object, + group='tekton.dev', + version='v1beta1', + namespace=self.namespace, + ) + return getter(plural=plural)['items'] + + def create_from_yaml(self, yaml_text: str) -> list[dict]: + specs = list(yaml.safe_load_all(yaml_text)) + return kubernetes.utils.create_from_yaml( + kubernetes.client.CustomObjectsApi(self.client.api_client), + yaml_objects=specs, + namespace=self.namespace, + ) + + +def read_logs(): + task_hello_world = k8s.get_tekton_object(plural='tasks', name='echo-hello-world') + taskruns = k8s.list_tekton_object(plural='taskruns') + + pod_name = taskruns['items'][0]['status']['podName'] + container_name = taskruns['items'][0]['status']['containerName'] + logs = k8s.read_logs(pod_name, container_name) + print(logs) + + +k8s = Kubernetes.from_kubeconfig('/Users/abdus/Desktop/kubeconfig.yaml', namespace='api-server') + + +def get_pipelineru_status(): + result = k8s.get_tekton_object('pipelineruns', name='pr-2rr6t') + status = result['status']['conditions'][0]['status'] + reason = result['status']['conditions'][0]['reason'] + failed_taskruns = [ + name for name, it in result['status']['taskRuns'].items() if it['status']['conditions'][0]['status'] == 'False' + ] + + logs = [] + for it in result['status']['taskRuns'].values(): + task_name = it['pipelineTaskName'] + task_status = it['status']['conditions'][0]['reason'] + pod_name = it['status']['podName'] + + if task_status != 'Failed': + continue + + # fetch all logs for all steps to get a better picture of the error + # (regardless of whether it is successful or not) + for i, step in enumerate(it['status']['steps'], start=1): + container_name = step['container'] + header = f'# {task_name}, step {i}: {step["name"]}' + logs.append('#' * len(header)) + logs.append(header) + logs.append('#' * len(header)) + step_logs = k8s.read_logs(pod_name, container_name, timestamps=True) + logs.append(step_logs) + + log_text = '\n'.join(logs) + return { + 'pipelinerun_name': result['metadata']['name'], + 'namespace': result['metadata']['namespace'], + 'status': status, + 'reason': reason, + 'failed_taskruns': failed_taskruns, + 'logs': log_text, + } + + +def create_taskrun(): + k = kubernetes.client.CustomObjectsApi() + res = k.create_namespaced_custom_object( + group='tekton.dev', + version='v1beta1', + namespace='default', + plural='taskruns', + body={ + 'apiVersion': 'tekton.dev/v1beta1', + 'kind': 'TaskRun', + 'metadata': {'name': 'testing1234' + uuid.uuid4().hex}, + 'spec': { + 'taskRef': {'name': 'task-with-json'}, + 'params': [ + { + 'name': 'json_value', + 'value': '{"foo": "bar", "nested": {"a": "1"}}', + } + ], + }, + }, + ) + print(res) + + +def migrate_pipelines(): + k = kubernetes.client.CustomObjectsApi() + + pipeline_yaml = Path('sample_pipeline.yaml').read_text() + parsed_pipeline = yaml.safe_load(pipeline_yaml) + + params = dict(group='tekton.dev', version='v1beta1', namespace='default', plural='pipelines') + k.delete_namespaced_custom_object( + **params, + name=parsed_pipeline['metadata']['name'], + ) + res = k.create_namespaced_custom_object( + **params, + body=parsed_pipeline, + ) + return parsed_pipeline + + +def trigger_pipeline(): + k = kubernetes.client.CustomObjectsApi() + pipeline = migrate_pipelines() + pipeline_name = pipeline['metadata']['name'] + + payload = { + 'image': 'nginx', + 'json_value': {'nested': {'nested2': {'a': 1}}}, + } + + res = k.create_namespaced_custom_object( + group='tekton.dev', + version='v1beta1', + namespace='default', + plural='pipelineruns', + body={ + 'apiVersion': 'tekton.dev/v1beta1', + 'kind': 'PipelineRun', + 'metadata': {'name': 'deploy-app-' + uuid.uuid4().hex}, + 'spec': { + 'pipelineRef': {'name': pipeline_name}, + 'params': [ + {'name': k, 'value': json.dumps(v) if not isinstance(v, str) else v} for k, v in payload.items() + ], + }, + }, + ) + + +if __name__ == '__main__': + main() diff --git a/torrentify.py b/torrentify.py new file mode 100755 index 0000000..0bc8ef3 --- /dev/null +++ b/torrentify.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3.9 +import argparse +import json +import logging +import os +import re +import shutil +import sys +import time +from turtle import back +import typing +from pathlib import Path +from typing import Optional + +import ffmpeg +from ffmpeg_strip import clean_video + +logging.basicConfig(level=logging.INFO, format=f"%(asctime)s {logging.BASIC_FORMAT}") + + +def force_import(module: str): + import importlib + import subprocess + import sys + + try: + return importlib.import_module(module) + except ModuleNotFoundError: + subprocess.run([sys.executable, "-m", "pip", "install", module]) + importlib.invalidate_caches() + return importlib.import_module(module) + + +try: + import torf +except ImportError: + torf = force_import("torf") + +try: + import httpx +except ImportError: + httpx = force_import("httpx") + +try: + import inquirer +except ImportError: + inquirer = force_import("inquirer") + +TORRENT_TRACKER_URL = os.getenv( + "TORRENT_TRACKER_URL", "http://tracker.empornium.sx:2710/tegqucis10uanp672qh6nhn393xlkncs/announce" +) +TORRENT_CREATED_BY = os.getenv("TORRENT_CREATED_BY", "zzzp") +TORRENT_DIR = Path(os.getenv("TORRENT_DIR", "/mnt/box/files/_torrents/_new2/")) + +VIDEO_EXTENSIONS = {".mp4", ".mpg", ".mkv"} + + +def make_torrent(source_dir: Path, tracker_url: str = TORRENT_TRACKER_URL) -> Path: + torrent_path = source_dir / f"{source_dir.name}.torrent" + if torrent_path.is_file(): + logging.info("Torrent file is already created") + return torrent_path + + t = torf.Torrent( + path=str(source_dir.resolve()), + name=source_dir.resolve().name, + trackers=[tracker_url], + private=True, + created_by=TORRENT_CREATED_BY, + exclude_globs=["*.txt", "post.txt", "*.torrent", "*.gif"], + ) + t.generate() + t.write(torrent_path, overwrite=True) + + return torrent_path + + +def human_size(size: int) -> str: + suffix = "B" + for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: + if abs(size) < 1024.0: + return "%3.2f%s%s" % (size, unit, suffix) + size /= 1024.0 + return "%.2f%s%s" % (size, "Yi", suffix) + + +def find_video(root_dir: Path) -> Optional[Path]: + for ext in VIDEO_EXTENSIONS: + for f in root_dir.rglob(f"*{ext}"): + return f + return None + + +def pick_name(video_path: Path) -> str: + try: + if parse_filename(video_path.stem): + return video_path.stem + except ValueError: + pass + + candidates = set() + + parent = video_path.parent + while parent.exists(): + try: + if parse_filename(parent.name): + candidates.add(parent.name) + except ValueError: + pass + if parent.parent == parent: + break + parent = parent.parent + + if len(candidates) == 1: + return candidates.pop() + elif len(candidates) > 1: + questions = [ + inquirer.List( + "best_name", + message="Pick the best filename", + choices=[*candidates, ""], + ), + ] + + answers = inquirer.prompt(questions) or {} + best_name = answers.get("best_name") + if best_name != "": + return best_name + + while True: + try: + answers = inquirer.prompt([inquirer.Editor("best_name", message="New name", default=video_path.stem)]) + if not answers: + raise Exception("Cancelled") + best_name = answers["best_name"].splitlines(keepends=False)[0].strip() + parse_filename(best_name) + return best_name + except ValueError: + continue + + +def move_video(video_path: Path, new_name: typing.Optional[str], link: bool = False) -> Path: + """ + Move video to the torrent directory and returns the video path + """ + if new_name: + best_name = Path(new_name).name + else: + best_name = pick_name(video_path) + assert not best_name.startswith('.') + logging.info("using name: %s", best_name) + + best_name = best_name.removesuffix('.nometadata') + if Path(best_name).suffix in VIDEO_EXTENSIONS: + best_name = Path(best_name).stem + + renamed_path = video_path.with_stem(best_name) + if link: + video_path = video_path.link_to(renamed_path) + else: + video_path = video_path.rename(renamed_path) + video_path = renamed_path + + new_loc = TORRENT_DIR / video_path.stem + new_loc.mkdir(parents=True, exist_ok=True) + target_path = new_loc / video_path.name + video_path.rename(target_path) + + for f in video_path.parent.glob("*.jpg"): + if f.stem.startswith(video_path.stem): + f.rename(new_loc / f.name) + break + + return target_path + + +def upload_image(image_path: Path) -> str: + session = httpx.Client() + res = session.get("https://jerking.empornium.ph/?agree-consent", follow_redirects=True) + try: + auth_token = re.search(r'name="auth_token" value="([^"]+)"', res.text).group(1) + except AttributeError: + auth_token = re.search(r'auth_token = "([^"]+)"', res.text).group(1) + + with image_path.open("rb") as f: + res = session.post( + "https://jerking.empornium.ph/json", + headers={"accept": "application/json"}, + data={ + "thumb_width": "160", + "thumb_height": "160", + "thumb_crop": "false", + "medium_width": "500", + "medium_crop": "false", + "type": "file", + "action": "upload", + "timestamp": str(round(time.time() * 1000)), + "auth_token": auth_token, + "nsfw": "0", + }, + files={ + "source": ("thumbs.jpg", f), + }, + ) + + data = res.json() + image_url = data["image"]["url"] + thumbnail_url = data["image"]["display_url"] + + return image_url + + +ParsedFilename = typing.TypedDict('ParsedFilename', {'actors': list[str], 'studio': str, 'date': str, 'title': str, 'tags': list[str]}) + +def parse_filename(filename: str) -> ParsedFilename: + parsed = { + "tags": [], + } + filename = re.sub(r"\s*\[\d+[^]]+]", "", filename) + if m := re.search(r"(?P.+)\s+-+\s+@(?P\S+)\s+-+\s+(?P.+)\s+-+\s+(?P<date>[\d-]+)", filename): + parsed.update(m.groupdict()) + elif m := re.search(r"(?P<actors>.+)\s+-+\s+@(?P<studio>\S+)\s+-+\s+(?P<date>[\d-]+)", filename): + parsed.update(m.groupdict()) + elif m := re.search(r"(?P<actors>.+)\s+-+\s+(?P<title>.+)\s+-+\s+(?P<date>[\d-]+)", filename): + parsed.update(m.groupdict()) + elif m := re.search(r"(?P<actors>.+)\s+-+\s+(?P<title>\D.+)", filename): + parsed.update(m.groupdict()) + else: + raise ValueError("Unknown filename format") + + if "actors" in parsed: + names = re.split(r"\s*,\s*", parsed["actors"]) + with_aliases = [] + for it in names: + with_aliases.extend(re.split(r"\s+aka\s+", it)) + parsed["actors"] = names + parsed["tags"].extend([it.lower().replace(" ", ".") for it in with_aliases]) + if "studio" in parsed: + parsed["tags"].append(parsed["studio"].lower() + ".com") + if "date" in parsed: + year, ym = parsed["date"][:4], parsed["date"][:7].replace("-", ".") + parsed["tags"].append(year) + parsed["tags"].append(ym) + + return parsed + +def generate_title(parsed: ParsedFilename) -> str: + parts = [] + if actors := parsed.get('actors'): + parts.append(', '.join(actors)) + if studio := parsed.get('studio'): + parts.append(f'@{studio}') + if title := parsed.get('title'): + parts.append(title) + if date := parsed.get('date'): + parts.append(date) + return ' -- '.join(parts) + + +def generate_post_bbcode(video_path: Path, thumbnails_path: Path) -> str: + try: + image_url = upload_image(thumbnails_path) + image_bbcode = f"[img]{image_url}[/img]" + except: + logging.exception("Failed to upload image") + image_bbcode = "" + + parsed = parse_filename(video_path.stem) + probe = ffmpeg.ffprobe(video_path) + + if 1900 <= probe.width <= 2200: + hd = "1080p" + elif 1200 <= probe.width <= 1400: + hd = "720p" + elif 3000 <= probe.width: + hd = "4K" + else: + hd = None + + if hd: + parsed["tags"].append(hd.lower()) + if probe.codec == "hevc": + parsed["tags"].extend(["x265", "x265.reencode", "hevc.x265"]) + + table = { + "Duration": probe.duration_human, + "Format": video_path.suffix.strip("."), + "Filesize": human_size(video_path.stat().st_size), + "Resolution": f"{probe.width}x{probe.height}", + "Codec": probe.codec, + "Bit rate": f"{probe.bitrate // 1000} kbit/s", + "FPS": probe.fps, + } + release_date = parsed.get("date", "") + + if release_date: + table = { + "Release Date": release_date, + **table, + } + + lines = [ + "[table=nball]", + *(f"[tr][th=20]{k}[/th][td]{v}[/td][/tr]" for k, v in table.items()), + "[/table]", + ] + table_bbcode = "\n".join(lines) + + m_performers = re.search(r"(.+?) -+\s", video_path.stem) + performers = m_performers.group(1) if m_performers else "" + + title = generate_title(parsed) + + metadata = { + **parsed, + "is_hevc": probe.codec == "hevc", + "hd": hd, + } + metadata_json = json.dumps(metadata) + + return f""" +{metadata_json} +--- +[b]{title}[/b] + +[cast] +{performers} + +[details] + +[info] +{table_bbcode} + +[screens] +{image_bbcode} +""".strip() + + +def add_torrent(torrent_path: Path): + import time + from functools import partial + + make_id = partial(time.time_ns) + + # log in + logging.debug("Connecting to Deluge") + session = httpx.Client(base_url="https://t.zzzp.win/", timeout=30) + res = session.post("/json", json={"method": "auth.login", "params": ["xAsametk50"], "id": make_id()}) + res.raise_for_status() + + # find first available host + res = session.post("/json", json={"method": "web.get_hosts", "params": [], "id": make_id()}) + res.raise_for_status() + host_id = res.json()["result"][0][0] + + # connect to a host + res = session.post("/json", json={"method": "web.connect", "params": [host_id], "id": make_id()}) + res.raise_for_status() + + # upload torrent + logging.debug("Uploading torrent file") + with torrent_path.open("rb") as f: + res = session.post("/upload", files={"file": f}) + res.raise_for_status() + remote_path: str = res.json()["files"][0] + + # add torrent + logging.debug("Adding torrent file") + res = session.post( + "/json", + json={ + "method": "web.add_torrents", + "params": [ + [ + { + "path": remote_path, + "options": { + "file_priorities": [1], + "add_paused": True, + "sequential_download": False, + "pre_allocate_storage": False, + "download_location": "/dl/_new2", + "move_completed": False, + "move_completed_path": "/root/Downloads", + "prioritize_first_last_pieces": True, + "seed_mode": True, + "super_seeding": False, + }, + } + ] + ], + "id": make_id(), + }, + ) + + res.raise_for_status() + torrent_id = res.json()["result"][0][1] + + # force recheck + # logging.info("Triggering a forced recheck") + # time.sleep(5) + # try: + # _ = session.post( + # "/json", json={"method": "core.force_recheck", "params": [[torrent_id]], "id": make_id()}, timeout=0.1 + # ) + # except httpx.HTTPError: + # pass + + logging.debug("Torrent file has been added successfully.") + + +def parse_args(argv: list[str]): + arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + arger.add_argument("dir_or_video", type=Path, help="Directory or video file to add") + arger.add_argument("--name", "-n", help="Torrent name") + arger.add_argument("--no-metadata", action='store_true', dest='remove_metadata', help="Remove all metadata") + arger.add_argument("--post-only", action="store_true", help="Generate only the post bbcode") + arger.add_argument("--link", action="store_true", help="Link video file instead of moving") + arger.add_argument("--thumb-path", dest='thumbnail_path', type=Path, help="Path to thumbnail sheet") + if not argv: + arger.print_help() + sys.exit(0) + return arger.parse_args(argv) + + +def validate_name(base_name: str): + if len(base_name) > (128 - len(".mp4")): + raise ValueError(f"Filename is too long: {base_name} ({len(base_name)} chars)") + if "?" in base_name: + raise ValueError(f"Filename contains invalid characters: {base_name}") + + +def main(): + args = parse_args(sys.argv[1:]) + dir_or_video: Path = args.dir_or_video.expanduser().resolve() + if dir_or_video.is_dir(): + dir_path = dir_or_video + if not dir_path.is_relative_to(TORRENT_DIR): + target_path = TORRENT_DIR / dir_path.name + logging.info(f"Linking to {target_path}") + shutil.copytree(src=dir_path, dst=target_path, copy_function=os.link, symlinks=False) + dir_path = target_path + torrent_path = make_torrent(dir_path) + logging.info(f"Saved torrent at {torrent_path}") + + logging.info("Adding torrent file to Deluge") + add_torrent(torrent_path) + return + + if not dir_or_video.is_file(): + logging.error("Invalid file or directory") + return + + video_path = dir_or_video + if video_path.suffix not in VIDEO_EXTENSIONS: + logging.error('not a video') + return + + if args.remove_metadata: + logging.info('Removing video metadata') + save_path = video_path.with_stem(f'{video_path.stem}.nometadata') + video_path = clean_video(video_path=video_path, save_path=save_path) + + video_path = move_video(video_path, new_name=args.name, link=args.link) + dir_path = video_path.parent + + if p := args.thumbnail_path: + thumbnail_path = video_path.with_suffix('.jpg') + p.link_to(thumbnail_path) + else: + logging.info("Creating thumbnail tile") + old_thumb_path = video_path.with_suffix(".thumbnail.jpg") + thumbnail_path = video_path.with_suffix(".jpg") + if old_thumb_path.is_file(): + old_thumb_path.rename(thumbnail_path) + ffmpeg.make_thumbnail_tile(video_path, image_path=thumbnail_path, skip_if_exists=True) + logging.info(f"Saved thumbnails at {thumbnail_path}") + + post_bbcode = generate_post_bbcode(video_path, thumbnail_path) + video_path.with_name("post.txt").write_text(post_bbcode) + print(post_bbcode) + + if args.post_only: + return + + total_files = len(list(dir_path.rglob("*"))) + logging.info(f"Creating torrent file from {dir_path}. Total files: {total_files}") + torrent_path = make_torrent(dir_path) + logging.info(f"Saved torrent at {torrent_path}") + + logging.info("Adding torrent file to Deluge") + add_torrent(torrent_path) + + +if __name__ == "__main__": + main() diff --git a/torrentify_test.py b/torrentify_test.py new file mode 100644 index 0000000..72908b1 --- /dev/null +++ b/torrentify_test.py @@ -0,0 +1,60 @@ +from cmath import exp +from torrentify import generate_title, parse_filename, ParsedFilename +import pytest + + +@pytest.mark.parametrize( + ["filename", "expected"], + [ + [ + "To Ki -- @Studio -- 2022-10-04", + { + "actors": ["To Ki"], + "date": "2022-10-04", + "studio": "Studio", + "tags": ["to.ki", "studio.com", "2022", "2022.10"], + }, + ], + [ + "To Ki, Ki To -- @Studio -- Title -- 2022-10-04", + { + "actors": ["To Ki", "Ki To"], + "date": "2022-10-04", + "studio": "Studio", + "title": "Title", + "tags": ["to.ki", "ki.to", "studio.com", "2022", "2022.10"], + }, + ], + ], +) +def test_parse_filename(filename: str, expected: dict): + parsed = parse_filename(filename) + print(filename, parsed) + assert parsed == expected + + +@pytest.mark.parametrize( + ["parsed", "expected"], + [ + [ + { + "actors": ["To Ki"], + "date": "2022-10-04", + "studio": "Studio", + }, + 'To Ki -- @Studio -- 2022-10-04', + ], + [ + { + "actors": ["To Ki", "Ki To"], + "date": "2022-10-04", + "title": "Title", + "studio": "Studio", + }, + 'To Ki, Ki To -- @Studio -- Title -- 2022-10-04', + ], + ], +) +def test_generate_title(parsed: ParsedFilename, expected: str): + generated = generate_title(parsed) + assert generated == expected diff --git a/ui_prompts.py b/ui_prompts.py new file mode 100644 index 0000000..5c3f575 --- /dev/null +++ b/ui_prompts.py @@ -0,0 +1,22 @@ +import webview +from webview import Window + + +def webview_file_dialog(): + def open_file_dialog(w: Window): + try: + return w.create_file_dialog(webview.FOLDER_DIALOG)[0] + except TypeError: + pass # user exited file dialog without picking + finally: + w.hide() + w.destroy() + + window = webview.create_window("", hidden=True) + webview.start(open_file_dialog, window) + # file will either be a string or None + return file + + +if __name__ == "__main__": + print(webview_file_dialog()) diff --git a/upload_progress.py b/upload_progress.py new file mode 100644 index 0000000..635bc12 --- /dev/null +++ b/upload_progress.py @@ -0,0 +1,62 @@ +import dataclasses +import functools +import io +import pathlib +from datetime import timedelta, datetime +from io import FileIO +from typing import Callable + +import httpx + + +class ProgressIO(io.BufferedReader): + @dataclasses.dataclass + class Progress: + total: int + processed: int + + def percent(self) -> float: + return round(100 * self.processed / self.total, 2) + + def __init__(self, file: pathlib.Path, on_progress: Callable[[Progress], None]): + super().__init__(raw=FileIO(str(file))) + self._size = file.stat().st_size + self._on_progress_throttled = self.throttle(1)(on_progress) + self._on_progress = on_progress + self._processed = 0 + self._iter = None + + def __next__(self) -> bytes: + chunk = super().__next__() + self._processed += len(chunk) + progress = ProgressIO.Progress(self._size, self._processed) + if self._processed == self._size: + self._on_progress(progress) + else: + self._on_progress_throttled(progress) + + return chunk + + @staticmethod + def throttle(seconds: float = 0): + period = timedelta(seconds=seconds) + + def decorator(fn): + last_called = datetime.min + + @functools.wraps(fn) + def wrapper(*args, **kwargs): + now = datetime.now() + nonlocal last_called + if now - last_called > period: + last_called = now + return fn(*args, **kwargs) + + return wrapper + + return decorator + + +if __name__ == "__main__": + f = pathlib.Path(r"/path/to/file") + httpx.post("http://httpbin.org/post", files={"file": ProgressIO(f, on_progress=lambda p: print(p.percent()))}) diff --git a/vercel.json b/vercel.json new file mode 100644 index 0000000..6bd5404 --- /dev/null +++ b/vercel.json @@ -0,0 +1,15 @@ +{ + "version": 2, + "builds": [ + { + "src": "./app.py", + "use": "@vercel/python" + } + ], + "routes": [ + { + "src": "/(.*)", + "dest": "/app.py" + } + ] +} \ No newline at end of file diff --git a/worker.py b/worker.py new file mode 100644 index 0000000..76f3ec3 --- /dev/null +++ b/worker.py @@ -0,0 +1,69 @@ +import flask +import huey +import sentry_sdk +import sentry_sdk.integrations.flask +from huey import signals + +sentry_sdk.init( + "https://379e370d4dee471b8ca5c7884d2137c0@o271257.ingest.sentry.io/6214795", + # Set traces_sample_rate to 1.0 to capture 100% + # of transactions for performance monitoring. + # We recommend adjusting this value in production. + traces_sample_rate=1.0, + with_locals=True, + max_breadcrumbs=100, + integrations=[sentry_sdk.integrations.flask.FlaskIntegration()], +) + +worker = huey.SqliteHuey(filename='worker.sqlite3') + + +@worker.signal(signals.SIGNAL_EXECUTING, signals.SIGNAL_LOCKED, signals.SIGNAL_CANCELED, signals.SIGNAL_REVOKED) +def task_executing_handler(signal: str, task: huey.api.Task): + sentry_sdk.add_breadcrumb( + category='worker', + data=dict( + signal=signal, + task_id=task.id, + task_name=task.name, + task_retries=task.retries, + task_args=task.args, + task_kwargs=task.kwargs, + ), + ) + + +@worker.signal(signals.SIGNAL_ERROR) +def task_not_executed_handler(signal: str, task: huey.api.Task, exc: Exception): + with sentry_sdk.push_scope(): + sentry_sdk.capture_exception(exc) + + +class SomeService: + @classmethod + @worker.task() + def do_something(cls, *args, **kwargs): + print(args, kwargs) + + @classmethod + @worker.task() + def fail_something(cls, *args, **kwargs): + raise Exception('boom') + + def do(self): + self.do_something(2, a=1) + + def fail(self): + self.fail_something(2, a=1) + + +def test_request(): + flask_app = flask.Flask(__name__) + + @flask_app.post('/') + def fail(): + SomeService().fail() + return {'a': 5} + + flask_app.add_url_rule('/', 'home', fail) + flask_app.test_client().post('/', json={'a': 10}) diff --git a/worker_app.py b/worker_app.py new file mode 100644 index 0000000..e4bd86d --- /dev/null +++ b/worker_app.py @@ -0,0 +1,4 @@ +from worker import SomeService + +if __name__ == '__main__': + SomeService().fail() diff --git a/wsgi_server.py b/wsgi_server.py new file mode 100644 index 0000000..acc4f4c --- /dev/null +++ b/wsgi_server.py @@ -0,0 +1,19 @@ +from wsgiref.util import setup_testing_defaults +from wsgiref.simple_server import make_server + + +def simple_app(environ, start_response): + setup_testing_defaults(environ) + + status = "200 OK" + headers = [("Content-type", "text/plain; charset=utf-8")] + + start_response(status, headers) + + ret = [("%s: %s\n" % (key, value)).encode("utf-8") for key, value in environ.items()] + return ret + + +if __name__ == "__main__": + with make_server("", 8001, simple_app) as httpd: + httpd.handle_request() diff --git a/xss.py b/xss.py new file mode 100644 index 0000000..79d9dde --- /dev/null +++ b/xss.py @@ -0,0 +1,10 @@ +html = '<b>i am html</b>' + + +def a_func(): + raise ValueError(html) + + +if __name__ == '__main__': + a_func() # breakpoint here + ... \ No newline at end of file