feat(snot): add search history and purge command

Standalone script that drops and re-establishes the router's PPPoE session
without a full reboot. Mirrors router_ui.py's RouterSession pattern: a
context manager that always logs out on exit, the CSRFToken header required
for writes since ABSB.5, and tolerant handling of the Disconnect/Connect
calls timing out with a Connect retry if the link doesn't come back up.

Confirmed working live: WAN IP and public IP (checked via api.ipify.org)
both changed after a run.

aria2_on_done.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+import base64
+import logging
+import os
+import sys
+from pathlib import Path
+from urllib.error import HTTPError, URLError
+from urllib.parse import quote
+from urllib.request import Request, urlopen
+
+logging.basicConfig(
+    filename=Path(__file__).with_suffix(".log"),
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+
+def is_temp(file_path: Path) -> bool:
+    return ".PRT" in file_path.name
+
+
+def delete_from_remote(file_path: Path) -> None:
+    url = f"https://u201686.your-storagebox.de/files/_raw/prt/{quote(file_path.name)}"
+    credentials = "u201686:T6672ICVoWAedECH"
+    request = Request(url, method="DELETE", headers={"Authorization": f"Basic {base64.b64encode(credentials.encode()).decode()}"})
+
+    try:
+        with urlopen(request, timeout=2):
+            return
+    except HTTPError as e:
+        if e.code == 404:
+            return
+
+
+def main():
+    logger.info(sys.argv)
+
+    if len(sys.argv) < 3:
+        logger.error("Missing expected argument: downloaded file path.")
+        return
+
+    downloaded_path = Path(sys.argv[2])
+    if not downloaded_path.is_file():
+        logger.warning("File does not exist: %s", downloaded_path)
+        return
+
+    if not is_temp(downloaded_path):
+        logger.info("File evaluated as non-temp: %s", downloaded_path)
+        return
+
+    target_dir = Path("~/Downloads/temp/").expanduser()
+    target_dir.mkdir(parents=True, exist_ok=True)
+
+    target_path = target_dir / downloaded_path.name
+    downloaded_path.rename(target_path)
+    logger.info("Moved %s to %s", downloaded_path, target_path)
+    try:
+        delete_from_remote(downloaded_path)
+    except Exception as e:
+        logger.error("Error deleting %s from remote: %s", downloaded_path, e)
+
+
+if __name__ == "__main__":
+    main()

berlin.py

@@ -1,48 +0,0 @@
-import contextlib
-
-from playwright.sync_api import sync_playwright, Page
-from time import sleep
-
-
-@contextlib.contextmanager
-def launch_browser() -> Page:
-    with sync_playwright() as playwright:
-        with playwright.chromium.launch(headless=False) as browser:
-            with browser.new_context() as context:
-                with context.new_page() as page:
-                    yield page
-
-
-def book_slot():
-    with launch_browser() as page:
-        page.goto("https://service.berlin.de/dienstleistung/120686/")
-        page.get_by_role("complementary").get_by_role("link", name="Termin berlinweit suchen").click()
-
-        with page.expect_navigation(url='https://service.berlin.de/terminvereinbarung/termin/time/*', timeout=0):
-            pass
-
-        with page.expect_navigation(url='https://service.berlin.de/terminvereinbarung/termin/register/*', timeout=0):
-            pass
-
-        page.get_by_label("Ihr Vor- und Nachname *").fill("Paulina Gutkowska")
-        page.get_by_label("Ihre E-Mail Adresse *").fill("gutpaula11@gmail.com")
-        if page.is_visible("Ihre Telefonnummer oder Mobilfunknummer *"):
-            page.get_by_label("Ihre Telefonnummer oder Mobilfunknummer *").fill("+48501099723")
-        page.get_by_role("combobox").select_option("0")
-        page.get_by_label(
-            "Ich erkläre mich mit den Nutzungsbedingungen einverstanden und akzeptiere diese. (Pflichtangabe. Termineintrag nur bei Einverständnis möglich) *"
-        ).check()
-
-        with page.expect_navigation():
-            page.get_by_role("button", name="Termin eintragen").click()
-        
-        sleep(10)
-        input()
-
-
-def main():
-    book_slot()
-
-
-if __name__ == '__main__':
-    main()

berlin_anmeldung.py

@@ -1,75 +0,0 @@
-import contextlib
-import datetime
-import locale
-
-import httpx
-from bs4 import BeautifulSoup
-
-import berlin
-
-http = httpx.Client(
-    headers={
-        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36'
-    },
-    follow_redirects=True,
-)
-
-
-@contextlib.contextmanager
-def override_locale(category: int, val: str) -> None:
-    prev = locale.getlocale(category)
-    locale.setlocale(category, val)
-    yield
-    locale.setlocale(category, prev)
-
-
-def get_available_slots():
-    res = http.get('https://service.berlin.de/dienstleistung/120686/')
-    res.raise_for_status()
-
-    soup = BeautifulSoup(res.text, 'html.parser')
-    link = soup.select_one('[role="complementary"] .zmstermin-multi a')
-    calendar_url = link.attrs['href']
-
-    try:
-        # redirect fails, we just need the cookies
-        _ = http.get(calendar_url)
-    except:
-        pass
-    res = http.get('https://service.berlin.de/terminvereinbarung/termin/day/')
-    res.raise_for_status()
-
-    return extract_days(res.text)
-
-
-def extract_days(html: str) -> list[datetime.date]:
-    soup = BeautifulSoup(html, 'html.parser')
-
-    found = []
-    for it in soup.select('.calendar-month-table .buchbar'):
-        month_year = it.find_parent(attrs={'class': 'calendar-month-table'}).select_one('thead .month').text.strip()
-        day = it.text.strip()
-        found.append(f'{day} {month_year}')
-
-    with override_locale(locale.LC_TIME, 'de_DE'):
-        days = [datetime.datetime.strptime(it, '%d %B %Y').date() for it in found]
-
-    return days
-
-
-def main():
-    days = get_available_slots()
-    good_days = [it for it in days if it < datetime.date(2023, 1, 27)]
-
-    if not good_days:
-        print('no slots')
-        return
-
-    print('found slots')
-    berlin.book_slot()
-    for it in days:
-        print(it)
-
-
-if __name__ == '__main__':
-    main()

berlin_immigration.py

@@ -1,156 +0,0 @@
-import argparse
-import asyncio
-import contextlib
-import dataclasses
-import datetime
-import logging
-import random
-import re
-import subprocess
-import urllib.parse
-from pathlib import Path
-
-from playwright.async_api import async_playwright, Browser, BrowserContext, Page
-
-
-@contextlib.asynccontextmanager
-async def launch_browser(headless: bool = False) -> Page:
-    async with async_playwright() as playwright:
-        browser = await playwright.chromium.launch(headless=headless)
-        browser: Browser
-        async with browser:
-            ctx = await browser.new_context()
-            ctx: BrowserContext
-            async with ctx:
-                page = await ctx.new_page()
-                async with page:
-                    yield page
-
-
-class Error(Exception):
-    pass
-
-
-def alert(text: str):
-    subprocess.run(['open', f'alfred://runtrigger/dev.abdus.integrations/remind/?argument={urllib.parse.quote(text)}'])
-
-
-dump_dir = Path('~/Desktop').expanduser() / 'immigration'
-dump_dir.mkdir(parents=True, exist_ok=True)
-
-
-@dataclasses.dataclass
-class ResidencyApplication:
-    citizenship_country: str
-    num_applicants: int
-    is_living_with_family: bool
-    residency_category: str
-    residency_purpose: str
-
-
-async def check_slots(page: Page, values: ResidencyApplication, retry_count: int = 1) -> list[datetime.datetime]:
-    await page.add_init_script('''Object.defineProperty(navigator, 'webdriver', { get: () => false })''')
-    window_id = random.randint(1000, 9999)
-    req_id = random.randint(0, 999)
-    await page.goto(
-        f'https://otv.verwalt-berlin.de/ams/TerminBuchen/wizardng?dswid={window_id}&dsrid={req_id}',
-        wait_until='networkidle',
-    )
-
-    async with page.expect_navigation():
-        await page.click('.langBlock [name="txtEn"] a')
-
-    await page.check('[name="gelesen"]')
-
-    async with page.expect_navigation(url=re.compile('st=2'), wait_until='networkidle'):
-        await page.click('[name="applicationForm:managedForm:proceed"]')
-
-    await page.get_by_role("combobox", name="Citizenship *").select_option(label=values.citizenship_country)
-    await asyncio.sleep(0.3)
-
-    # how many applicants
-    await page.select_option('[name="personenAnzahl_normal"]', value=str(values.num_applicants))
-    await asyncio.sleep(0.3)
-
-    # living with family
-    await page.select_option('[name="lebnBrMitFmly"]', value='1' if values.is_living_with_family else '2')
-    await asyncio.sleep(0.3)
-
-    await page.get_by_text("Apply for a residence title").click()
-    await page.locator("label").filter(has_text=values.residency_category).click()
-    await page.get_by_text(values.residency_purpose).click()
-
-    stop_at = datetime.datetime.now() + datetime.timedelta(minutes=28)
-
-    while 'st=2' in page.url:
-        retry_count -= 1
-
-        async with page.expect_navigation(url=re.compile(r'st='), wait_until='networkidle', timeout=60_000):
-            await page.get_by_role("button", name="Next").click()
-
-        if datetime.datetime.now() > stop_at:
-            raise TimeoutError('could not find a slot')
-
-        if not retry_count:
-            return []
-
-        await asyncio.sleep(random.randint(4, 10))
-        continue
-
-    if 'st=3' in page.url:
-        dates = []
-        for el in await page.query_selector_all('.ui-datepicker-calendar td:not(.ui-datepicker-unselectable)'):
-            dates.append(
-                datetime.date(
-                    year=int(await el.get_attribute('data-year')),
-                    month=int(await el.get_attribute('data-month')),
-                    day=int(await el.inner_text()),
-                )
-            )
-
-        alert("\n".join(['empty slots found', *[it.isoformat() for it in dates]]))
-
-        now = int(datetime.datetime.now().timestamp())
-        html_path = dump_dir / f'immigration_{now}.html'
-        html_path.write_text(await page.content())
-        await page.screenshot(path=dump_dir / f'immigration_{now}.png')
-
-        input()
-
-    input()
-
-
-def parse_args():
-    arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    arger.add_argument(
-        '--headless', '--silent', '--quiet', action='store_true', dest='headless', help='Run in headless mode'
-    )
-    arger.add_argument('--retry-count', '--retry', type=int, default=1, help='Run in headless mode')
-
-    args, _ = arger.parse_known_args()
-    return args
-
-
-async def main():
-    args = parse_args()
-    async with launch_browser(headless=args.headless) as page:
-        try:
-            values = ResidencyApplication(
-                citizenship_country='India',
-                num_applicants=1,
-                is_living_with_family=False,
-                residency_category='Educational purposes',
-                residency_purpose='Residence permit for the purpose of studying (sect. 16b)',
-            )
-            slots = await check_slots(page, values=values, retry_count=args.retry_count)
-            if not slots:
-                print('no slots yet')
-                return
-        except Exception as e:
-            logging.exception('got an error')
-            input()
-
-
-if __name__ == '__main__':
-    asyncio.run(main())
-    logging.basicConfig(level=logging.INFO)

bump_version.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+import argparse
+import logging
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Create a tag at HEAD by bumping the highest local X.Y.Z or vX.Y.Z tag. "
+        "Preserves the latest tag's prefix; starts from v0.0.0 when no version tags exist.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    group = parser.add_mutually_exclusive_group(required=True)
+    for part in ("major", "minor", "patch"):
+        group.add_argument(f"--{part}", dest="part", action="store_const", const=part,
+                           help=f"Bump the {part} version and reset lower components")
+    parser.add_argument("--cwd", type=Path, default=Path.cwd(), help="Git repository directory")
+    parser.add_argument("--push", action="store_true", help="Push only the new tag to origin")
+    return parser.parse_args()
+
+
+def git(cwd: Path, arguments: list[str]) -> str:
+    result = subprocess.run(
+        args=["git", *arguments], cwd=cwd, check=True, capture_output=True, text=True,
+    )
+    return result.stdout.strip()
+
+
+def next_tag(tags: list[str], part: str) -> str:
+    versions = []
+    for tag in tags:
+        match = re.fullmatch(r"(v?)(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)", tag)
+        if match:
+            versions.append((tuple(int(value) for value in match.groups()[1:]), match[1]))
+    version, prefix = max(versions, default=((0, 0, 0), "v"))
+    components = list(version)
+    index = ("major", "minor", "patch").index(part)
+    components[index] += 1
+    components[index + 1:] = [0] * (2 - index)
+    return f"{prefix}{'.'.join(str(value) for value in components)}"
+
+
+def main() -> int:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s")
+    arguments = parse_args()
+    try:
+        tags = git(cwd=arguments.cwd, arguments=["tag", "--list"]).splitlines()
+        tag = next_tag(tags=tags, part=arguments.part)
+        git(cwd=arguments.cwd, arguments=["tag", "--", tag, "HEAD"])
+        logging.info(f"Created {tag} at HEAD")
+        if arguments.push:
+            git(cwd=arguments.cwd, arguments=["push", "origin", f"refs/tags/{tag}:refs/tags/{tag}"])
+            logging.info(f"Pushed {tag} to origin")
+    except subprocess.CalledProcessError as error:
+        logging.error(f"Git failed: {(error.stderr or str(error)).strip()}")
+        return 1
+    except OSError as error:
+        logging.error(f"{error}")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

bump_version_test.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env -S uv run --script
+# /// script
+# dependencies = ["pytest"]
+# ///
+
+from pathlib import Path
+import subprocess
+import sys
+
+import pytest
+
+import bump_version as script
+
+
+@pytest.mark.parametrize("tags, part, expected", [
+    ([], "patch", "v0.0.1"),
+    ([], "minor", "v0.1.0"),
+    ([], "major", "v1.0.0"),
+    (["v1.9.9", "v1.10.2", "v1.2.3"], "patch", "v1.10.3"),
+    (["v1.9.9", "v1.10.2"], "minor", "v1.11.0"),
+    (["v1.9.9", "v1.10.2"], "major", "v2.0.0"),
+    (["1.2.3", "v1.0.0"], "patch", "1.2.4"),
+    (["release", "v9.0.0-rc.1", "v01.0.0", "v1.2.3+build", "v1.2.3"], "patch", "v1.2.4"),
+])
+def test_next_tag(tags: list[str], part: str, expected: str) -> None:
+    assert script.next_tag(tags=tags, part=part) == expected
+
+
+@pytest.mark.parametrize("push", [False, True])
+def test_create_and_push_tag(tmp_path: Path, push: bool) -> None:
+    repository = tmp_path / "repository"
+    remote = tmp_path / "remote.git"
+    subprocess.run(args=["git", "init", "--initial-branch=master", str(repository)], check=True, capture_output=True)
+    subprocess.run(args=["git", "init", "--bare", str(remote)], check=True, capture_output=True)
+    script.git(cwd=repository, arguments=["-c", "user.name=Test", "-c", "user.email=test@example.com",
+                                          "-c", "commit.gpgsign=false", "commit", "--allow-empty", "--message=Initial"])
+    script.git(cwd=repository, arguments=["remote", "add", "origin", str(remote)])
+    script.git(cwd=repository, arguments=["tag", "v1.9.9"])
+    script.git(cwd=repository, arguments=["tag", "v1.10.2"])
+    command = [sys.executable, str(Path(script.__file__).resolve()), "--minor", "--cwd", str(repository)]
+    if push:
+        command.append("--push")
+    subprocess.run(args=command, check=True, capture_output=True, text=True)
+    assert script.git(cwd=repository, arguments=["rev-parse", "v1.11.0"]) == script.git(
+        cwd=repository, arguments=["rev-parse", "HEAD"],
+    )
+    assert script.git(cwd=remote, arguments=["tag", "--list"]) == ("v1.11.0" if push else "")
+    assert script.git(cwd=remote, arguments=["for-each-ref", "refs/heads"]) == ""
+
+
+@pytest.mark.parametrize("arguments", [[], ["--major", "--patch"]])
+def test_requires_exactly_one_bump(arguments: list[str]) -> None:
+    result = subprocess.run(
+        args=[sys.executable, str(Path(script.__file__).resolve()), *arguments], capture_output=True,
+    )
+    assert result.returncode == 2
+
+
+def test_missing_repository(tmp_path: Path) -> None:
+    result = subprocess.run(
+        args=[sys.executable, str(Path(script.__file__).resolve()), "--patch", "--cwd", str(tmp_path)],
+        capture_output=True, text=True,
+    )
+    assert result.returncode == 1
+    assert "Git failed:" in result.stderr
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(args=[str(Path(__file__).resolve()), *sys.argv[1:]]))

chat/index.html

@@ -441,7 +441,7 @@
                     <summary>History</summary>
                     <div class="history-list">
                         <template x-if="historyItems.length === 0">
-                            <div style="opacity: 0.7">No conversations today</div>
+                            <div style="opacity: 0.7">No conversations in the last 24 hours</div>
                         </template>
                         <template x-for="(item, idx) in historyItems" :key="idx">
                             <div class="history-item" @click="restoreConversation(item.messages)">
@@ -526,7 +526,7 @@
                 pendingImages: [],
                 attachment: initialAttachment,
                 promptText: initialPromptText,
-                activeModelName: "gemini-3.8-flash",
+                activeModelName: "claude-haiku-5-5",
                 historyItems: [],
                 isBusy: false,
                 abortController: null,
@@ -537,7 +537,7 @@
                     { name: "gpt-6-luna", chat: chatWithOpenAI, key: window.env.OPENAI_API_KEY },
                     { name: "gemini-3.5-flash-lite", chat: chatWithGemini, key: window.env.GEMINI_API_KEY },
                     { name: "gemini-3.8-flash", chat: chatWithGemini, key: window.env.GEMINI_API_KEY },
-                    { name: "claude-haiku-4-5-20251001", chat: chatWithClaude, key: window.env.ANTHROPIC_API_KEY },
+                    { name: "claude-haiku-5-5", chat: chatWithClaude, key: window.env.ANTHROPIC_API_KEY },
                 ],
 
                 async init() {
@@ -640,7 +640,7 @@
                     } finally {
                         this.isBusy = false;
                         this.abortController = null;
-                        if (!this.attachment) updateSessionHistory(this.sessionId, this.messages);
+                        if (!this.attachment) this.historyItems = updateSessionHistory(this.sessionId, this.messages);
                     }
                 },
 
@@ -669,7 +669,7 @@
                     const idx = this.messages.indexOf(msg);
                     if (idx === -1) return;
                     this.messages = this.messages.slice(0, idx);
-                    if (!this.attachment) updateSessionHistory(this.sessionId, this.messages);
+                    if (!this.attachment) this.historyItems = updateSessionHistory(this.sessionId, this.messages);
                     this.promptText = msg.content;
                     this.pendingImages = msg.images ? JSON.parse(JSON.stringify(msg.images)) : [];
                     this.$nextTick(() => this.$refs.promptInput.focus());
@@ -977,29 +977,60 @@
             const STORAGE_KEY = "chat_conversations";
             const todayKey = () => new Date().toISOString().slice(0, 10);
 
+            const readHistory = () => {
+                const all = JSON.parse(localStorage.getItem(STORAGE_KEY) || "{}");
+                const cutoff = Date.now() - 24 * 60 * 60 * 1000;
+                let changed = false;
+                for (const day of Object.keys(all)) {
+                    const recent = all[day].filter((item) => Date.parse(item.time) >= cutoff);
+                    if (recent.length !== all[day].length) changed = true;
+                    if (recent.length) all[day] = recent;
+                    else {
+                        delete all[day];
+                        changed = true;
+                    }
+                }
+                if (changed) localStorage.setItem(STORAGE_KEY, JSON.stringify(all));
+                return all;
+            };
+
+            const historyItemsFrom = (all) => Object.values(all).flat().sort((a, b) => Date.parse(b.time) - Date.parse(a.time));
+
             const loadHistory = () => {
                 try {
-                    return JSON.parse(localStorage.getItem(STORAGE_KEY) || "{}")[todayKey()] || [];
+                    return historyItemsFrom(readHistory());
                 } catch {
                     return [];
                 }
             };
 
             const updateSessionHistory = (sessionId, messages) => {
-                const all = JSON.parse(localStorage.getItem(STORAGE_KEY) || "{}");
+                const all = readHistory();
-                const today = todayKey();
+                const day = Object.keys(all).find((day) => all[day].some((item) => item.sessionId === sessionId)) || todayKey();
-                all[today] ??= [];
+                all[day] ??= [];
 
-                const idx = all[today].findIndex((item) => item.sessionId === sessionId);
+                const idx = all[day].findIndex((item) => item.sessionId === sessionId);
                 if (idx >= 0) {
-                    all[today][idx] = { sessionId, time: all[today][idx].time, messages };
+                    all[day][idx] = { sessionId, time: all[day][idx].time, messages };
                 } else {
-                    all[today].unshift({ sessionId, time: new Date().toISOString(), messages });
+                    all[day].unshift({ sessionId, time: new Date().toISOString(), messages });
                 }
 
-                all[today] = all[today].slice(0, 5);
+                all[day] = all[day].slice(0, 5);
+                while (true) {
+                    try {
                         localStorage.setItem(STORAGE_KEY, JSON.stringify(all));
-                return all[today];
+                        break;
+                    } catch (error) {
+                        if (error.name !== "QuotaExceededError") throw error;
+                        const oldestDay = Object.keys(all).sort().find((day) => all[day].some((item) => item.sessionId !== sessionId));
+                        if (!oldestDay) throw error;
+                        const idx = all[oldestDay].findLastIndex((item) => item.sessionId !== sessionId);
+                        all[oldestDay].splice(idx, 1);
+                        if (!all[oldestDay].length) delete all[oldestDay];
+                    }
+                }
+                return historyItemsFrom(all);
             };
 
             const clearAllHistory = () => {

csv_to_html.py

@@ -1,167 +0,0 @@
-import csv
-import html
-import sys
-
-# language=css
-page_css = """
-body {
-  margin: 0;
-  padding: 0.5rem;
-  font-family: menlo, consolas, monospace;
-  font-size: 14px;
-}
-input {
-  font-size: inherit;
-  font-family: inherit;
-  background-color: transparent;
-  padding: 0.25em 0.5em;
-}
-table { 
-  border-collapse: collapse; 
-  width: 100%; 
-}
-th, td { 
-  border: 1px solid black; 
-  padding: 8px; 
-  text-align: left; 
-}
-th { 
-  background-color: #f2f2f2;
-}
-tr:nth-child(even) {
-  background-color: #fdfdfd;
-}
-"""
-
-# language=js
-page_script = """
-function sortableTable(table) {
-  const sortStates = new Map();
-  
-  table.querySelectorAll('th').forEach((header, colIndex) => {
-    header.style.cursor = 'pointer';
-    header.onclick = () => {
-      const isAsc = !sortStates.get(header);
-      const tbody = table.querySelector('tbody');
-      const rows = [...tbody.rows];
-      
-      rows.sort((a, b) => {
-        const [valA, valB] = [a, b].map(row => 
-          row.cells[colIndex].textContent.trim());
-        
-        return /^\d+$/.test(valA)
-          ? (valA - valB) * (isAsc ? 1 : -1)
-          : valA.localeCompare(valB) * (isAsc ? 1 : -1);
-      });
-
-      sortStates.set(header, isAsc);
-      tbody.replaceChildren(...rows);  // Better than innerHTML = ''
-    };
-  });
-}
-
-function debounce(func, delay) {
-  let timeoutId;
-
-  return function(...args) {
-    clearTimeout(timeoutId);
-    timeoutId = setTimeout(() => {
-      func.apply(this, args);
-    }, delay);
-  };
-}
-
-function addFilterInput(tableElement) {
-  const filterInput = document.createElement('input');
-  filterInput.type = 'text';
-  filterInput.placeholder = "Search...";
-
-  tableElement.parentNode.insertBefore(filterInput, tableElement);
-
-  const debouncedFilter = debounce(filter, 100);
-
-  filterInput.addEventListener('input', () => {
-    debouncedFilter();
-  });
-
-  function filter() {
-    const filterValue = filterInput.value.trim();
-
-    const rows = tableElement.querySelectorAll('tbody tr');
-
-    rows.forEach(row => {
-      const rowData = row.textContent.toLowerCase();
-
-      if (filterValue.startsWith('/') && filterValue.endsWith('/')) {
-        const regex = new RegExp(filterValue.slice(1, -1), 'ig');
-        row.style.display = regex.test(rowData) ? '' : 'none';
-      } else {
-        row.style.display = rowData.includes(filterValue.toLowerCase()) ? '' : 'none';
-      }
-    });
-  }
-}
-
-sortableTable(document.querySelector('table'));
-addFilterInput(document.querySelector('table'));
-"""
-
-
-def csv_to_html(csv_input: str) -> str:
-    try:
-        lines = csv_input.splitlines()
-        delimiter = "\t" if "\t" in lines[0] else ","
-        reader = csv.reader(lines, delimiter=delimiter)
-        header = next(reader)  # Get the header row
-
-        header_html = f'{"".join(f"<th>{html.escape(col)}</th>" for col in header)}'
-        rows = [
-            f"<tr>{''.join(f'<td>{html.escape(cell)}</td>' for cell in row)}</tr>"
-            for row in reader
-        ]
-        rows_html = "\n".join(rows)
-    except csv.Error as e:
-        raise ValueError(f"Error parsing CSV: {e}")
-    except StopIteration:
-        return "<p>No data found in CSV input.</p>"
-
-    rendered = f"""
-        <!DOCTYPE html>
-        <html>
-        <head>
-            <title>Query Results</title>
-            <style>{page_css}</style>
-        </head>
-        <body>
-            <table>
-                <thead>
-                    <tr>{header_html}</tr>
-                </thead>
-                <tbody>
-                    {rows_html}
-                </tbody>
-            </table>
-            <script>{page_script}</script>
-        </body>
-        </html>
-    """
-
-    return rendered
-
-
-def main():
-    try:
-        csv_data = sys.stdin.read()
-        html_output = csv_to_html(csv_data)
-        if html_output:
-            print(html_output)
-    except UnicodeDecodeError as e:
-        raise ValueError(
-            f"Error decoding input: {e}.  Ensure your CSV data is encoded correctly (e.g., UTF-8)."
-        )
-    except Exception as e:
-        raise Exception(f"An unexpected error occurred: {e}")
-
-
-if __name__ == "__main__":
-    main()

cull_edits.py

@@ -1,939 +0,0 @@
-#!/usr/bin/env -S uv run
-# /// script
-# requires-python = ">=3.14"
-# dependencies = [
-#     "bottle>=0.13.0",
-# ]
-# ///
-
-import argparse
-import functools
-import os
-import re
-import socket
-import sys
-import threading
-import time
-import webbrowser
-from pathlib import Path
-from datetime import datetime
-from typing import TypedDict
-import bottle
-
-IMAGE_EXTENSIONS: set[str] = {".jpg", ".jpeg", ".png", ".webp", ".webm"}
-
-
-class ImageGroup(TypedDict):
-    """Type definition for grouped images."""
-
-    base: str
-    original: Path
-    edits: list[Path]
-
-
-def find_available_port() -> int:
-    """Find an available random port."""
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        s.bind(("127.0.0.1", 0))
-        s.listen(1)
-        port = s.getsockname()[1]
-    return port
-
-
-def is_image(path: str | Path) -> bool:
-    """Check if file is a supported image."""
-    return Path(path).suffix.lower() in IMAGE_EXTENSIONS
-
-
-def discover_images(paths: list[str]) -> list[Path]:
-    """Discover all images from mixed list of files and folders."""
-    images = []
-    for path_str in paths:
-        path = Path(path_str)
-        if path.is_file() and is_image(path):
-            images.append(path.resolve())
-        elif path.is_dir():
-            for item in path.iterdir():
-                if item.is_file() and is_image(item):
-                    images.append(item.resolve())
-    return sorted(set(images))
-
-
-def group_images(images: list[Path]) -> list[ImageGroup]:
-    """
-    Group images by original + edits.
-    Original: image.jpg
-    Edits: image-edit1.jpg, image-edit2.jpg, etc.
-    Returns list of dicts: {original: Path, edits: [Path, ...]}
-    """
-    groups = {}
-
-    for img in images:
-        stem = img.stem
-
-        # Check if this is an edit
-        match = re.match(r"^(.+?)-edit", stem)
-        if match:
-            base = match.group(1)
-            if base not in groups:
-                groups[base] = {"original": None, "edits": []}
-            groups[base]["edits"].append(img)
-        else:
-            # This is an original
-            if stem not in groups:
-                groups[stem] = {"original": None, "edits": []}
-            groups[stem]["original"] = img
-
-    # Filter out groups without originals, convert to list
-    result = []
-    for base, group in groups.items():
-        if group["original"]:
-            result.append(ImageGroup(base=base, original=group["original"], edits=sorted(group["edits"])))
-
-    return sorted(result, key=lambda g: g["original"].name)
-
-
-def safe_image_path(filepath: str) -> Path:
-    """Verify image path is safe (prevent directory traversal)."""
-    # Search for the image in all_images by filename
-    for img in all_images:
-        if img.name == filepath:
-            return img
-    raise ValueError(f"Unauthorized path: {filepath}")
-
-
-def hardlink_images(src_paths: list[Path | str], dest_dir: str | Path) -> None:
-    """Hard link multiple images to destination directory."""
-    dest_path = Path(dest_dir)
-    make_dirs(dest_path)
-
-    for src in src_paths:
-        src = Path(src)
-        dest = dest_path / src.name
-        try:
-            # Remove existing file if present
-            if dest.exists():
-                dest.unlink()
-            os.link(src, dest)
-        except Exception as e:
-            print(f"Error hard linking {src} to {dest}: {e}", file=sys.stderr)
-            raise
-
-
-# Global state
-app = bottle.Bottle()
-all_images = set()
-image_groups = []
-current_group_idx = 0
-last_heartbeat = datetime.now()
-should_exit = False
-picks_dir = Path()
-flags = []
-flag_dirs = {}
-
-HTML_TEMPLATE = """
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Image Culler</title>
-    <script defer src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"></script>
-    <style>
-        * {
-            margin: 0;
-            padding: 0;
-            box-sizing: border-box;
-        }
-        body {
-            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
-            background: #1e1e1e;
-            color: #e0e0e0;
-            overflow: hidden;
-            height: 100vh;
-        }
-        .container {
-            display: flex;
-            height: calc(100vh - 46px);
-            gap: 20px;
-            padding: 20px;
-        }
-        .panel {
-            display: flex;
-            flex-direction: column;
-            gap: 15px;
-        }
-        .left-panel {
-            flex: 0 0 40%;
-        }
-        .right-panel {
-            flex: 0 0 60%;
-            overflow-y: auto;
-        }
-        .image-display {
-            width: 100%;
-            height: 100%;
-            background: #2a2a2a;
-            border-radius: 8px;
-            display: flex;
-            align-items: center;
-            justify-content: center;
-            overflow: hidden;
-            border: 2px solid transparent;
-            transition: border-color 0.2s;
-        }
-        .image-display:hover {
-            border-color: #4a9eff;
-        }
-        .image-display.picked {
-            border-color: #4ade80;
-            background: rgba(74, 222, 128, 0.1);
-        }
-        .image-display img {
-            max-width: 100%;
-            max-height: 100%;
-            object-fit: contain;
-        }
-        .image-info {
-            background: #2a2a2a;
-            padding: 12px;
-            border-radius: 8px;
-            font-size: 14px;
-        }
-        dialog {
-            background: #2a2a2a;
-            color: #e0e0e0;
-            border: 1px solid #404040;
-            border-radius: 8px;
-            padding: 20px;
-            max-width: 400px;
-        }
-        dialog::backdrop {
-            background: rgba(0, 0, 0, 0.5);
-        }
-        .grid {
-            display: grid;
-            grid-template-columns: repeat(2, 1fr);
-            gap: 10px;
-        }
-        .grid[data-image-count="1"] {
-            grid-template-columns: 1fr;
-        }
-        .grid-item {
-            position: relative;
-            background: #2a2a2a;
-            border-radius: 8px;
-            overflow: hidden;
-            cursor: pointer;
-            border: 2px solid transparent;
-            transition: all 0.2s;
-        }
-        .grid-item:hover {
-            border-color: #4a9eff;
-        }
-        .grid-item img {
-            width: 100%;
-            height: auto;
-            display: block;
-            object-fit: contain;
-        }
-        .grid[data-image-count="1"] .grid-item img {
-            max-height: 90vh;
-        }
-        .grid-item.picked {
-            border-color: #4ade80;
-            background: rgba(74, 222, 128, 0.1);
-        }
-        .flag-pill {
-            background: #404040;
-            color: #e0e0e0;
-            border: 1px solid #555;
-        }
-        .flag-pill:hover {
-            background: #505050;
-            border-color: #666;
-        }
-        .flag-pill-active {
-            background: #4a9eff;
-            color: #000;
-            border: 1px solid #4a9eff;
-            font-weight: 500;
-        }
-        .flag-pill-active:hover {
-            background: #6ab0ff;
-            border-color: #6ab0ff;
-        }
-    </style>
-</head>
-<body>
-    <div x-data="app()" x-init="init()" @keydown.window="handleKeydown($event)">
-        <div style="background: #2a2a2a; padding: 12px 20px; border-bottom: 1px solid #404040;">
-            <div style="font-size: 16px; font-weight: 500; color: #e0e0e0; word-break: break-all;">
-                <span x-text="currentGroup ? currentGroup.original.name : 'No images'"></span>
-            </div>
-        </div>
-        <div class="container">
-            <!-- Left Panel: Original Image -->
-            <div class="panel left-panel">
-                <div class="image-display" :class="{ picked: originalPicked }" style="position: relative;">
-                    <template x-if="currentGroup">
-                        <img :src="`/image/${currentGroup.original.name}`" :alt="currentGroup.original.name">
-                        <template x-if="imageFlagAssignments[currentGroup.original.name]">
-                            <div style="position: absolute; bottom: 12px; left: 12px; background: #4a9eff; color: #000; padding: 4px 8px; border-radius: 4px; font-size: 12px; font-weight: bold;">
-                                <span x-text="imageFlagAssignments[currentGroup.original.name]"></span>
-                            </div>
-                        </template>
-                    </template>
-                    <template x-if="!currentGroup">
-                        <div style="text-align: center; color: #888;">No images</div>
-                    </template>
-                </div>
-                <div class="image-info">
-                    <template x-if="currentGroup">
-                        <div>
-                            <div style="display: flex; align-items: center; gap: 8px;">
-                                <strong>Position:</strong>
-                                <input 
-                                    type="number" 
-                                    :value="currentGroupIdx + 1"
-                                    @keyup.enter="jumpToIndex($event)"
-                                    @change="jumpToIndex($event)"
-                                    min="1" 
-                                    :max="imageGroups.length"
-                                    style="width: 90px; background: #404040; color: #e0e0e0; border: 1px solid #555; padding: 4px 8px; border-radius: 4px; font-size: 14px;"
-                                >
-                                <span style="color: #888;">/ <span x-text="imageGroups.length"></span></span>
-                            </div>
-                            <template x-if="availableFlags.length > 0">
-                                <div style="margin-top: 10px;">
-                                    <strong style="font-size: 13px;">Flags:</strong>
-                                    <div style="display: flex; flex-wrap: wrap; gap: 6px; margin-top: 6px;">
-                                        <template x-for="(flag, idx) in availableFlags" :key="idx">
-                                            <div 
-                                                @click="handleFlagAssignment(flag)"
-                                                :class="imageFlagAssignments[currentGroup.original.name] === flag ? 'flag-pill-active' : 'flag-pill'"
-                                                style="cursor: pointer; padding: 4px 8px; border-radius: 4px; font-size: 12px; white-space: nowrap; transition: all 0.2s;"
-                                            >
-                                                <span x-text="`${flag} (${idx + 1})`"></span>
-                                            </div>
-                                        </template>
-                                    </div>
-                                </div>
-                            </template>
-                        </div>
-                    </template>
-                </div>
-            </div>
-
-            <!-- Right Panel: Edit Variations Grid -->
-            <div class="panel right-panel">
-                <template x-if="currentGroup && currentGroup.edits.length > 0">
-                    <div class="grid" :data-image-count="currentGroup.edits.length">
-                        <template x-for="(edit, idx) in currentGroup.edits" :key="idx">
-                            <div 
-                                class="grid-item"
-                                :class="{ picked: pickedEdits.includes(idx) }"
-                                @click="toggleEditPick(idx)"
-                            >
-                                <img :src="`/image/${edit.name}`" :alt="`Edit ${idx + 1}`">
-                            </div>
-                        </template>
-                    </div>
-                </template>
-                <template x-if="!currentGroup || currentGroup.edits.length === 0">
-                    <div style="color: #888; text-align: center; padding: 40px;">No variations</div>
-                </template>
-            </div>
-        </div>
-
-        <!-- Help Dialog -->
-        <dialog id="helpDialog">
-            <h2>Keyboard Shortcuts</h2>
-            <div style="margin-top: 15px; font-size: 14px; line-height: 1.8;">
-                <div style="margin-bottom: 10px;">
-                    <span style="background: #404040; padding: 2px 6px; border-radius: 3px; font-weight: bold;">P</span>
-                    Toggle pick (all edits or hover one)
-                </div>
-                <div style="margin-bottom: 10px;">
-                    <span style="background: #404040; padding: 2px 6px; border-radius: 3px; font-weight: bold;">←→</span>
-                    Navigate any image
-                </div>
-                <div style="margin-bottom: 10px;">
-                    <span style="background: #404040; padding: 2px 6px; border-radius: 3px; font-weight: bold;">K</span>
-                    Skip to next pending
-                </div>
-                <div style="margin-top: 15px; font-size: 13px; color: #aaa;">
-                    <strong>Flags:</strong>
-                </div>
-                <div>
-                    <span style="background: #404040; padding: 2px 6px; border-radius: 3px; font-weight: bold;">1-9</span>
-                    Assign/unassign to flag
-                </div>
-                <div>
-                    <span style="background: #404040; padding: 2px 6px; border-radius: 3px; font-weight: bold;">/</span>
-                    Toggle reject flag
-                </div>
-            </div>
-            <div style="margin-top: 20px; text-align: right;">
-                <button @click="closeHelp()" style="background: #404040; color: #e0e0e0; border: none; padding: 8px 16px; border-radius: 4px; cursor: pointer;">Close</button>
-            </div>
-        </dialog>
-    </div>
-
-    <script>
-        function app() {
-            return {
-                imageGroups: [],
-                currentGroupIdx: 0,
-                currentGroup: null,
-                pickedEdits: [],
-                originalPicked: false,
-                groupStatus: {},
-                availableFlags: [],
-                imageFlagAssignments: {},
-
-                async init() {
-                    // Start heartbeat
-                    this.startHeartbeat();
-
-                    // Track mouse position globally for hover detection
-                    document.addEventListener('mousemove', (e) => {
-                        window.lastMouseEvent = e;
-                    });
-
-                    // Notify server when tab/window is closed
-                    window.addEventListener('beforeunload', () => {
-                        fetch('/api/shutdown', { method: 'POST' }).catch(() => {});
-                    });
-
-                    // Load images
-                    await this.loadImages();
-
-                    // Load first group
-                    this.loadGroup(0);
-                },
-
-                async loadImages() {
-                    try {
-                        const resp = await fetch('/api/images');
-                        const data = await resp.json();
-                        this.imageGroups = data.groups;
-                        this.availableFlags = data.flags_list || [];
-                        this.groupStatus = {};
-                        this.imageFlagAssignments = data.flag_assignments || {};
-                        this.imageGroups.forEach((_, idx) => {
-                            this.groupStatus[idx] = 'pending';
-                        });
-                    } catch (err) {
-                        console.error('Error loading images:', err.message);
-                    }
-                },
-
-                loadGroup(idx) {
-                    if (idx < 0 || idx >= this.imageGroups.length) return;
-                    this.currentGroupIdx = idx;
-                    this.currentGroup = this.imageGroups[idx];
-                    this.pickedEdits = [];
-                    this.originalPicked = false;
-                },
-
-                getGroupStatus(idx) {
-                    return this.groupStatus[idx] || 'pending';
-                },
-
-                jumpToIndex(event) {
-                    const value = parseInt(event.target.value);
-                    if (!isNaN(value) && value >= 1 && value <= this.imageGroups.length) {
-                        this.loadGroup(value - 1);
-                    }
-                },
-
-                toggleEditPick(idx) {
-                    if (this.pickedEdits.includes(idx)) {
-                        this.pickedEdits = this.pickedEdits.filter(i => i !== idx);
-                    } else {
-                        this.pickedEdits.push(idx);
-                    }
-                },
-
-                getHoveredElement() {
-                    // Find the element currently under the mouse pointer
-                    const mouseEvent = window.lastMouseEvent;
-                    if (!mouseEvent) return null;
-                    return document.elementFromPoint(mouseEvent.clientX, mouseEvent.clientY);
-                },
-
-                async handleKeydown(e) {
-                    // Skip keybindings if focused on input/textarea
-                    const target = e.target;
-                    const isInput = target.tagName === 'INPUT' || target.tagName === 'TEXTAREA' || target.contentEditable === 'true';
-                    if (isInput) {
-                        return;
-                    }
-
-                    const key = e.key.toLowerCase();
-
-                    if (key === 'p') {
-                        e.preventDefault();
-                        await this.handlePick();
-                    } else if (key === '?') {
-                        e.preventDefault();
-                        this.toggleHelp();
-                    } else if (key === '/') {
-                        e.preventDefault();
-                        await this.handleFlagAssignment('reject');
-                    } else if (key === 'arrowleft') {
-                        e.preventDefault();
-                        let nextIdx = this.currentGroupIdx - 1;
-                        if (nextIdx < 0) nextIdx = this.imageGroups.length - 1;
-                        this.loadGroup(nextIdx);
-                    } else if (key === 'arrowright') {
-                        e.preventDefault();
-                        let nextIdx = this.currentGroupIdx + 1;
-                        if (nextIdx >= this.imageGroups.length) nextIdx = 0;
-                        this.loadGroup(nextIdx);
-                    } else if (key === 'k') {
-                        e.preventDefault();
-                        for (let i = this.currentGroupIdx + 1; i < this.imageGroups.length; i++) {
-                            if (this.getGroupStatus(i) === 'pending') {
-                                this.loadGroup(i);
-                                return;
-                            }
-                        }
-                    } else if (key >= '1' && key <= '9') {
-                        e.preventDefault();
-                        const flagIdx = parseInt(key) - 1;
-                        if (flagIdx < this.availableFlags.length) {
-                            const flagName = this.availableFlags[flagIdx];
-                            await this.handleFlagAssignment(flagName);
-                        }
-                    }
-                },
-
-                async handlePick() {
-                    if (!this.currentGroup) return;
-
-                    const currentStatus = this.getGroupStatus(this.currentGroupIdx);
-                    
-                    // If already picked, unpick it
-                    if (currentStatus === 'picked') {
-                        try {
-                            const resp = await fetch('/api/unpick', {
-                                method: 'POST',
-                                headers: { 'Content-Type': 'application/json' },
-                                body: JSON.stringify({
-                                    original: this.currentGroup.original.name
-                                })
-                            });
-                            if (resp.ok) {
-                                this.groupStatus[this.currentGroupIdx] = 'pending';
-                                this.originalPicked = false;
-                                this.pickedEdits = [];
-                            }
-                        } catch (err) {
-                            console.error('Error unpicking:', err);
-                        }
-                        return;
-                    }
-
-                    const hoveredEl = this.getHoveredElement();
-                    const isHoveringOriginal = hoveredEl?.closest('.image-display:not(.right-panel *)');
-
-                    // If hovering over original, pick just the original
-                    if (isHoveringOriginal) {
-                        try {
-                            const resp = await fetch('/api/pick', {
-                                method: 'POST',
-                                headers: { 'Content-Type': 'application/json' },
-                                body: JSON.stringify({
-                                    original: this.currentGroup.original.name,
-                                    edits: [],
-                                    includeOriginal: true
-                                })
-                            });
-                            if (resp.ok) {
-                                this.originalPicked = true;
-                            }
-                        } catch (err) {
-                            console.error('Error picking original:', err);
-                        }
-                        return;
-                    }
-
-                    // If hovering on edit, pick just that edit
-                    const hoveredGridItem = hoveredEl?.closest('.grid-item');
-                    if (hoveredGridItem && this.currentGroup.edits.length > 0) {
-                        const idx = Array.from(hoveredGridItem.parentElement.querySelectorAll('.grid-item')).indexOf(hoveredGridItem);
-                        if (idx !== -1) {
-                            const editName = this.currentGroup.edits[idx].name;
-                        try {
-                            const resp = await fetch('/api/pick', {
-                                method: 'POST',
-                                headers: { 'Content-Type': 'application/json' },
-                                body: JSON.stringify({
-                                    original: this.currentGroup.original.name,
-                                    edits: [editName],
-                                    includeOriginal: false
-                                })
-                            });
-                            if (resp.ok) {
-                                this.pickedEdits.push(idx);
-                            }
-                        } catch (err) {
-                            console.error('Error picking edit:', err);
-                        }
-                        return;
-                        }
-                    }
-
-                    // If not hovering, pick all selected edits and advance
-                    let editsToPick = this.pickedEdits.map(i => this.currentGroup.edits[i].name);
-
-                    // If no edits selected but there's exactly 1 edit, pick it automatically
-                    if (editsToPick.length === 0 && this.currentGroup.edits.length === 1) {
-                        editsToPick = [this.currentGroup.edits[0].name];
-                    }
-
-                    // Only proceed if there are edits to pick
-                    if (editsToPick.length === 0) return;
-
-                    try {
-                        const resp = await fetch('/api/pick', {
-                            method: 'POST',
-                            headers: { 'Content-Type': 'application/json' },
-                            body: JSON.stringify({
-                                original: this.currentGroup.original.name,
-                                edits: editsToPick,
-                                includeOriginal: false
-                            })
-                        });
-                        if (resp.ok) {
-                            this.groupStatus[this.currentGroupIdx] = 'picked';
-                            this.loadGroup(this.currentGroupIdx + 1);
-                        }
-                    } catch (err) {
-                        console.error('Error picking edits:', err);
-                    }
-                },
-
-                startHeartbeat() {
-                    setInterval(async () => {
-                        try {
-                            await fetch('/api/heartbeat', { method: 'POST' });
-                        } catch (err) {
-                            console.error('Heartbeat failed:', err);
-                        }
-                    }, 1000);
-                },
-
-                toggleHelp() {
-                    const dialog = document.getElementById('helpDialog');
-                    if (dialog.open) {
-                        dialog.close();
-                    } else {
-                        dialog.showModal();
-                    }
-                },
-
-                closeHelp() {
-                    document.getElementById('helpDialog').close();
-                },
-
-                async handleFlagAssignment(flagName) {
-                    if (!this.currentGroup) return;
-
-                    const imageKey = this.currentGroup.original.name;
-                    
-                    // Toggle: if already assigned to this flag, unassign
-                    if (this.imageFlagAssignments[imageKey] === flagName) {
-                        try {
-                            await fetch('/api/unassign-flag', {
-                                method: 'POST',
-                                headers: { 'Content-Type': 'application/json' },
-                                body: JSON.stringify({
-                                    original: imageKey,
-                                    flag: flagName
-                                })
-                            });
-                            delete this.imageFlagAssignments[imageKey];
-                        } catch (err) {
-                            console.error('Error unassigning flag:', err);
-                        }
-                    } else {
-                        // Assign to new flag
-                        try {
-                            const resp = await fetch('/api/assign-flag', {
-                                method: 'POST',
-                                headers: { 'Content-Type': 'application/json' },
-                                body: JSON.stringify({
-                                    original: imageKey,
-                                    flag: flagName
-                                })
-                            });
-                            if (resp.ok) {
-                                this.imageFlagAssignments[imageKey] = flagName;
-                            }
-                        } catch (err) {
-                            console.error('Error assigning flag:', err);
-                        }
-                    }
-                }
-            };
-        }
-    </script>
-</body>
-</html>
-"""
-
-
-@app.get("/")
-def index() -> str:
-    return HTML_TEMPLATE
-
-
-@app.get("/api/images")
-def api_images():
-    """Return grouped images and current flag assignments."""
-    global image_groups, flags, flag_dirs
-    result = []
-    for group in image_groups:
-        result.append(
-            {
-                "base": group["base"],
-                "original": {"name": group["original"].name, "path": str(group["original"])},
-                "edits": [{"name": e.name, "path": str(e)} for e in group["edits"]],
-            }
-        )
-
-    # Build flag assignments by checking which flag directories contain which original images
-    flag_assignments = {}
-    for flag, flag_dir in flag_dirs.items():
-        if flag_dir.exists():
-            for item in flag_dir.iterdir():
-                if item.is_file():
-                    # Map the file name to its flag
-                    flag_assignments[item.name] = flag
-
-    return {"groups": result, "flags_list": flags, "flag_assignments": flag_assignments}
-
-
-@app.get("/image/<filename>")
-def serve_image(filename: str):
-    """Serve an image file safely."""
-    try:
-        target = safe_image_path(filename)
-        return bottle.static_file(target.name, root=str(target.parent), mimetype="image/jpeg")
-    except ValueError:
-        bottle.response.status = 403
-        return {"error": "Unauthorized path"}
-
-
-@app.post("/api/pick")
-def api_pick():
-    """Hard link picked images to _picks directory."""
-    global image_groups, current_group_idx, picks_dir
-
-    # Create directory before writing
-    make_dirs(picks_dir)
-
-    data = bottle.request.json
-    original_name = data.get("original")
-    edits_names = data.get("edits", [])
-    include_original = data.get("includeOriginal", False)
-
-    files_to_pick = []
-
-    # Add original if requested
-    if include_original:
-        for img in all_images:
-            if img.name == original_name:
-                files_to_pick.append(img)
-                break
-
-    # Add edits by finding them in all_images
-    for edit_name in edits_names:
-        for img in all_images:
-            if img.name == edit_name:
-                files_to_pick.append(img)
-                break
-
-    try:
-        hardlink_images(files_to_pick, picks_dir)
-        return {"status": "ok"}
-    except Exception as e:
-        bottle.response.status = 500
-        return {"error": str(e)}
-
-
-@app.post("/api/unpick")
-def api_unpick():
-    """Remove picked images from _picks directory."""
-    global picks_dir
-
-    data = bottle.request.json
-    original_name = data.get("original")
-
-    try:
-        # Find and remove the original and any edits from picks directory
-        if picks_dir.exists():
-            for item in picks_dir.iterdir():
-                if item.is_file() and (item.name == original_name or item.stem.startswith(original_name.rsplit(".", 1)[0] + "-edit")):
-                    item.unlink()
-        return {"status": "ok"}
-    except Exception as e:
-        bottle.response.status = 500
-        return {"error": str(e)}
-
-
-@app.post("/api/assign-flag")
-def api_assign_flag():
-    """Assign an image to a flag."""
-    global flag_dirs, all_images
-
-    data = bottle.request.json
-    original_name = data.get("original")
-    flag_name = data.get("flag")
-
-    # Find original file
-    original_path = None
-    for img in all_images:
-        if img.name == original_name:
-            original_path = img
-            break
-
-    if not original_path or flag_name not in flag_dirs:
-        bottle.response.status = 400
-        return {"error": "Invalid image or flag"}
-
-    # Create directory before writing
-    make_dirs(flag_dirs[flag_name])
-
-    try:
-        hardlink_images([original_path], flag_dirs[flag_name])
-        return {"status": "ok"}
-    except Exception as e:
-        bottle.response.status = 500
-        return {"error": str(e)}
-
-
-@app.post("/api/unassign-flag")
-def api_unassign_flag():
-    """Remove an image from a flag."""
-    global flag_dirs
-
-    data = bottle.request.json
-    original_name = data.get("original")
-    flag_name = data.get("flag")
-
-    if flag_name not in flag_dirs:
-        bottle.response.status = 400
-        return {"error": "Invalid flag"}
-
-    try:
-        target_file = flag_dirs[flag_name] / original_name
-        if target_file.exists():
-            target_file.unlink()
-        return {"status": "ok"}
-    except Exception as e:
-        bottle.response.status = 500
-        return {"error": str(e)}
-
-
-@app.post("/api/heartbeat")
-def api_heartbeat():
-    """Update last heartbeat timestamp."""
-    global last_heartbeat
-    last_heartbeat = datetime.now()
-    return {"status": "ok"}
-
-
-@app.post("/api/shutdown")
-def api_shutdown():
-    """Shutdown the server."""
-    global should_exit
-    should_exit = True
-    return {"status": "ok"}
-
-
-@functools.cache
-def make_dirs(path: Path) -> None:
-    """Create directory if it doesn't exist."""
-    path.mkdir(parents=True, exist_ok=True)
-
-
-def heartbeat_monitor() -> None:
-    """Monitor for shutdown signal."""
-    global should_exit
-
-    while not should_exit:
-        time.sleep(0.5)
-
-    print("\nShutting down server.", file=sys.stderr)
-    os._exit(0)
-
-
-def main() -> None:
-    global all_images, image_groups, last_heartbeat, picks_dir, flags, flag_dirs
-
-    parser = argparse.ArgumentParser(description="Image culling app for AI image2image transforms")
-    parser.add_argument("paths", nargs="+", help="Image files or folders to cull")
-    parser.add_argument("--picks-dir", default=None, help="Directory for picked images (default: $cwd/_picks)")
-    parser.add_argument("--rejects-dir", default=None, help="Directory for rejected images (default: $cwd/_rejects)")
-    parser.add_argument("--flags", default=None, help="Comma-separated flag names (e.g., 'a,b,c,d')")
-    parser.add_argument("--flags-dir", default=None, help="Directory for flag subdirectories (default: $cwd)")
-
-    args = parser.parse_args()
-
-    # Set directories
-    cwd = Path.cwd()
-    picks_dir = Path(args.picks_dir or (cwd / "_picks"))
-    rejects_dir = Path(args.rejects_dir or (cwd / "_rejects"))
-    flags_dir = Path(args.flags_dir or cwd)
-
-    # Set up flags
-    if args.flags:
-        flags = [f.strip() for f in args.flags.split(",")]
-        for flag in flags:
-            flag_dir = flags_dir / f"_picks_flag_{flag}"
-            flag_dirs[flag] = flag_dir
-
-    # Always add reject as a flag at the end
-    flags.append("reject")
-    flag_dirs["reject"] = rejects_dir
-
-    # Discover images
-    images = discover_images(args.paths)
-    all_images = set(images)
-
-    if not images:
-        print("No images found in provided paths.", file=sys.stderr)
-        sys.exit(1)
-
-    # Group images
-    image_groups = group_images(images)
-
-    if not image_groups:
-        print("No original images found (looking for files matching naming convention).", file=sys.stderr)
-        sys.exit(1)
-
-    print(f"Found {len(image_groups)} original image(s) with variations", file=sys.stderr)
-
-    # Find available port
-    port = find_available_port()
-    url = f"http://127.0.0.1:{port}"
-
-    # Start heartbeat monitor
-    monitor_thread = threading.Thread(target=heartbeat_monitor, daemon=True)
-    monitor_thread.start()
-
-    # Open browser
-    print(f"Opening browser at {url}", file=sys.stderr)
-    webbrowser.open(url)
-
-    # Start Flask server
-    last_heartbeat = datetime.now()
-    bottle.run(app, host="127.0.0.1", port=port, quiet=True)
-
-
-if __name__ == "__main__":
-    main()

download_imageset.py

@@ -7,8 +7,11 @@ import dataclasses
 import logging
 import os
 import re
+import shlex
+import shutil
 import struct
 import subprocess
+import tempfile
 import urllib.parse
 import zipfile
 from functools import partial
@@ -17,7 +20,7 @@ from typing import Callable
 
 import click
 import httpx
-from PIL import Image
+from PIL import Image, ImageOps
 
 run_command = partial(
     subprocess.run,
@@ -200,13 +203,78 @@ def _has_images(d: Path) -> bool:
     return any(f.suffix.lower() in IMAGE_EXTS for f in d.rglob("*") if f.is_file())
 
 
+_TRICKLEBAR_APP = Path("/Applications/TrickleBar.app")
+
+
+def download_url(url: str, destination: Path) -> None:
+    destination = destination.resolve()
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    if _TRICKLEBAR_APP.is_dir():
+        logging.info(f"downloading with TrickleBar → {destination}")
+        download_url_with_tricklebar(url=url, destination=destination)
+    else:
+        logging.info(f"downloading with aria2 → {destination}")
+        download_url_with_aria2(url=url, destination=destination)
+
+
+def download_url_with_aria2(url: str, destination: Path) -> None:
+    run_command(args=[
+        "aria2c", "--continue=true", f"--dir={destination.parent}",
+        f"--out={destination.name}", url,
+    ])
+
+
+def download_url_with_tricklebar(url: str, destination: Path) -> None:
+    with tempfile.TemporaryDirectory(prefix="imageset-") as temporary_directory:
+        directory = Path(temporary_directory)
+        completion_pipe = directory / "completion"
+        callback = directory / "on_done.sh"
+        os.mkfifo(path=completion_pipe, mode=0o600)
+        callback.write_text(data=(
+            "#!/bin/sh\n"
+            f"[ -p {shlex.quote(str(completion_pipe))} ] || exit 0\n"
+            f"printf '%s\\0' \"$1\" \"$2\" > {shlex.quote(str(completion_pipe))}\n"
+        ))
+        callback.chmod(mode=0o700)
+        query = urllib.parse.urlencode(query={
+            "on_done": str(callback), "url": url,
+            "dir": str(destination.parent), "out": destination.name, "continue": "true",
+        }, quote_via=urllib.parse.quote)
+        logging.info("waiting for TrickleBar; press Ctrl+C to stop waiting")
+        run_command(args=["open", f"tricklebar://add-download?{query}"])
+        with completion_pipe.open(mode="rb") as completion:
+            status, downloaded_path, _ = completion.read().split(b"\0", maxsplit=2)
+        if status != b"complete":
+            raise click.ClickException(
+                f"TrickleBar download failed: {os.fsdecode(status)}"
+            )
+        if not downloaded_path or not Path(os.fsdecode(downloaded_path)).is_file():
+            raise click.ClickException("TrickleBar completed without a downloaded file")
+        if Path(os.fsdecode(downloaded_path)).resolve() != destination.resolve():
+            raise click.ClickException(
+                f"TrickleBar downloaded to {os.fsdecode(downloaded_path)} instead of {destination}; "
+                "update TrickleBar to a version supporting dir/out URL options"
+            )
+        if destination.with_name(f"{destination.name}.aria2").exists():
+            raise click.ClickException(f"TrickleBar download is unfinished: {destination}")
+
+
 def _needs_resize(d: Path) -> bool:
-    images = [f for f in d.rglob("*") if f.is_file() and f.suffix.lower() in IMAGE_EXTS]
+    images = [f for f in d.rglob("*") if f.is_file() and f.suffix.lower() in IMAGE_EXTS and not f.is_relative_to(d / "resized")]
     if not images:
         return False
     mean_mb = sum(f.stat().st_size for f in images) / len(images)
     files_are_large = mean_mb >= 2 * 1_048_567
-    images_are_large = any(max(read_dims(img)) > 5000 for img in images)
+    images_are_large = False
+    for image in images:
+        try:
+            if max(read_dims(image)) > 5000:
+                images_are_large = True
+                break
+        except Exception as error:
+            logging.warning(f"could not read dimensions of {image}; trying resizer: {error}")
+            images_are_large = True
+            break
 
     return files_are_large or images_are_large
 
@@ -216,46 +284,114 @@ def _needs_resize(d: Path) -> bool:
 # ---------------------------------------------------------------------------
 
 
-def _extract_and_process(output_dir: Path) -> None:
+ImageProcessor = Callable[..., None]
-    archives = [
-        f for pattern in ("*.zip", "*.rar", "*.7z") for f in output_dir.glob(pattern)
-    ]
-    for archive in archives:
-        print(f"extracting {archive.name} …")
-        run_command(["unar", "-o", str(output_dir), str(archive)])
-        move_to_trash([archive])
 
-    subdirs = [d for d in output_dir.iterdir() if d.is_dir() and d.name != "resized"]
-    targets = subdirs if subdirs else [output_dir]
 
-    to_process = [d for d in targets if _needs_resize(d)]
+def detect_resizer() -> ImageProcessor:
-    if not to_process:
+    vips_binary = shutil.which(cmd="vips")
-        print("no dirs need resizing")
+    if vips_binary:
+        return partial(resize_with_vips, vips_binary=vips_binary)
+    return resize_with_pillow
+
+
+def resize_with_vips(image_path: Path, output_path: Path, vips_binary: str) -> None:
+    try:
+        subprocess.run(
+            args=[
+                vips_binary,
+                "thumbnail",
+                str(image_path),
+                f"{output_path}[Q=75]",
+                "5000",
+                "--height",
+                "5000",
+                "--size",
+                "down",
+            ],
+            check=True,
+            capture_output=True,
+        )
+    except (OSError, subprocess.CalledProcessError) as error:
+        logging.warning(f"vips failed for {image_path}; using Pillow: {error}")
+        resize_with_pillow(image_path=image_path, output_path=output_path)
+
+
+def resize_with_pillow(image_path: Path, output_path: Path) -> None:
+    with Image.open(fp=image_path) as image:
+        with ImageOps.exif_transpose(image=image) as oriented:
+            oriented.thumbnail(size=(5000, 5000), resample=Image.Resampling.LANCZOS)
+            with oriented.convert(mode="RGB") as resized:
+                resized.save(fp=output_path, format="JPEG", quality=75)
+
+
+def process_imageset(path: Path, processor: ImageProcessor) -> None:
+    if not path.exists():
+        logging.warning(f"skipping {path}: does not exist")
+        return
+    if path.is_file() and path.suffix.lower() in {".zip", ".rar", ".7z"}:
+        output_dir = path.with_suffix("")
+        archives = [path]
+    elif path.is_dir():
+        output_dir = path
+        archives = [f for f in path.iterdir() if f.is_file() and f.suffix.lower() in {".zip", ".rar", ".7z"}]
     else:
+        logging.warning(f"skipping {path}: expected a directory or archive")
+        return
+    if output_dir.exists() and not output_dir.is_dir():
+        logging.warning(f"skipping {path}: {output_dir} is not a directory")
+        return
+
+    for archive in archives:
+        if archive.with_name(f"{archive.name}.aria2").exists():
+            logging.warning(f"skipping {archive}: download is unfinished")
+            return
+        logging.info(f"extracting {archive} → {output_dir}")
+        if archive.suffix.lower() == ".zip":
+            try:
+                with zipfile.ZipFile(file=archive) as zipped:
+                    zipped.extractall(path=output_dir)
+            except zipfile.BadZipFile:
+                logging.warning(f"skipping {archive}: invalid ZIP file")
+                return
+        else:
+            run_command(args=["unar", "--output-directory", str(output_dir), str(archive)])
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    failed_images: set[Path] = set()
+    if _needs_resize(output_dir):
         resized_dir = output_dir / "resized"
         resized_dir.mkdir(exist_ok=True)
-        originals = [
+        originals = [f for f in output_dir.rglob("*") if is_image(f) and not f.is_relative_to(resized_dir)]
-            f
+        logging.info(f"resizing {len(originals)} image(s) → {resized_dir}")
-            for d in to_process
+        resized_originals: list[Path] = []
-            for f in d.rglob("*")
-            if f.is_file() and f.suffix.lower() in IMAGE_EXTS
-        ]
-        print(f"resizing {len(originals)} image(s) → {resized_dir}")
         for img in originals:
-            run_command(
+            output_path = resized_dir / f"{img.stem}.resized.jpg"
-                [
+            if output_path.is_file() and output_path.stat().st_size > 0:
-                    "imgz",
+                logging.info(f"skipping {img}: {output_path} already resized")
-                    "resize",
+                resized_originals.append(img)
-                    "-o",
+                continue
-                    str(resized_dir / f"{img.stem}.resized.jpg"),
+            try:
-                    str(img),
+                processor(image_path=img, output_path=output_path)
-                ]
+            except Exception as error:
+                logging.warning(f"skipping {img}: resizing failed: {error}")
+                failed_images.add(img)
+                output_path.unlink(missing_ok=True)
+                continue
+            resized_originals.append(img)
+        if resized_originals:
+            logging.info(f"trashing {len(resized_originals)} original(s)")
+            move_to_trash(file_paths=resized_originals)
+    else:
+        logging.info(f"skipping resizing {output_dir}: already within size limits")
+
+    rename_children(
+        source_dir=output_dir,
+        flatten=True,
+        filter_fn=lambda image: is_image(image) and image not in failed_images,
     )
-        if originals:
+    remove_empty_dirs(root_dir=output_dir)
-            print(f"trashing {len(originals)} original(s)")
+    if archives and not failed_images:
-            move_to_trash(originals)
+        move_to_trash(file_paths=archives)
-    rename_children(output_dir, flatten=True)
-    remove_empty_dirs(output_dir)
 
 
 def remove_empty_dirs(root_dir: Path) -> None:
@@ -267,7 +403,7 @@ def remove_empty_dirs(root_dir: Path) -> None:
     for d in removed:
         d.rmdir()
     if removed:
-        print(f"removed {len(removed)} empty dir(s) in {root_dir}")
+        logging.info(f"removed {len(removed)} empty dir(s) in {root_dir}")
 
 
 def move_to_trash(file_paths: list[Path]) -> None:
@@ -348,15 +484,15 @@ def gallery_cmd(urls: tuple[str, ...], name: str | None, cwd: Path | None) -> No
     output_dir = (cwd or Path(".")) / resolved_name
 
     if output_dir.exists() and _has_images(output_dir):
-        print(f"skipping gallery-dl: {output_dir} already has images")
+        logging.info(f"skipping gallery-dl: {output_dir} already has images")
     else:
-        print(f"gallery-dl → {output_dir}")
+        logging.info(f"gallery-dl → {output_dir}")
         args = ["uvx", "gallery-dl"]
         if name or cwd:
             args += ["-D", str(output_dir)]
         run_command(args + urls)
 
-    _extract_and_process(output_dir)
+    process_imageset(path=output_dir, processor=detect_resizer())
 
 
 @cli.command("filehost")
@@ -370,29 +506,41 @@ def filehost_cmd(urls: tuple[str, ...], name: str | None, cwd: Path, token: str)
     output_dir.mkdir(parents=True, exist_ok=True)
 
     for url in urls:
-        print(f"unrestricting {url}")
+        logging.info(f"unrestricting {url}")
         dl_url = resolve_download_url_with_alldebrid(token, url)
         meta = _resolve_url(dl_url)
         if meta.suggested_extension in _VIDEO_EXTS:
             video_path = (cwd or Path(".")) / (resolved_name + meta.suggested_extension)
-            print(f"video detected → {video_path}")
+            logging.info(f"video detected → {video_path}")
-            if video_path.exists() and video_path.stat().st_size > 5 * 1024 * 1024:
+            if video_path.with_name(f"{video_path.name}.aria2").exists():
-                print(f"skipping: {video_path.name} already downloaded")
+                logging.warning(f"unfinished download; resuming {video_path}")
+            if (
+                video_path.exists()
+                and not video_path.with_name(f"{video_path.name}.aria2").exists()
+                and video_path.stat().st_size > 5 * 1024 * 1024
+            ):
+                logging.info(f"skipping: {video_path.name} already downloaded")
                 continue
-            run_command(["aria2c", "-o", str(video_path), dl_url])
+            download_url(url=dl_url, destination=video_path)
             continue
 
         dest = output_dir / meta.suggested_filename
-        if dest.exists() and dest.stat().st_size > 5 * 1024 * 1024:
+        if dest.with_name(f"{dest.name}.aria2").exists():
-            print(f"skipping: {dest.name} already downloaded")
+            logging.warning(f"unfinished download; resuming {dest}")
+        if (
+            dest.exists()
+            and not dest.with_name(f"{dest.name}.aria2").exists()
+            and dest.stat().st_size > 5 * 1024 * 1024
+        ):
+            logging.info(f"skipping: {dest.name} already downloaded")
             continue
-        print(f"downloading {meta.suggested_filename} → {output_dir}")
+        logging.info(f"downloading {meta.suggested_filename} → {output_dir}")
-        run_command(["aria2c", f"--dir={output_dir}", dl_url])
+        download_url(url=dl_url, destination=dest)
 
-    _extract_and_process(output_dir)
+    process_imageset(path=output_dir, processor=detect_resizer())
 
 def is_filehost(url: str) -> bool:
-    kws = ["rg.to", "rapidgator.net", "trbt.cc"]
+    kws = ["rg.to", "rapidgator.net", "trbt.cc", "katfile"]
     return any(kw in url for kw in kws)
 
 @cli.command("download")
@@ -415,54 +563,10 @@ def download_cmd(
 @cli.command("process")
 @click.argument("paths", nargs=-1, required=True, type=click.Path(path_type=Path))
 def process_cmd(paths: tuple[Path, ...]) -> None:
-    """Process directories and ZIP files, extracting ZIPs into sibling directories."""
+    """Process directories and archives, extracting archives into sibling directories."""
-    for p in paths:
+    processor = detect_resizer()
-        archive_path: Path | None = None
+    for path in paths:
-        if not p.exists():
+        process_imageset(path=path, processor=processor)
-            logging.warning(f"skipping {p}: does not exist")
-            continue
-        if p.is_file() and p.suffix.lower() == ".zip":
-            output_dir = p.with_suffix("")
-            if output_dir.exists() and not output_dir.is_dir():
-                logging.warning(f"skipping {p}: {output_dir} is not a directory")
-                continue
-            try:
-                with zipfile.ZipFile(file=p) as archive:
-                    logging.info(f"extracting {p} → {output_dir}")
-                    archive.extractall(path=output_dir)
-            except zipfile.BadZipFile:
-                logging.warning(f"skipping {p}: invalid ZIP file")
-                continue
-            archive_path = p
-            p = output_dir
-        if not p.is_dir():
-            logging.warning(f"skipping {p}: expected a directory or ZIP file")
-            continue
-        if _needs_resize(p):
-            resized_dir = p / "resized"
-            resized_dir.mkdir(exist_ok=True)
-            originals = [f for f in p.rglob("*") if is_image(f)]
-            logging.info(f"resizing {len(originals)} image(s) in {p} → {resized_dir}")
-            for img in originals:
-                run_command(
-                    [
-                        "imgz",
-                        "resize",
-                        "--output",
-                        str(resized_dir / f"{img.stem}.resized.jpg"),
-                        str(img),
-                    ]
-                )
-            if originals:
-                logging.info(f"trashing {len(originals)} original(s)")
-                move_to_trash(originals)
-        else:
-            logging.info(f"skipping {p}: already within size limits")
-
-        rename_children(p, flatten=True)
-        remove_empty_dirs(p)
-        if archive_path is not None:
-            move_to_trash(file_paths=[archive_path])
 
 
 if __name__ == "__main__":

download_imageset_test.py

@@ -3,9 +3,15 @@
 # requires-python = ">=3.11"
 # dependencies = ["pytest", "httpx", "pillow", "click"]
 # ///
+import shutil
+import subprocess
 import sys
+import threading
+import urllib.parse
 import zipfile
+from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
+from functools import partial
 from unittest.mock import Mock
 
 import pytest
@@ -43,6 +49,47 @@ def test_process_zip_and_directory(
     trash.assert_called_once_with(file_paths=[archive_path])
 
 
+@pytest.mark.parametrize("command", ["process", "gallery"])
+@pytest.mark.parametrize("processing_fails", [False, True])
+def test_process_archives_in_directory(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    command: str,
+    processing_fails: bool,
+) -> None:
+    directory = tmp_path / "album"
+    directory.mkdir()
+    archive_path = directory / "download.ZIP"
+    image_path = tmp_path / "photo.png"
+    Image.new(mode="RGB", size=(10, 10)).save(fp=image_path)
+    with zipfile.ZipFile(file=archive_path, mode="w") as archive:
+        if processing_fails:
+            archive.writestr(zinfo_or_arcname="nested/photo.png", data=b"invalid image")
+        else:
+            archive.write(filename=image_path, arcname="nested/photo.png")
+    trash = Mock()
+    monkeypatch.setattr(download_imageset, "move_to_trash", trash)
+    monkeypatch.setattr(download_imageset, "run_command", Mock())
+    args = (
+        ["process", str(directory)]
+        if command == "process"
+        else ["gallery", "https://example.com/album", "--cwd", str(tmp_path)]
+    )
+
+    result = CliRunner().invoke(cli=download_imageset.cli, args=args)
+
+    if processing_fails:
+        assert result.exit_code == 0, result.output
+        assert archive_path.is_file()
+        assert (directory / "nested" / "photo.png").read_bytes() == b"invalid image"
+        trash.assert_not_called()
+    else:
+        assert result.exit_code == 0, result.output
+        assert (directory / "album__0000.png").is_file()
+        assert not (directory / "nested").exists()
+        trash.assert_called_once_with(file_paths=[archive_path])
+
+
 @pytest.mark.parametrize("filename", ["missing", "unsupported.txt", "broken.zip"])
 def test_process_skips_invalid_inputs(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch, filename: str
@@ -65,7 +112,7 @@ def test_process_skips_invalid_inputs(
 
 
 def test_process_keeps_zip_when_processing_fails(
-    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
 ) -> None:
     archive_path = tmp_path / "gallery.zip"
     with zipfile.ZipFile(file=archive_path, mode="w") as archive:
@@ -77,7 +124,350 @@ def test_process_keeps_zip_when_processing_fails(
         cli=download_imageset.cli, args=["process", str(archive_path)]
     )
 
-    assert result.exit_code != 0
+    assert result.exit_code == 0, result.output
+    assert archive_path.is_file()
+    trash.assert_not_called()
+    assert "resizing failed" in caplog.text
+
+
+def test_process_continues_after_resize_failure(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+    directory = tmp_path / "album"
+    directory.mkdir()
+    failed_image = directory / "broken.png"
+    failed_image.write_bytes(data=b"broken image")
+    good_image = directory / "good.png"
+    Image.new(mode="RGB", size=(6000, 60)).save(fp=good_image)
+
+    def processor(image_path: Path, output_path: Path) -> None:
+        if image_path == failed_image:
+            output_path.write_bytes(data=b"partial JPEG")
+            raise OSError("cannot decode image")
+        download_imageset.resize_with_pillow(image_path=image_path, output_path=output_path)
+
+    trash = Mock()
+    monkeypatch.setattr(download_imageset, "detect_resizer", lambda: processor)
+    monkeypatch.setattr(download_imageset, "move_to_trash", trash)
+
+    result = CliRunner().invoke(cli=download_imageset.cli, args=["process", str(directory)])
+
+    assert result.exit_code == 0, result.output
+    assert failed_image.read_bytes() == b"broken image"
+    assert not (directory / "resized" / "broken.resized.jpg").exists()
+    with Image.open(fp=directory / "album__0001.jpg") as resized:
+        assert resized.size == (5000, 50)
+    trash.assert_called_once_with(file_paths=[good_image])
+    assert f"skipping {failed_image}: resizing failed: cannot decode image" in caplog.text
+
+
+@pytest.mark.parametrize("use_vips", [False, True])
+@pytest.mark.parametrize(
+    ("size", "mode", "orientation", "expected_size"),
+    [
+        ((6000, 60), "RGB", 1, (5000, 50)),
+        ((60, 6000), "RGB", 1, (50, 5000)),
+        ((40, 20), "RGBA", 1, (40, 20)),
+        ((40, 20), "RGB", 6, (20, 40)),
+    ],
+)
+def test_resize_image(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    use_vips: bool,
+    size: tuple[int, int],
+    mode: str,
+    orientation: int,
+    expected_size: tuple[int, int],
+) -> None:
+    vips_binary = shutil.which(cmd="vips") if use_vips else None
+    if use_vips and vips_binary is None:
+        pytest.skip("vips is not installed")
+    processor = (
+        partial(download_imageset.resize_with_vips, vips_binary=vips_binary)
+        if vips_binary
+        else download_imageset.resize_with_pillow
+    )
+    image_path = tmp_path / "source.png"
+    output_path = tmp_path / "resized.jpg"
+    with Image.new(mode=mode, size=size) as image:
+        exif = Image.Exif()
+        exif[274] = orientation
+        image.save(fp=image_path, exif=exif)
+
+    processor(image_path=image_path, output_path=output_path)
+
+    with Image.open(fp=output_path) as resized:
+        resized.load()
+        assert resized.format == "JPEG"
+        assert resized.mode == "RGB"
+        assert resized.size == expected_size
+    assert image_path.is_file()
+
+
+@pytest.mark.parametrize(
+    "error", [OSError("vips unavailable"), subprocess.CalledProcessError(1, "vips")]
+)
+def test_resize_image_falls_back_when_vips_fails(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, error: Exception
+) -> None:
+    image_path = tmp_path / "source.png"
+    output_path = tmp_path / "resized.jpg"
+    Image.new(mode="RGB", size=(6000, 60)).save(fp=image_path)
+    monkeypatch.setattr(download_imageset.subprocess, "run", Mock(side_effect=error))
+
+    download_imageset.resize_with_vips(
+        image_path=image_path, output_path=output_path, vips_binary="vips"
+    )
+
+    with Image.open(fp=output_path) as resized:
+        resized.load()
+        assert resized.size == (5000, 50)
+
+
+@pytest.mark.parametrize("vips_binary", [None, "/usr/bin/vips"])
+def test_process_selects_processor_once(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, vips_binary: str | None
+) -> None:
+    directories = [tmp_path / "first", tmp_path / "second"]
+    for directory in directories:
+        directory.mkdir()
+        Image.new(mode="RGB", size=(10, 10)).save(fp=directory / "photo.png")
+    lookup = Mock(return_value=vips_binary)
+    vips_processor = Mock()
+    pillow_processor = Mock()
+    monkeypatch.setattr(download_imageset.shutil, "which", lookup)
+    monkeypatch.setattr(download_imageset, "resize_with_vips", vips_processor)
+    monkeypatch.setattr(download_imageset, "resize_with_pillow", pillow_processor)
+    monkeypatch.setattr(download_imageset, "_needs_resize", Mock(return_value=True))
+    monkeypatch.setattr(download_imageset, "move_to_trash", Mock())
+
+    result = CliRunner().invoke(
+        cli=download_imageset.cli, args=["process", *map(str, directories)]
+    )
+
+    assert result.exit_code == 0, result.output
+    lookup.assert_called_once_with(cmd="vips")
+    selected_processor = vips_processor if vips_binary else pillow_processor
+    unused_processor = pillow_processor if vips_binary else vips_processor
+    assert selected_processor.call_count == 2
+    unused_processor.assert_not_called()
+    for directory, call in zip(directories, selected_processor.call_args_list):
+        expected = {
+            "image_path": directory / "photo.png",
+            "output_path": directory / "resized" / "photo.resized.jpg",
+        }
+        if vips_binary:
+            expected["vips_binary"] = vips_binary
+        assert call.kwargs == expected
+
+
+@pytest.mark.parametrize("command", ["process", "gallery"])
+@pytest.mark.parametrize("existing_output", [None, b"", b"already resized"])
+def test_resume_resizing(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    command: str,
+    existing_output: bytes | None,
+) -> None:
+    directory = tmp_path / "album"
+    resized_dir = directory / "resized"
+    resized_dir.mkdir(parents=True)
+    image_path = directory / "photo.png"
+    Image.new(mode="RGB", size=(6000, 60)).save(fp=image_path)
+    output_path = resized_dir / "photo.resized.jpg"
+    if existing_output is not None:
+        output_path.write_bytes(data=existing_output)
+    processor = Mock(wraps=download_imageset.resize_with_pillow)
+    trash = Mock()
+    monkeypatch.setattr(download_imageset, "detect_resizer", lambda: processor)
+    monkeypatch.setattr(download_imageset, "move_to_trash", trash)
+    if command == "process":
+        args = ["process", str(directory)]
+    else:
+        args = ["gallery", "https://example.com/album", "--cwd", str(tmp_path)]
+
+    result = CliRunner().invoke(cli=download_imageset.cli, args=args)
+
+    assert result.exit_code == 0, result.output
+    if existing_output:
+        processor.assert_not_called()
+        assert (directory / "album__0001.jpg").read_bytes() == existing_output
+    else:
+        processor.assert_called_once_with(image_path=image_path, output_path=output_path)
+        with Image.open(fp=directory / "album__0001.jpg") as resized:
+            assert resized.size == (5000, 50)
+    trash.assert_called_once_with(file_paths=[image_path])
+
+
+@pytest.mark.parametrize("extension", [".zip", ".mp4"])
+@pytest.mark.parametrize("unfinished", [False, True])
+def test_filehost_resumes_unfinished_downloads(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+    extension: str,
+    unfinished: bool,
+) -> None:
+    directory = tmp_path / "album"
+    directory.mkdir()
+    destination = (
+        directory / "download.zip" if extension == ".zip" else tmp_path / "album.mp4"
+    )
+    with destination.open(mode="wb") as output:
+        output.truncate(6 * 1024 * 1024)
+    control_file = destination.with_name(f"{destination.name}.aria2")
+    if unfinished:
+        control_file.touch()
+    url = "https://example.com/download"
+    monkeypatch.setattr(
+        download_imageset, "resolve_download_url_with_alldebrid", Mock(return_value=url)
+    )
+    monkeypatch.setattr(
+        download_imageset,
+        "_resolve_url",
+        Mock(return_value=download_imageset.UrlMeta(
+            suggested_filename=f"download{extension}",
+            suggested_extension=extension,
+            size_bytes=destination.stat().st_size,
+        )),
+    )
+    run_command = Mock()
+    monkeypatch.setattr(download_imageset, "_TRICKLEBAR_APP", tmp_path / "missing.app")
+    monkeypatch.setattr(download_imageset, "run_command", run_command)
+    monkeypatch.setattr(download_imageset, "process_imageset", Mock())
+
+    result = CliRunner().invoke(
+        cli=download_imageset.cli,
+        args=["filehost", url, "--name", "album", "--cwd", str(tmp_path), "--token", "test"],
+    )
+
+    assert result.exit_code == 0, result.output
+    if unfinished:
+        assert f"unfinished download; resuming {destination}" in caplog.text
+        run_command.assert_called_once_with(args=[
+            "aria2c", "--continue=true", f"--dir={destination.parent}",
+            f"--out={destination.name}", url,
+        ])
+    else:
+        assert "unfinished download; resuming" not in caplog.text
+        run_command.assert_not_called()
+
+
+def test_aria2_download_failure_propagates(monkeypatch: pytest.MonkeyPatch) -> None:
+    failure = subprocess.CalledProcessError(returncode=1, cmd="aria2c")
+    monkeypatch.setattr(download_imageset, "run_command", Mock(side_effect=failure))
+
+    with pytest.raises(subprocess.CalledProcessError):
+        download_imageset.download_url_with_aria2(
+            url="https://example.com/archive.zip", destination=Path("archive.zip")
+        )
+
+
+@pytest.mark.parametrize(
+    ("status", "file_state", "error"),
+    [
+        ("complete", "valid", None),
+        ("error", "valid", "download failed"),
+        ("complete", "missing", "without a downloaded file"),
+        ("complete", "wrong_path", "instead of"),
+        ("complete", "unfinished", "download is unfinished"),
+    ],
+)
+def test_tricklebar_waits_for_callback(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
+    status: str, file_state: str, error: str | None,
+) -> None:
+    destination = tmp_path / "download & photos.zip"
+    downloaded_path = tmp_path / "wrong.zip" if file_state == "wrong_path" else destination
+    if file_state != "missing":
+        downloaded_path.write_bytes(data=b"downloaded archive")
+    if file_state == "unfinished":
+        destination.with_name(f"{destination.name}.aria2").touch()
+    url = "https://example.com/photos.zip?token=a+b&other=value"
+    opened = threading.Event()
+    callback_paths: list[Path] = []
+
+    def open_download(args: list[str]) -> None:
+        assert args[0] == "open"
+        parsed = urllib.parse.urlparse(url=args[1])
+        assert "+" not in parsed.query
+        assert parsed.scheme == "tricklebar"
+        assert parsed.netloc == "add-download"
+        query = urllib.parse.parse_qs(qs=parsed.query)
+        assert query["url"] == [url]
+        assert query["dir"] == [str(destination.parent)]
+        assert query["out"] == [destination.name]
+        assert query["continue"] == ["true"]
+        callback_paths.append(Path(query["on_done"][0]))
+        opened.set()
+
+    monkeypatch.setattr(download_imageset, "run_command", open_download)
+
+    with ThreadPoolExecutor(max_workers=1) as executor:
+        download = executor.submit(
+            download_imageset.download_url_with_tricklebar, url=url, destination=destination
+        )
+        assert opened.wait(timeout=5)
+        assert not download.done()
+        callback = callback_paths[0]
+        subprocess.run(args=[str(callback), status, str(downloaded_path), "gid", url], check=True)
+        if error is None:
+            assert download.result(timeout=5) is None
+        else:
+            with pytest.raises(download_imageset.click.ClickException, match=error):
+                download.result(timeout=5)
+
+    assert not callback.parent.exists()
+
+
+@pytest.mark.parametrize("installed", [False, True])
+def test_download_selects_backend(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, installed: bool
+) -> None:
+    application = tmp_path / "TrickleBar.app"
+    if installed:
+        application.mkdir()
+    monkeypatch.setattr(download_imageset, "_TRICKLEBAR_APP", application)
+    aria2 = Mock()
+    tricklebar = Mock()
+    monkeypatch.setattr(download_imageset, "download_url_with_aria2", aria2)
+    monkeypatch.setattr(download_imageset, "download_url_with_tricklebar", tricklebar)
+    destination = tmp_path / "photos" / "album.zip"
+    url = "https://example.com/album.zip"
+
+    download_imageset.download_url(url=url, destination=destination)
+
+    assert destination.parent.is_dir()
+    selected = tricklebar if installed else aria2
+    unused = aria2 if installed else tricklebar
+    selected.assert_called_once_with(url=url, destination=destination.resolve())
+    unused.assert_not_called()
+
+
+@pytest.mark.parametrize("process_directory", [False, True])
+def test_process_skips_unfinished_archive(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+    process_directory: bool,
+) -> None:
+    archive_path = tmp_path / "gallery.zip"
+    with zipfile.ZipFile(file=archive_path, mode="w") as archive:
+        archive.writestr(zinfo_or_arcname="photo.png", data=b"image data")
+    archive_path.with_name(f"{archive_path.name}.aria2").touch()
+    trash = Mock()
+    monkeypatch.setattr(download_imageset, "move_to_trash", trash)
+
+    result = CliRunner().invoke(
+        cli=download_imageset.cli,
+        args=["process", str(tmp_path if process_directory else archive_path)],
+    )
+
+    assert result.exit_code == 0, result.output
+    assert "download is unfinished" in caplog.text
+    assert not (tmp_path / "photo.png").exists()
+    assert not (tmp_path / "gallery").exists()
     assert archive_path.is_file()
     trash.assert_not_called()
 

download_js_sourcemaps.py

@@ -1,194 +0,0 @@
-#!/usr/bin/env -S uv run --script
-# /// script
-# dependencies = ["playwright", "httpx"]
-# ///
-
-import argparse
-import json
-import logging
-import re
-import sys
-from pathlib import Path
-from typing import Optional
-from urllib.parse import urljoin, urlparse
-
-import httpx
-from playwright.sync_api import sync_playwright
-
-logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s")
-logger = logging.getLogger(__name__)
-
-
-def str_to_bool(value: str) -> bool:
-    """Convert string to boolean."""
-    if isinstance(value, bool):
-        return value
-    return value.lower() in ("true", "1", "yes", "on")
-
-
-def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(
-        description="Download JavaScript files and reconstruct sources from source maps",
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-    )
-    parser.add_argument("url", help="URL of the page to scrape")
-    parser.add_argument(
-        "--output-dir",
-        "-o",
-        type=Path,
-        default=Path("output"),
-        help="Output directory for reconstructed sources",
-    )
-    parser.add_argument(
-        "--headless",
-        type=str_to_bool,
-        default=True,
-        help="Run browser in headless mode",
-    )
-    return parser.parse_args()
-
-
-def get_script_urls(page_url: str, headless: bool = True) -> list[str]:
-    """Extract all script URLs from a page using Playwright."""
-    script_urls = []
-
-    with sync_playwright() as playwright:
-        browser = playwright.chromium.launch(headless=headless)
-        page = browser.new_page()
-
-        def handle_response(response):
-            if response.request.resource_type == "script":
-                script_urls.append(response.url)
-                logger.debug(f"Detected script: {response.url}")
-
-        page.on("response", handle_response)
-        logger.info("Navigating to page...")
-        page.goto(url=page_url)
-        logger.info("Waiting for DOM ready...")
-        page.wait_for_load_state(state="domcontentloaded")
-        logger.info("Waiting for network idle (30s timeout)...")
-        try:
-            page.wait_for_load_state(state="networkidle", timeout=30000)
-        except Exception as e:
-            logger.warning(f"Network idle timeout: {e}")
-
-        browser.close()
-
-    return script_urls
-
-
-def download_file(url: str, timeout: int = 30) -> bytes:
-    """Download a file from a URL."""
-    try:
-        with httpx.Client(timeout=timeout, follow_redirects=True) as client:
-            response = client.get(url=url)
-            response.raise_for_status()
-            return response.content
-    except httpx.HTTPError as e:
-        logger.error(f"Failed to download {url}: {e}")
-        raise
-
-
-def get_sourcemap_url(script_content: str, script_url: str) -> Optional[str]:
-    """Extract source map URL from script content."""
-    match = re.search(r"//# sourceMappingURL=(.+?)(?:\n|$)", script_content)
-    if not match:
-        return None
-
-    sourcemap_ref = match.group(1).strip()
-
-    if sourcemap_ref.startswith("http"):
-        return sourcemap_ref
-
-    return urljoin(base=script_url, url=sourcemap_ref)
-
-
-def decode_sourcemap(sourcemap_data: dict) -> dict:
-    """Extract sources and content from source map."""
-    sources = sourcemap_data.get("sources", [])
-    sources_content = sourcemap_data.get("sourcesContent", [])
-
-    return {
-        "sources": sources,
-        "sources_content": sources_content,
-    }
-
-
-def reconstruct_sources(
-    script_content: str,
-    sourcemap_data: dict,
-    output_dir: Path,
-    script_name: str,
-) -> None:
-    """Reconstruct original sources from source map."""
-    sources = sourcemap_data.get("sources", [])
-    sources_content = sourcemap_data.get("sourcesContent", [])
-
-    logger.info(f"Source map has {len(sources)} source files")
-    if sources:
-        logger.debug(f"First few sources: {sources[:3]}")
-
-    if not sources_content or all(c is None for c in sources_content):
-        logger.error(f"No source content found in source map for {script_name}")
-        sys.exit(1)
-
-    for i, source_file in enumerate(sources):
-        if i < len(sources_content) and sources_content[i]:
-            source_path = output_dir / source_file
-            source_path.parent.mkdir(parents=True, exist_ok=True)
-
-            with open(source_path, mode="w", encoding="utf-8") as f:
-                f.write(sources_content[i])
-
-            logger.info(f"Reconstructed {source_file}")
-
-
-def main() -> None:
-    args = parse_args()
-    args.output_dir.mkdir(parents=True, exist_ok=True)
-
-    logger.info(f"Fetching page: {args.url}")
-    script_urls = get_script_urls(page_url=args.url, headless=args.headless)
-    logger.info(f"Found {len(script_urls)} script(s)")
-
-    if not script_urls:
-        logger.error("No scripts found on page")
-        sys.exit(1)
-
-    for script_url in script_urls:
-        logger.info(f"Processing {script_url}")
-
-        try:
-            script_content = download_file(url=script_url).decode(encoding="utf-8")
-        except Exception as e:
-            logger.error(f"Failed to get script {script_url}: {e}")
-            continue
-
-        sourcemap_url = get_sourcemap_url(script_content=script_content, script_url=script_url)
-
-        if not sourcemap_url:
-            logger.error(f"No source map found for {script_url}")
-            sys.exit(1)
-
-        logger.info(f"Downloading source map: {sourcemap_url}")
-
-        try:
-            sourcemap_content = download_file(url=sourcemap_url).decode(encoding="utf-8")
-            sourcemap_data = json.loads(sourcemap_content)
-        except Exception as e:
-            logger.error(f"Failed to download/parse source map: {e}")
-            sys.exit(1)
-
-        script_name = urlparse(script_url).path.split("/")[-1]
-        reconstruct_sources(
-            script_content=script_content,
-            sourcemap_data=sourcemap_data,
-            output_dir=args.output_dir,
-            script_name=script_name,
-        )
-
-    logger.info(f"All sources reconstructed to {args.output_dir}")
-
-
-if __name__ == "__main__":
-    main()

duration_parser.py

@@ -1,40 +0,0 @@
-import sys
-import re
-import math
-import datetime
-
-
-def parse_duration(text: str) -> datetime.timedelta:
-    re_duration = re.compile(r'(\d+(?:\.\d+)?(?:sec|min|hour|hr|h|m|s))')
-    re_digits = re.compile(r'([\d.]+)')
-    time_scales = {
-        's': 1,
-        'sec': 1,
-        'm': 60,
-        'min': 60,
-        'h': 3600,
-        'hr': 3600,
-    }
-
-    parts = re_duration.findall(text)
-    if parts:
-        total_secs = 0
-        for it in parts:
-            if m := re_digits.search(it):
-                _, end = m.span()
-                scale = it[end:]
-                total_secs += math.ceil(float(m.group(1)) * time_scales[scale])
-        return datetime.timedelta(seconds=total_secs)
-
-    parts = [float(it) for it in text.split(':')]
-    if len(parts) == 1:
-        return datetime.timedelta(seconds=parts[0])
-    if len(parts) == 2:
-        return datetime.timedelta(minutes=parts[0], seconds=parts[1])
-    if len(parts) == 3:
-        return datetime.timedelta(hours=parts[0], minutes=parts[1], seconds=parts[2])
-
-
-if __name__ == '__main__':
-    parsed = parse_duration(sys.argv[1])
-    seconds = int(parsed.total_seconds())

extract_archives.py

@@ -1,230 +0,0 @@
-#!/usr/bin/env -S uv run
-# /// script
-# requires-python = ">=3.13"
-# ///
-import argparse
-import logging
-import re
-import subprocess
-import sys
-from pathlib import Path
-
-logger = logging.getLogger(__name__)
-
-KNOWN_PASSWORDS = """
-torgo
-Yuzuki
-oron.com
-MonikGonPs
-koth
-drdoom-psuzy
-psuzy
-suzy
-idle
-dan4260
-""".strip().splitlines(keepends=False)
-
-
-class CorruptArchiveError(Exception):
-    pass
-
-
-def find_archive_files(root_dir: Path) -> list[Path]:
-    extractables = []
-    globs = ["*.rar", "*.zip*", "*.7z*"]
-    for glob in globs:
-        for f in root_dir.rglob(glob):
-            extractables.append(f)
-
-    extractables = [f for f in extractables if is_extractable(f)]
-
-    return sorted(extractables, key=str)
-
-
-def is_extractable(archive: Path) -> bool:
-    if ".corrupt" in archive.name:
-        return False
-    if re.search(r"\.zip|\.7z", archive.name):
-        zip_part = re.search(r"(\d+)$", archive.suffix)
-        if zip_part and int(zip_part.group(1)) == 1:
-            return True
-        if not zip_part:
-            return True
-
-    if archive.name.endswith(".rar"):
-        rar_part = re.search(r"\.part(\d+)", archive.name)
-        if not rar_part:
-            return True
-        if rar_part and int(rar_part.group(1)) == 1:
-            if archive.with_name(archive.name.replace("part1", "part2")).is_file():
-                return True
-
-    return False
-
-
-def extract_archive_rar(archive: Path, output_dir: Path, password: str | None = None) -> None:
-    args = [
-        "rar",
-        "e",
-        *([f"-p{password}"] if password else []),
-        f"{archive.name}",
-        f"{output_dir}",
-    ]
-    try:
-        subprocess.run(args, cwd=archive.parent, capture_output=True, text=True, check=True)
-    except subprocess.CalledProcessError as e:
-        output: str = e.stderr or e.stdout
-        if "corrupt file or wrong password" in output.lower():
-            raise LookupError("corrupt or invalid password")
-        elif e.returncode == 3:
-            raise FileNotFoundError("missing parts")
-        elif e.returncode == 10:
-            raise ValueError("not a rar file")
-        elif e.returncode == 11:
-            raise LookupError("invalid password")
-
-        raise
-
-
-def extract_archive_7z(archive: Path, output_dir: Path, password: str | None = None):
-    args = [
-        "7z",
-        "e",
-        *(["-p", password] if password else []),
-        f"-o{output_dir}",
-        "-y",
-        f"{archive}",
-    ]
-
-    try:
-        subprocess.run(args, capture_output=True, text=True, check=True)
-    except subprocess.CalledProcessError as e:
-        output: str = e.stderr or e.stdout
-        if "missing volume" in output.lower():
-            raise CorruptArchiveError("missing parts")
-        if "wrong password" in output.lower():
-            raise LookupError("invalid password")
-        raise
-
-
-def extract_archive_zip(archive: Path, output_dir: Path, password: str | None = None):
-    args = [
-        "unzip",
-        "-o",  # overwrite files without prompting
-        *(["-P", password] if password else []),
-        f"{archive}",
-        "-d",
-        f"{output_dir}",
-    ]
-
-    try:
-        subprocess.run(args, capture_output=True, text=True, check=True)
-    except subprocess.CalledProcessError as e:
-        output: str = e.stderr or e.stdout
-        if "zipfile corrupt" in output.lower():
-            raise CorruptArchiveError("corrupt zip file")
-        if "cannot find zipfile" in output.lower():
-            raise FileNotFoundError("zip file not found")
-        if "incorrect password" in output.lower() or "password required" in output.lower():
-            raise LookupError("invalid password")
-        raise
-
-
-def extract_archive(archive_path: Path, output_dir: Path, passwords: list[str]) -> None:
-    for p in ["", *passwords]:
-        try:
-            fn: callable
-            if ".rar" in archive_path.name:
-                fn = extract_archive_rar
-            elif ".zip" == archive_path.suffix:
-                fn = extract_archive_zip
-            else:
-                fn = extract_archive_7z
-            fn(archive_path, output_dir, p)
-        except subprocess.CalledProcessError as e:
-            logger.error(f"error extracting {archive_path}: {e.stderr}")
-            raise
-        except LookupError as e:
-            logger.info(f"invalid password: {p!r}. {e}")
-            continue
-        except Exception as e:
-            logger.error(f"failed to extract: {e}")
-            raise
-
-
-def delete_archive(archive: Path):
-    logger.info(f"deleting: {archive.name}")
-    archive.unlink()
-    stem = archive.stem.rsplit(".", maxsplit=1)[0]
-    for f in archive.parent.rglob(f"*{archive.suffix}"):
-        if archive.stem.startswith(stem):
-            logger.info(f"deleting: {f.name}")
-            f.unlink()
-
-
-def unlink_if_empty(dir_path: Path) -> None:
-    """Unlink the directory if it is empty."""
-    try:
-        if not any(dir_path.iterdir()):
-            dir_path.rmdir()
-            logger.info(f"deleted empty directory: {dir_path}")
-    except OSError as e:
-        logger.error(f"error deleting directory: {e}")
-        raise
-
-
-def parse_args():
-    arger = argparse.ArgumentParser()
-    arger.add_argument("archive_paths", type=lambda v: Path(v.strip()), nargs="+", help="List of archive paths")
-    arger.add_argument("--cwd", type=Path, default=Path.cwd(), help="working directory")
-    arger.add_argument("--find", default=False, help="find archives and print to stdout", action="store_true")
-    arger.add_argument("--clean", default=False, help="delete file(s) after successful extraction", action="store_true")
-
-    if len(sys.argv) < 2:
-        arger.print_help()
-        exit(1)
-
-    return arger.parse_args()
-
-
-def main():
-    logging.basicConfig(level=logging.INFO)
-
-    args = parse_args()
-    cwd: Path = args.cwd
-
-    if args.find:
-        archives = find_archive_files(cwd)
-        for f in archives:
-            print(str(f.resolve()))
-        return
-
-    archives = args.archive_paths
-
-    failed = []
-    for i, f in enumerate(archives, start=1):
-        f: Path
-        progress = f"{i:02}/{len(archives):02}"
-        logger.info(f"extracting {progress}: {f.relative_to(cwd)}")
-        output_dir = f.parent / f.stem.strip()
-        try:
-            output_dir.mkdir(exist_ok=True, parents=True)
-            extract_archive(f, output_dir=output_dir, passwords=KNOWN_PASSWORDS)
-            if args.clean:
-                delete_archive(f)
-        except CorruptArchiveError as e:
-            if ".corrupt" not in f.name:
-                f.rename(f.with_stem(f"{f.stem}.corrupt"))
-        except (ValueError, FileNotFoundError) as e:
-            logger.error(str(e))
-            failed.append(f)
-        finally:
-            unlink_if_empty(output_dir)
-    if failed:
-        logger.info(f"failed to extract {len(failed)} archives")
-        for f in failed:
-            logger.info(f"{f.relative_to(cwd)}")
-
-
-if __name__ == "__main__":
-    main()

ffmpeg_remove_still.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python3
+# /// script
+# dependencies = []
+# ///
+
+import argparse
+from bisect import bisect_left, bisect_right
+import json
+import logging
+import math
+import re
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Shorten still sections using FFmpeg stream copying, without re-encoding.",
+        epilog="Cuts follow video keyframes and audio packets. Still sections may remain longer than requested. "
+               "Audio during removed time is also removed, including any narration.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("input", type=Path, help="Input video")
+    parser.add_argument("--output", type=Path, help="Output video (default: INPUT.short with the original extension)")
+    parser.add_argument("--min-still", type=float, default=2.0,
+                        help="Minimum still section duration to detect, in seconds")
+    parser.add_argument("--keep-still", type=float, default=0.5,
+                        help="Target seconds to retain per still section; keyframe alignment may retain more")
+    parser.add_argument("--noise", type=float, default=0.001,
+                        help="Frame difference tolerance, from 0 to 1; higher values ignore more movement")
+    parser.add_argument("--dry-run", action="store_true", help="Report cuts without writing output")
+    parser.add_argument("--overwrite", action="store_true", help="Replace an existing output file")
+    arguments = parser.parse_args()
+    if not math.isfinite(arguments.min_still) or arguments.min_still <= 0:
+        parser.error("--min-still must be positive and finite")
+    if not math.isfinite(arguments.keep_still) or arguments.keep_still <= 0:
+        parser.error("--keep-still must be positive and finite")
+    if not 0 <= arguments.noise <= 1:
+        parser.error("--noise must be between 0 and 1")
+    return arguments
+
+
+def probe_video(path: Path) -> tuple[float, float]:
+    result = subprocess.run(
+        args=["ffprobe", "-v", "error", "-show_streams", "-show_format", "-of", "json", str(path)],
+        check=True, capture_output=True, text=True,
+    )
+    metadata = json.loads(result.stdout)
+    video = next((stream for stream in metadata["streams"] if stream["codec_type"] == "video"), None)
+    if video is None:
+        raise ValueError("Input has no video stream")
+    duration = float(video.get("duration") or metadata["format"].get("duration", 0))
+    if not math.isfinite(duration) or duration <= 0:
+        raise ValueError("Cannot determine video duration")
+    start_time = float(video.get("start_time") or metadata["format"].get("start_time", 0))
+    return duration, start_time
+
+
+def probe_keyframes(path: Path, start_time: float) -> list[tuple[float, float]]:
+    result = subprocess.run(
+        args=["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_packets",
+              "-show_entries", "packet=pts_time,dts_time,flags", "-of", "csv=p=0", str(path)],
+        check=True, capture_output=True, text=True,
+    )
+    keyframes = []
+    for line in result.stdout.splitlines():
+        fields = line.split(",")
+        if len(fields) >= 3 and "K" in fields[2] and fields[0] != "N/A":
+            presentation_time = float(fields[0]) - start_time
+            decode_time = float(fields[1]) - start_time if fields[1] != "N/A" else presentation_time
+            keyframes.append((max(0.0, presentation_time), decode_time))
+    if not keyframes:
+        raise ValueError("No video keyframes found; cannot safely trim by stream copying")
+    return sorted(keyframes)
+
+
+def parse_freezes(log: str, duration: float) -> list[tuple[float, float]]:
+    freezes = []
+    start = None
+    for kind, timestamp in re.findall(r"lavfi\.freezedetect\.freeze_(start|end):\s*([-+\d.eE]+)", log):
+        timestamp = min(duration, max(0.0, float(timestamp)))
+        if kind == "start":
+            start = timestamp
+        elif start is not None:
+            if timestamp > start:
+                freezes.append((start, timestamp))
+            start = None
+    if start is not None and duration > start:
+        freezes.append((start, duration))
+    return freezes
+
+
+def detect_freezes(path: Path, duration: float, min_still: float, noise: float) -> list[tuple[float, float]]:
+    logging.info("Detecting still sections...")
+    result = subprocess.run(
+        args=[
+            "ffmpeg", "-hide_banner", "-nostdin", "-nostats", "-i", str(path),
+            "-map", "0:v:0", "-vf", f"setpts=PTS-STARTPTS,freezedetect=noise={noise}:duration={min_still}",
+            "-an", "-f", "null", "-",
+        ],
+        check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True,
+    )
+    return parse_freezes(log=result.stderr, duration=duration)
+
+
+def plan_segments(duration: float, freezes: list[tuple[float, float]], keep_still: float,
+                  keyframes: list[float]) -> list[tuple[float, float]]:
+    segments = []
+    cursor = 0.0
+    for start, end in freezes:
+        if end - start <= keep_still:
+            continue
+        first = bisect_left(keyframes, start + keep_still / 2)
+        last = bisect_right(keyframes, end - keep_still / 2) - 1
+        if first >= len(keyframes) or last < 0 or keyframes[first] >= keyframes[last]:
+            logging.info(f"Still section {start:.2f}–{end:.2f}s: no removable span between keyframes")
+            continue
+        cut_start = keyframes[first]
+        cut_end = keyframes[last]
+        logging.info(f"Still section {start:.2f}–{end:.2f}s: retain {end - start - (cut_end - cut_start):.2f}s")
+        if cut_start > cursor:
+            segments.append((cursor, cut_start))
+        cursor = cut_end
+    if cursor < duration:
+        segments.append((cursor, duration))
+    return segments
+
+
+def copy_video(source: Path, output: Path, segments: list[tuple[float, float]], duration: float,
+               start_time: float, keyframes: list[tuple[float, float]], overwrite: bool) -> None:
+    decode_times = dict(keyframes)
+    with tempfile.TemporaryDirectory() as temporary_directory:
+        playlist = Path(temporary_directory) / "segments.ffconcat"
+        local_source = Path(temporary_directory) / f"source{source.suffix}"
+        local_source.symlink_to(target=source)
+        entries = ["ffconcat version 1.0"]
+        for start, end in segments:
+            entries.extend([f"file '{local_source.name}'", f"inpoint {start + start_time:.6f}"])
+            if end < duration:
+                # The concat demuxer's outpoint uses DTS, while inpoint uses PTS.
+                entries.append(f"outpoint {decode_times[end] + start_time:.6f}")
+            entries.append(f"duration {end - start:.6f}")
+        playlist.write_text(data="\n".join(entries) + "\n")
+        command = [
+            "ffmpeg", "-hide_banner", "-nostdin", "-loglevel", "warning", "-stats",
+            "-y" if overwrite else "-n", "-f", "concat", "-safe", "0", "-auto_convert", "0", "-i", str(playlist),
+            "-map", "0:v:0", "-map", "0:a?", "-c", "copy",
+        ]
+        if output.suffix.lower() in {".mp4", ".mov"}:
+            command.extend(["-movflags", "+faststart"])
+        command.append(str(output))
+        subprocess.run(args=command, check=True)
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s")
+    arguments = parse_args()
+    try:
+        for executable in ("ffmpeg", "ffprobe"):
+            if shutil.which(cmd=executable) is None:
+                raise ValueError(f"{executable} must be installed and available on PATH")
+        source = arguments.input.resolve(strict=True)
+        output = (arguments.output or source.with_name(f"{source.stem}.short{source.suffix or '.mkv'}")).resolve()
+        if source == output:
+            raise ValueError("Input and output must be different files")
+        if output.exists() and source.samefile(output):
+            raise ValueError("Input and output must be different files")
+        if not arguments.dry_run and output.exists() and not arguments.overwrite:
+            raise ValueError(f"Output exists: {output}; use --overwrite to replace it")
+        duration, start_time = probe_video(path=source)
+        freezes = detect_freezes(path=source, duration=duration, min_still=arguments.min_still, noise=arguments.noise)
+        keyframes = probe_keyframes(path=source, start_time=start_time)
+        segments = plan_segments(duration=duration, freezes=freezes, keep_still=arguments.keep_still,
+                                 keyframes=[time for time, _ in keyframes])
+        retained = sum(end - start for start, end in segments)
+        logging.info(f"{duration:.2f}s → {retained:.2f}s; removing {duration - retained:.2f}s ({(1 - retained / duration):.1%})")
+        if arguments.dry_run:
+            return
+        copy_video(source=source, output=output, segments=segments, duration=duration,
+                   start_time=start_time, keyframes=keyframes, overwrite=arguments.overwrite)
+        logging.info(f"Saved {output}")
+    except (OSError, ValueError, subprocess.CalledProcessError) as error:
+        logging.error(str(error))
+        if isinstance(error, subprocess.CalledProcessError) and error.stderr:
+            logging.error(error.stderr)
+        raise SystemExit(1) from error
+
+
+if __name__ == "__main__":
+    main()

ffmpeg_remove_still_test.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env -S uv run --script
+# /// script
+# dependencies = ["pytest"]
+# ///
+
+import json
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+import ffmpeg_remove_still as script
+
+
+@pytest.mark.parametrize("log, expected", [
+    ("", []),
+    ("lavfi.freezedetect.freeze_start: 0", [(0.0, 10.0)]),
+    ("lavfi.freezedetect.freeze_start: 2\nlavfi.freezedetect.freeze_end: 5", [(2.0, 5.0)]),
+    ("lavfi.freezedetect.freeze_start: 2\nlavfi.freezedetect.freeze_end: 5\n"
+     "lavfi.freezedetect.freeze_start: 8", [(2.0, 5.0), (8.0, 10.0)]),
+])
+def test_parse_freezes(log: str, expected: list[tuple[float, float]]) -> None:
+    assert script.parse_freezes(log=log, duration=10.0) == expected
+
+
+@pytest.mark.parametrize("freezes, keep_still, keyframes, expected", [
+    ([], 1.0, [0.0], [(0.0, 10.0)]),
+    ([(0.0, 10.0)], 1.0, [0.0, 0.5, 9.5], [(0.0, 0.5), (9.5, 10.0)]),
+    ([(2.0, 6.0)], 2.0, [0.0, 3.0, 5.0], [(0.0, 3.0), (5.0, 10.0)]),
+    ([(2.0, 3.0)], 2.0, [0.0, 2.5], [(0.0, 10.0)]),
+    ([(0.0, 4.0), (6.0, 10.0)], 1.0, [0.0, 0.5, 3.5, 6.5, 9.5],
+     [(0.0, 0.5), (3.5, 6.5), (9.5, 10.0)]),
+    ([(2.0, 6.0)], 1.0, [0.0, 3.0, 5.0, 7.0], [(0.0, 3.0), (5.0, 10.0)]),
+    ([(2.0, 6.0)], 1.0, [0.0, 4.0, 8.0], [(0.0, 10.0)]),
+    ([(2.0, 6.0)], 1.0, [0.0, 8.0], [(0.0, 10.0)]),
+])
+def test_plan_segments(freezes: list[tuple[float, float]], keep_still: float,
+                       keyframes: list[float], expected: list[tuple[float, float]]) -> None:
+    assert script.plan_segments(duration=10.0, freezes=freezes, keep_still=keep_still, keyframes=keyframes) == expected
+
+
+@pytest.mark.skipif(not shutil.which(cmd="ffmpeg") or not shutil.which(cmd="ffprobe"), reason="Requires FFmpeg")
+@pytest.mark.parametrize("has_audio, keep_still, keyframe_interval, expected_duration", [
+    (False, 0.5, 5, 4.0), (True, 0.5, 5, 4.0), (True, 1.5, 5, 6.0),
+    (True, 0.5, 100, 8.0),
+])
+def test_copies_packets_without_encoding(tmp_path: Path, has_audio: bool, keep_still: float,
+                                        keyframe_interval: int, expected_duration: float) -> None:
+    source = tmp_path / "demo's recording.mp4"
+    output = tmp_path / "short.mp4"
+    command = ["ffmpeg", "-hide_banner", "-loglevel", "error", "-nostdin"]
+    for pattern in ["testsrc2=size=64x64:rate=10:duration=1", "color=red:size=64x64:rate=10:duration=3",
+                    "testsrc2=size=64x64:rate=10:duration=1", "color=blue:size=64x64:rate=10:duration=3"]:
+        command.extend(["-f", "lavfi", "-i", pattern])
+    if has_audio:
+        command.extend(["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=8"])
+    command.extend(["-filter_complex", "[0:v][1:v][2:v][3:v]concat=n=4:v=1:a=0[v]", "-map", "[v]"])
+    if has_audio:
+        command.extend(["-map", "4:a", "-c:a", "aac"])
+    command.extend(["-c:v", "libx264", "-g", str(keyframe_interval), "-keyint_min", str(keyframe_interval),
+                    "-sc_threshold", "0", "-bf", "2", "-flags", "+cgop", str(source)])
+    subprocess.run(args=command, check=True, capture_output=True)
+    copied = subprocess.run(
+        args=[sys.executable, str(Path(script.__file__).resolve()), str(source), "--output", str(output),
+              "--min-still", "2", "--keep-still", str(keep_still)],
+        check=True, capture_output=True, text=True,
+    )
+    assert "non-monoton" not in copied.stderr.lower()
+    result = subprocess.run(
+        args=["ffprobe", "-v", "error", "-show_streams", "-show_format", "-of", "json", str(output)],
+        check=True, capture_output=True, text=True,
+    )
+    metadata = json.loads(result.stdout)
+    video = next(stream for stream in metadata["streams"] if stream["codec_type"] == "video")
+    assert float(video["duration"]) == pytest.approx(expected_duration, abs=0.15)
+    audio = [stream for stream in metadata["streams"] if stream["codec_type"] == "audio"]
+    assert bool(audio) == has_audio
+    if has_audio:
+        assert float(audio[0]["duration"]) == pytest.approx(float(video["duration"]), abs=0.1)
+    packet_hashes = []
+    for path in (source, output):
+        packets = subprocess.run(
+            args=["ffprobe", "-v", "error", "-show_packets", "-show_data_hash", "sha256",
+                  "-show_entries", "packet=stream_index,data_hash", "-of", "json", str(path)],
+            check=True, capture_output=True, text=True,
+        )
+        packet_hashes.append({(packet["stream_index"], packet["data_hash"])
+                              for packet in json.loads(packets.stdout)["packets"]})
+    assert packet_hashes[1] <= packet_hashes[0]
+    decoded = subprocess.run(
+        args=["ffmpeg", "-v", "error", "-i", str(output), "-f", "null", "-"],
+        check=True, capture_output=True, text=True,
+    )
+    assert decoded.stderr == ""
+    original = source.read_bytes()
+    refused = subprocess.run(
+        args=[sys.executable, str(Path(script.__file__).resolve()), str(source), "--output", str(source), "--overwrite"],
+        capture_output=True, text=True,
+    )
+    assert refused.returncode == 1
+    assert source.read_bytes() == original
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(args=[str(Path(__file__).resolve()), *sys.argv[1:]]))

ffmpeg_strip.py

@@ -1,81 +0,0 @@
-#!/usr/bin/env python3.9
-import argparse
-import json
-import logging
-import subprocess
-import sys
-from pathlib import Path
-import typing
-
-
-def parse_args(argv: list[str]):
-    arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    arger.add_argument("video_path", type=Path, help="Video path")
-    arger.add_argument("save_path", nargs="?", type=Path, help="Output path")
-    if len(argv) == 0:
-        arger.print_help()
-        exit(1)
-
-    return arger.parse_args(argv)
-
-
-def ffprobe(video_path: Path) -> dict:
-    proc = subprocess.run(
-        [
-            "ffprobe",
-            "-v",
-            "quiet",
-            "-print_format",
-            "json",
-            "-show_format",
-            "-show_streams",
-            str(video_path),
-        ],
-        check=True,
-        capture_output=True,
-    )
-    return json.loads(proc.stdout)
-
-
-def clean_video(video_path: Path, save_path: Path) -> Path:
-    media_info = ffprobe(video_path)
-    # check if the first video stream has hevc codec
-    if media_info["streams"][0]["codec_name"] == "hevc":
-        logging.debug(f"Video codec is hevc, adding hvc1 tag")
-        extra_args = ["-tag:v", "hvc1"]
-    else:
-        extra_args = []
-
-    # fmt: off
-    args = [
-        'ffmpeg',
-        '-i',
-        str(video_path),
-        '-c:v', 'copy',
-        # '-movflags', '+faststart',
-        '-map_metadata', '-1',
-        *extra_args,
-        '-c:a', 'copy',
-        '-y',
-        str(save_path),
-    ]
-    # fmt: on
-    logging.info(f"calling ffmpeg with {args=}")
-    subprocess.run(args, check=True)
-    return save_path
-
-
-def main():
-    logging.basicConfig(level=logging.INFO)
-    args = parse_args(sys.argv[1:])
-    save_path: typing.Optional[Path] = args.save_path
-    if save_path and save_path.resolve().is_dir():
-        save_path = save_path / args.video_path.name
-    if save_path is None:
-        save_path = args.video_path.with_suffix(".clean.mp4")
-
-    clean_video(video_path=args.video_path, save_path=save_path)
-
-
-if __name__ == "__main__":
-    main()

flux2_private_server.py

@@ -1,461 +0,0 @@
-#!/usr/bin/env -S uv run --script
-# /// script
-# requires-python = ">=3.10,<3.11"
-# dependencies = [
-#   "torch==2.9.1+cu128",
-#   "torchvision==0.24.1+cu128",
-#   "torchaudio==2.9.1+cu128",
-#   "diffusers @ git+https://github.com/huggingface/diffusers.git",
-#   "transformers>=4.57.0",
-#   "accelerate>=1.0.0",
-#   "safetensors>=0.4.0",
-#   "huggingface_hub>=0.25.0",
-#   "numpy>=1.26.0",
-#   "pillow>=10.0.0",
-#   "fastapi>=0.115.0",
-#   "uvicorn>=0.29.0",
-# ]
-# [tool.uv]
-# extra-index-url = ["https://download.pytorch.org/whl/cu128"]
-# ///
-
-import argparse
-import base64
-import binascii
-import io
-import os
-import random
-import threading
-import time
-from typing import Any
-
-os.environ.setdefault("HF_XET_HIGH_PERFORMANCE", "1")
-
-import torch
-from diffusers import Flux2KleinPipeline
-from fastapi import FastAPI, HTTPException
-from fastapi.responses import HTMLResponse, Response
-from pydantic import BaseModel, Field
-from PIL import Image
-
-
-def _resolve_model_size() -> str:
-    parser = argparse.ArgumentParser(add_help=False)
-    parser.add_argument("--model", choices=["4b", "9b"], default="9b")
-    args, _ = parser.parse_known_args()
-    return args.model
-
-
-def _checkpoint_for(model_size: str) -> str:
-    if model_size == "4b":
-        return "black-forest-labs/FLUX.2-klein-4B"
-    return "black-forest-labs/FLUX.2-klein-9B"
-
-
-MODEL_SIZE = _resolve_model_size()
-CHECKPOINT = _checkpoint_for(MODEL_SIZE)
-USE_COMPILE = os.environ.get("FLUX2_COMPILE", "1") != "0"
-WARMUP_RUNS = int(os.environ.get("FLUX2_WARMUP_RUNS", "2"))
-WARMUP_STEPS = int(os.environ.get("FLUX2_WARMUP_STEPS", "4"))
-WARMUP_PROMPT = "A high-resolution photo of a red fox in natural daylight"
-MAX_SEED = (2**31) - 1
-MAX_PIXELS = 4 * 1024 * 1024
-
-DEFAULT_STEPS = 4
-DEFAULT_GUIDANCE = 1.0
-
-app = FastAPI(title="FLUX2 Klein Private Server")
-PIPE: Flux2KleinPipeline | None = None
-PIPE_LOCK = threading.Lock()
-
-
-class GenerateRequest(BaseModel):
-    prompt: str = Field(..., min_length=1)
-    width: int = 1024
-    height: int = 1024
-    num_inference_steps: int = DEFAULT_STEPS
-    guidance_scale: float = DEFAULT_GUIDANCE
-    seed: int = 0
-    randomize_seed: bool = False
-    image_data_url: str | None = None
-
-
-class CompatRequest(BaseModel):
-    model: str | None = None
-    prompt: str = Field(..., min_length=1)
-    imageDataUrl: str | None = None
-    size: str | None = None
-    width: int | None = None
-    height: int | None = None
-    num_inference_steps: int | None = None
-    guidance_scale: float | None = None
-    seed: int | None = None
-    randomize_seed: bool = False
-    response_format: str = "b64_json"
-    n: int = 1
-    negative_prompt: str | None = None
-    negativePrompt: str | None = None
-
-
-def _ensure_cuda() -> None:
-    if not torch.cuda.is_available():
-        raise RuntimeError("CUDA is required for this server")
-
-
-def _compile_pipe(pipe: Flux2KleinPipeline) -> bool:
-    if not USE_COMPILE:
-        return False
-    print("Compiling transformer and VAE decoder...")
-    start = time.perf_counter()
-    pipe.transformer = torch.compile(
-        pipe.transformer,
-        mode="max-autotune",
-        dynamic=False,
-        fullgraph=True,
-    )
-    pipe.vae.decode = torch.compile(
-        pipe.vae.decode,
-        mode="max-autotune",
-        dynamic=False,
-        fullgraph=True,
-    )
-    print(f"Compile finished in {time.perf_counter() - start:.2f}s")
-    return True
-
-
-def _warmup_pipe(pipe: Flux2KleinPipeline) -> None:
-    if WARMUP_RUNS <= 0:
-        return
-    print(f"Warmup started ({WARMUP_RUNS} runs)...")
-    start = time.perf_counter()
-    for run_idx in range(WARMUP_RUNS):
-        generator = torch.Generator(device="cuda").manual_seed(run_idx)
-        pipe(
-            prompt=WARMUP_PROMPT,
-            height=1024,
-            width=1024,
-            num_inference_steps=WARMUP_STEPS,
-            guidance_scale=1.0,
-            generator=generator,
-        ).images[0]
-    torch.cuda.synchronize()
-    print(f"Warmup finished in {time.perf_counter() - start:.2f}s")
-
-
-def _load_pipe(ckpt: str) -> Flux2KleinPipeline:
-    print(f"Loading checkpoint: {ckpt}")
-    pipe = Flux2KleinPipeline.from_pretrained(ckpt, torch_dtype=torch.bfloat16)
-    pipe = pipe.to("cuda")
-    pipe.transformer.fuse_qkv_projections()
-    pipe.vae.fuse_qkv_projections()
-    pipe.vae.to(memory_format=torch.channels_last)
-    pipe.transformer.set_attention_backend("_native_flash")
-    if _compile_pipe(pipe):
-        _warmup_pipe(pipe)
-    return pipe
-
-
-def _normalize_dims(width: int, height: int) -> tuple[int, int]:
-    if width <= 0 or height <= 0:
-        raise HTTPException(status_code=422, detail="width and height must be > 0")
-    if width % 16 != 0 or height % 16 != 0:
-        raise HTTPException(
-            status_code=422, detail="width and height must be multiples of 16"
-        )
-    if width * height > MAX_PIXELS:
-        raise HTTPException(status_code=422, detail="max resolution is 4 megapixels")
-    return width, height
-
-
-def _parse_size(
-    size: str | None, width: int | None, height: int | None
-) -> tuple[int, int]:
-    if width and height:
-        return _normalize_dims(width, height)
-    if not size:
-        return _normalize_dims(1024, 1024)
-    parts = size.lower().split("x")
-    if len(parts) != 2:
-        raise HTTPException(status_code=422, detail="size must look like 1024x1024")
-    try:
-        parsed_w = int(parts[0])
-        parsed_h = int(parts[1])
-    except ValueError as exc:
-        raise HTTPException(
-            status_code=422, detail="size values must be integers"
-        ) from exc
-    return _normalize_dims(parsed_w, parsed_h)
-
-
-def _decode_image_data_url(image_data_url: str | None) -> Image.Image | None:
-    if not image_data_url:
-        return None
-    if "," not in image_data_url:
-        raise HTTPException(status_code=422, detail="imageDataUrl must be a data URL")
-    _, payload = image_data_url.split(",", 1)
-    try:
-        raw = base64.b64decode(payload)
-    except (ValueError, binascii.Error) as exc:
-        raise HTTPException(
-            status_code=422, detail="invalid base64 in imageDataUrl"
-        ) from exc
-    with Image.open(io.BytesIO(raw)) as image:
-        return image.convert("RGB")
-
-
-def _generate_png_bytes(
-    prompt: str,
-    width: int,
-    height: int,
-    num_inference_steps: int,
-    guidance_scale: float,
-    seed: int,
-    image: Image.Image | None,
-) -> tuple[bytes, int]:
-    if PIPE is None:
-        raise HTTPException(status_code=503, detail="model not loaded yet")
-
-    generator = torch.Generator(device="cuda").manual_seed(seed)
-    kwargs: dict[str, Any] = {
-        "prompt": prompt,
-        "height": height,
-        "width": width,
-        "num_inference_steps": num_inference_steps,
-        "guidance_scale": guidance_scale,
-        "generator": generator,
-    }
-
-    with PIPE_LOCK:
-        try:
-            if image is not None:
-                try:
-                    output = PIPE(image=image, **kwargs)
-                except TypeError:
-                    output = PIPE(images=[image], **kwargs)
-            else:
-                output = PIPE(**kwargs)
-        except Exception as exc:
-            raise HTTPException(
-                status_code=500, detail=f"generation failed: {exc}"
-            ) from exc
-
-    result_image = output.images[0]
-    buf = io.BytesIO()
-    result_image.save(buf, format="PNG")
-    return buf.getvalue(), seed
-
-
-@app.on_event("startup")
-def _startup() -> None:
-    global PIPE
-    _ensure_cuda()
-    PIPE = _load_pipe(CHECKPOINT)
-    print("Server ready")
-
-
-@app.get("/", response_class=HTMLResponse)
-def index() -> str:
-    return """<!doctype html>
-<html lang=\"en\">
-<head>
-  <meta charset=\"utf-8\" />
-  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />
-  <title>FLUX.2 Klein Playground</title>
-  <style>
-    :root { --bg:#f4f0e8; --card:#fff9ee; --text:#1f1f1f; --accent:#125b50; --muted:#6b6b6b; }
-    * { box-sizing: border-box; }
-    body { margin:0; font-family: ui-sans-serif, -apple-system, Segoe UI, sans-serif; background: radial-gradient(circle at top left,#fff6df,var(--bg)); color:var(--text); }
-    .wrap { max-width: 980px; margin: 1.5rem auto; padding: 1rem; }
-    .card { background: var(--card); border: 1px solid #eadfc8; border-radius: 14px; padding: 1rem; box-shadow: 0 8px 20px rgba(0,0,0,.06); }
-    h1 { margin: 0 0 .8rem 0; font-size: 1.4rem; }
-    .grid { display:grid; gap:.8rem; grid-template-columns: 1fr 1fr; }
-    .full { grid-column: 1 / -1; }
-    label { display:block; font-weight:600; margin-bottom:.3rem; }
-    input, textarea, button { width:100%; border-radius:10px; border:1px solid #d8ccb7; padding:.65rem; font-size:.95rem; }
-    textarea { min-height: 110px; resize: vertical; }
-    button { background: var(--accent); color:white; border:none; font-weight:700; cursor:pointer; }
-    button:disabled { opacity:.6; cursor:default; }
-    .muted { color: var(--muted); font-size:.88rem; }
-    .out img { width:100%; border-radius:12px; border:1px solid #e4d8c3; background:white; }
-    @media (max-width: 760px) { .grid { grid-template-columns: 1fr; } }
-  </style>
-</head>
-<body>
-  <div class=\"wrap\">
-    <div class=\"card\">
-      <h1>FLUX.2 Klein Playground</h1>
-      <p class=\"muted\">Private single-user UI at <code>/</code>. Distilled defaults: 4 steps, guidance 1.0.</p>
-      <form id=\"form\" class=\"grid\">
-        <div class=\"full\">
-          <label for=\"prompt\">Prompt</label>
-          <textarea id=\"prompt\" required>A cinematic photo portrait, natural skin texture, realistic lighting, high detail.</textarea>
-        </div>
-        <div>
-          <label for=\"width\">Width (multiple of 16)</label>
-          <input id=\"width\" type=\"number\" value=\"1024\" step=\"16\" min=\"16\" />
-        </div>
-        <div>
-          <label for=\"height\">Height (multiple of 16)</label>
-          <input id=\"height\" type=\"number\" value=\"1024\" step=\"16\" min=\"16\" />
-        </div>
-        <div>
-          <label for=\"steps\">Inference steps</label>
-          <input id=\"steps\" type=\"number\" value=\"4\" min=\"1\" max=\"100\" />
-        </div>
-        <div>
-          <label for=\"guidance\">Guidance scale</label>
-          <input id=\"guidance\" type=\"number\" value=\"1.0\" step=\"0.1\" />
-        </div>
-        <div>
-          <label for=\"seed\">Seed</label>
-          <input id=\"seed\" type=\"number\" value=\"0\" min=\"0\" />
-        </div>
-        <div>
-          <label for=\"edit\">Edit image (optional)</label>
-          <input id=\"edit\" type=\"file\" accept=\"image/*\" />
-        </div>
-        <div class=\"full\">
-          <button id=\"submit\" type=\"submit\">Generate</button>
-          <p id=\"status\" class=\"muted\">Idle</p>
-        </div>
-      </form>
-    </div>
-
-    <div class=\"card out\" style=\"margin-top:1rem\">
-      <h1>Output</h1>
-      <img id=\"preview\" alt=\"Generated image preview\" />
-      <p class=\"muted\">Tip: right-click image to save.</p>
-    </div>
-  </div>
-
-  <script>
-    const form = document.getElementById('form');
-    const statusEl = document.getElementById('status');
-    const preview = document.getElementById('preview');
-    const button = document.getElementById('submit');
-
-    async function fileToDataUrl(file) {
-      return await new Promise((resolve, reject) => {
-        const reader = new FileReader();
-        reader.onload = () => resolve(reader.result);
-        reader.onerror = reject;
-        reader.readAsDataURL(file);
-      });
-    }
-
-    form.addEventListener('submit', async (event) => {
-      event.preventDefault();
-      button.disabled = true;
-      statusEl.textContent = 'Generating...';
-      try {
-        const prompt = document.getElementById('prompt').value;
-        const width = Number(document.getElementById('width').value);
-        const height = Number(document.getElementById('height').value);
-        const steps = Number(document.getElementById('steps').value);
-        const guidance = Number(document.getElementById('guidance').value);
-        const seed = Number(document.getElementById('seed').value);
-        const file = document.getElementById('edit').files[0];
-        const imageDataUrl = file ? await fileToDataUrl(file) : null;
-
-        const payload = {
-          model: 'black-forest-labs/FLUX.2-klein-9B',
-          prompt,
-          size: `${width}x${height}`,
-          num_inference_steps: steps,
-          guidance_scale: guidance,
-          seed,
-          response_format: 'b64_json',
-          imageDataUrl,
-        };
-
-        const resp = await fetch('/v1/images/generations', {
-          method: 'POST',
-          headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify(payload),
-        });
-
-        if (!resp.ok) {
-          const text = await resp.text();
-          throw new Error(text || `HTTP ${resp.status}`);
-        }
-
-        const data = await resp.json();
-        const b64 = data?.data?.[0]?.b64_json;
-        if (!b64) throw new Error('No image returned');
-        preview.src = `data:image/png;base64,${b64}`;
-        statusEl.textContent = `Done. Seed: ${data.seed ?? seed}`;
-      } catch (error) {
-        statusEl.textContent = `Error: ${error.message}`;
-      } finally {
-        button.disabled = false;
-      }
-    });
-  </script>
-</body>
-</html>
-"""
-
-
-@app.get("/health")
-def health() -> dict[str, str]:
-    return {"status": "ok", "model": CHECKPOINT}
-
-
-@app.post("/generate")
-def generate(req: GenerateRequest) -> Response:
-    width, height = _normalize_dims(req.width, req.height)
-    seed = req.seed if not req.randomize_seed else random.randint(0, MAX_SEED)
-    image = _decode_image_data_url(req.image_data_url)
-    png_bytes, used_seed = _generate_png_bytes(
-        prompt=req.prompt,
-        width=width,
-        height=height,
-        num_inference_steps=req.num_inference_steps,
-        guidance_scale=req.guidance_scale,
-        seed=seed,
-        image=image,
-    )
-    return Response(
-        content=png_bytes,
-        media_type="image/png",
-        headers={"X-Seed": str(used_seed)},
-    )
-
-
-@app.post("/v1/images/generations")
-def compat_generate(req: CompatRequest) -> dict[str, Any]:
-    width, height = _parse_size(req.size, req.width, req.height)
-    seed_value = req.seed if req.seed is not None else 0
-    seed = seed_value if not req.randomize_seed else random.randint(0, MAX_SEED)
-    image = _decode_image_data_url(req.imageDataUrl)
-
-    png_bytes, used_seed = _generate_png_bytes(
-        prompt=req.prompt,
-        width=width,
-        height=height,
-        num_inference_steps=req.num_inference_steps or DEFAULT_STEPS,
-        guidance_scale=req.guidance_scale or DEFAULT_GUIDANCE,
-        seed=seed,
-        image=image,
-    )
-    b64_png = base64.b64encode(png_bytes).decode("ascii")
-    return {
-        "created": int(time.time()),
-        "model": CHECKPOINT,
-        "seed": used_seed,
-        "data": [{"b64_json": b64_png}],
-    }
-
-
-if __name__ == "__main__":
-    import uvicorn
-
-    parser = argparse.ArgumentParser(description="Private FLUX.2 Klein FastAPI server")
-    parser.add_argument("--model", choices=["4b", "9b"], default=MODEL_SIZE)
-    parser.add_argument("--host", default="127.0.0.1")
-    parser.add_argument("--port", type=int, default=6006)
-    args = parser.parse_args()
-
-    MODEL_SIZE = args.model
-    CHECKPOINT = _checkpoint_for(MODEL_SIZE)
-    uvicorn.run(app, host=args.host, port=args.port)

fpup.py

@@ -1,68 +0,0 @@
-#!/usr/bin/env -S uv run
-# /// script
-# dependencies = ["httpx"]
-# ///
-import argparse
-import concurrent.futures
-import json
-import re
-import sys
-import typing
-from pathlib import Path
-from typing import Optional
-import logging
-
-
-import httpx
-
-
-def upload_image(image_path: Path) -> dict:
-    session = httpx.Client(timeout=20)
-    logging.info(f"uploading {image_path.name}")
-
-    with image_path.open("rb") as f:
-        res = session.post(
-            "https://fapping.empornium.sx/upload.php",
-            files={
-                "ImageUp": f,
-            },
-        )
-
-    res = session.get("https://fapping.empornium.sx/uploaded/", follow_redirects=True)
-    if m := re.search(r"var ImagesUp = (\[.+\]);", res.text, re.MULTILINE):
-        data: list[dict] = json.loads(m.group(1))
-        logging.info(f"upload result on the page={data}")
-
-        return dict(
-            image_url=data[0]["image_url"],
-            thumbnail_url=data[0]["image_thumb_url"],
-        )
-
-    logging.error(f"failed to find upload result on the page. html={res.text}")
-
-    raise ValueError(f"failed to parse upload result for {image_path.name}")
-
-
-def parse_args(argv: Optional[typing.Sequence[str]] = None) -> argparse.Namespace:
-    arger = argparse.ArgumentParser()
-    arger.add_argument("image_paths", nargs="+", action="store", type=Path, help="Path to image to upload")
-    if not argv or len(argv) == 0:
-        arger.print_help()
-        sys.exit(1)
-
-    return arger.parse_args(argv)
-
-
-def main():
-    args = parse_args(sys.argv[1:])
-    image_paths = args.image_paths
-
-    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
-        results = list(pool.map(upload_image, image_paths))
-    for it in results:
-        print(it["image_url"])
-
-
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.INFO)
-    main()

go.mod

@@ -1,5 +0,0 @@
-module github.com/abdusco/playground
-
-go 1.23.0
-
-require golang.org/x/sync v0.10.0

go.sum

@@ -1,2 +0,0 @@
-golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=

html_popup.py

@@ -1,133 +0,0 @@
-import http.server
-import random
-import socketserver
-import sys
-import threading
-import time
-import typing
-import webbrowser
-
-# language=html
-inject_script = """
-<script>
-    // Send periodic heartbeat
-    let heartbeatInterval = setInterval(() => {
-        fetch('/heartbeat', {method: 'POST'})
-            .catch(() => {}); // Ignore errors
-    }, 1000);
-
-    // Multiple event handlers for different close scenarios
-    function sendShutdown() {
-        clearInterval(heartbeatInterval);
-        // Synchronous request to ensure it gets sent
-        const xhr = new XMLHttpRequest();
-        xhr.open('POST', '/shutdown', false);  // false makes it synchronous
-        try {
-            xhr.send();
-        } catch (e) {}
-    }
-
-    window.addEventListener('beforeunload', sendShutdown);
-    window.addEventListener('unload', sendShutdown);
-    // setTimeout(sendShutdown, 5000);
-
-    // Handle visibility change (tab hidden/visible)
-    document.addEventListener('visibilitychange', () => {
-        if (document.visibilityState === 'hidden') {
-            sendShutdown();
-        }
-    });
-</script>
-"""
-
-
-def handler_for_html(html: str) -> typing.Type[http.server.SimpleHTTPRequestHandler]:
-    class Handler(http.server.SimpleHTTPRequestHandler):
-        def do_GET(self):
-            if self.path == "/":
-                self.send_response(200)
-                self.send_header("Content-type", "text/html")
-                self.end_headers()
-                self.wfile.write(html.encode())
-            else:
-                self.send_error(404)
-
-        def do_POST(self):
-            if self.path == "/shutdown":
-                self.send_response(200)
-                self.end_headers()
-                self.server.last_heartbeat = 0  # Force shutdown
-                self.server.shutdown()
-            elif self.path == "/heartbeat":
-                self.send_response(200)
-                self.end_headers()
-                self.server.last_heartbeat = time.time()
-            else:
-                self.send_error(404)
-
-        def log_message(self, format, *args):
-            # Suppress logging
-            pass
-
-    return Handler
-
-
-class ServerWithHeartbeat(socketserver.TCPServer):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.last_heartbeat = time.time()
-
-
-def monitor_heartbeat(server: ServerWithHeartbeat, timeout: int = 2):
-    while True:
-        time.sleep(0.5)
-        if time.time() - server.last_heartbeat > timeout:
-            print("No heartbeat received, shutting down...")
-            server.shutdown()
-            break
-
-
-def run_server(handler: http.server.SimpleHTTPRequestHandler, port: int = 8000):
-    with ServerWithHeartbeat(("", port), handler) as httpd:
-        print(f"Serving at port {port}")
-
-        # Start heartbeat monitor in separate thread
-        monitor_thread = threading.Thread(target=monitor_heartbeat, args=(httpd,))
-        monitor_thread.daemon = True
-        monitor_thread.start()
-
-        httpd.serve_forever()
-
-
-def main():
-    html = sys.stdin.read()
-    if "<body>" in html:
-        html_to_display = html.replace("</body>", f"{inject_script}</body>")
-    else:
-        html_to_display = f"<html><body>{html}{inject_script}</body></html>"
-
-    handler = handler_for_html(html_to_display)
-
-    # Start the server in a separate thread
-    port = random.randint(20000, 65535)
-    server_thread = threading.Thread(target=run_server, kwargs=dict(handler=handler, port=port))
-    server_thread.daemon = True
-    server_thread.start()
-
-    # Wait a moment for the server to start
-    time.sleep(0.5)
-
-    # Open the web browser
-    webbrowser.open(f"http://localhost:{port}")
-
-    # Wait for the server thread to finish
-    server_thread.join()
-    print("Server stopped")
-
-
-if __name__ == "__main__":
-    try:
-        main()
-    except KeyboardInterrupt:
-        print("\nReceived keyboard interrupt, exiting...")
-        raise SystemExit(0)

htmlpopup.go

@@ -1,119 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"log"
-	"net"
-	"net/http"
-	"os"
-	"os/exec"
-	"os/signal"
-	"runtime"
-	"strings"
-	"syscall"
-	"time"
-)
-
-// language=html
-const injectScript = `
-<script>
-	const sendShutdown = () => navigator.sendBeacon('/shutdown');
-
-	window.addEventListener('beforeunload', sendShutdown);
-	window.addEventListener('unload', sendShutdown);
-</script>
-`
-
-func main() {
-	if err := run(); err != nil {
-		log.Fatal(err)
-	}
-}
-
-func newHandler(html string, onShutdown func()) *http.ServeMux {
-	handler := http.NewServeMux()
-
-	handler.HandleFunc("POST /shutdown", func(w http.ResponseWriter, r *http.Request) {
-		onShutdown()
-	})
-
-	handler.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Type", "text/html")
-		w.Write([]byte(html))
-	})
-
-	return handler
-}
-
-func run() error {
-	html, err := io.ReadAll(os.Stdin)
-	if err != nil {
-		return fmt.Errorf("failed to read HTML from stdin: %w", err)
-	}
-
-	htmlStr := string(html)
-	if strings.Contains(htmlStr, "</body>") {
-		htmlStr = strings.ReplaceAll(string(html), "</body>", injectScript+"</body>")
-	} else {
-		htmlStr += injectScript
-	}
-
-	listener, err := net.Listen("tcp", ":0")
-	if err != nil {
-		return fmt.Errorf("failed to create listener: %w", err)
-	}
-
-	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
-	defer cancel()
-
-	port := listener.Addr().(*net.TCPAddr).Port
-
-	server := &http.Server{Addr: listener.Addr().String()}
-	server.Handler = newHandler(htmlStr, func() {
-		cancel()
-	})
-
-	servedURL := fmt.Sprintf("http://localhost:%d", port)
-
-	go func() {
-		log.Printf("serving at %s", servedURL)
-		if err := server.Serve(listener); err != nil && err != http.ErrServerClosed {
-			log.Fatalf("failed to serve: %v", err)
-		}
-	}()
-
-	go func() {
-		// wait for server to start
-		time.Sleep(250 * time.Millisecond)
-		if err := openBrowser(servedURL); err != nil {
-			log.Fatalf("failed to open browser: %v", err)
-		}
-	}()
-
-	<-ctx.Done()
-
-	log.Printf("shutting down server")
-
-	if err := server.Shutdown(ctx); err != nil {
-		return fmt.Errorf("failed to shutdown server: %w", err)
-	}
-
-	return nil
-}
-
-func openBrowser(url string) error {
-	var cmd *exec.Cmd
-	switch runtime.GOOS {
-	case "darwin":
-		cmd = exec.Command("open", url)
-	case "linux":
-		cmd = exec.Command("xdg-open", url)
-	case "windows":
-		cmd = exec.Command("cmd", "/c", "start", url)
-	default:
-		return fmt.Errorf("invalid URL: %s", url)
-	}
-	return cmd.Run()
-}

imgsheet.py

@@ -1,220 +0,0 @@
-#!/usr/bin/env python3.9
-import argparse
-import dataclasses
-import io
-import itertools
-import logging
-import math
-import statistics
-import sys
-import typing
-from pathlib import Path
-
-from PIL import Image
-from PIL import ImageDraw
-from PIL import ImageFont
-
-T = typing.TypeVar("T")
-
-logger = logging.getLogger("imgsheet")
-
-
-def chunk(it: typing.Iterable[T], size: int) -> typing.Iterable[typing.Iterable[T]]:
-    it = iter(it)
-    sentinel = ()
-    return iter(lambda: tuple(itertools.islice(it, size)), sentinel)
-
-
-@dataclasses.dataclass
-class PositionedImage:
-    path: Path
-    image_size: typing.Tuple[int, int]
-    size: typing.Tuple[int, int] = dataclasses.field(default_factory=lambda: (0, 0))
-    position: typing.Tuple[int, int] = dataclasses.field(default_factory=lambda: (0, 0))
-
-    @property
-    def top(self) -> int:
-        return self.position[1]
-
-    @property
-    def left(self) -> int:
-        return self.position[0]
-
-    @property
-    def bottom(self) -> int:
-        return self.height + self.top
-
-    @property
-    def right(self) -> int:
-        return self.width + self.left
-
-    @property
-    def width(self) -> int:
-        return self.size[0]
-
-    @property
-    def height(self) -> int:
-        return self.size[1]
-
-    def __post_init__(self):
-        self.size = self.image_size
-
-    def scale(self, scale: float) -> None:
-        self.size = (math.ceil(self.width * scale), math.ceil(self.height * scale))
-
-
-class ImageSheet:
-    def __init__(
-        self,
-        width: int,
-        columns: int = 6,
-        font_path: typing.Optional[Path] = None,
-        label_format: typing.Optional[str] = None,
-        label_font_size: typing.Optional[int] = 10,
-    ):
-        self.width = width
-        self.padding = 20
-        self.columns = columns
-        self.font_path = font_path
-        self.label_format = label_format
-        self.label_font_size = label_font_size
-
-    def create(self, image_paths: typing.Iterable[Path]) -> io.BytesIO:
-        images = []
-        for it in image_paths:
-            with Image.open(it) as img:
-                images.append(PositionedImage(it, img.size))
-
-        if self.font_path and self.font_path.is_file():
-            font = ImageFont.truetype(str(self.font_path), self.label_font_size)
-        else:
-            font = None
-
-        positioned = self._calculate_positions(images)
-        image_sheet = Image.new("RGB", (self.width, 0))
-        for row_images in positioned:
-            height = row_images[0].top + row_images[0].size[1]
-
-            extended_sheet = Image.new("RGB", (self.width, height + self.padding))
-            extended_sheet.paste(image_sheet, (0, 0))
-            image_sheet = extended_sheet
-
-            draw = ImageDraw.Draw(image_sheet)
-            for it in row_images:
-                img = Image.open(it.path)
-                img.thumbnail(it.size, Image.LANCZOS)
-                image_sheet.paste(img, it.position)
-
-                if font:
-                    # write filename on image
-                    text_x, text_y = it.left + 2, it.bottom + 2
-                    filesize = it.path.stat().st_size / 1_048_576
-
-                    text = self._render_text(
-                        width=it.image_size[0],
-                        height=it.image_size[1],
-                        size_mb=filesize,
-                        filename=it.path.name,
-                    )
-                    if text:
-                        draw.text((text_x, text_y), text, (255, 255, 255), font=font, align="left", anchor="la")
-
-                print(f"{it.path.name}\t{it.position}\t{it.size}")
-
-        f = io.BytesIO()
-        image_sheet.save(f, format="JPEG")
-        return f
-
-    def _render_text(self, width: int, height: int, size_mb: float, filename: str) -> typing.Optional[str]:
-        if not self.label_format:
-            return None
-
-        return self.label_format.format(width=width, height=height, size_mb=size_mb, filename=filename)
-
-    def _calculate_positions(self, images: list[PositionedImage]) -> list[list[PositionedImage]]:
-        cols = self.columns
-        width = self.width
-        padding = self.padding
-
-        positioned = []
-
-        rows = list(chunk(images, cols))
-        x, y = 0, 0
-        for row, row_images in enumerate(rows):
-            row_images = list(row_images)
-
-            usable_width = width - (len(row_images) - 1) * padding
-            max_height = max(image.height for image in row_images)
-            # check if there are enough images to fill the row
-            if len(row_images) < cols:
-                mean_height = statistics.median(prow[0].height for prow in positioned)
-                max_height = mean_height
-
-            for img in row_images:
-                y_scaling = max_height / img.height
-                img.scale(y_scaling)
-
-            if len(row_images) >= cols:
-                total_width = sum(image.width for image in row_images)
-                x_scaling = usable_width / total_width
-                max_height = math.ceil(max_height * x_scaling)
-                for img in row_images:
-                    img.scale(x_scaling)
-
-            is_last_row = row == len(rows) - 1
-            x = 0
-            for i, img in enumerate(row_images):
-                img.position = (x, y)
-                is_last_col = i == len(row_images) - 1
-                x = x + img.width + (0 if is_last_col else padding)
-            y = y + max_height + (0 if is_last_row else padding)
-
-            positioned.append(row_images)
-
-        return positioned
-
-
-def parse_args(argv: typing.List[str]) -> argparse.Namespace:
-    arger = argparse.ArgumentParser(
-        description="Create a sheet of images", formatter_class=argparse.ArgumentDefaultsHelpFormatter
-    )
-    arger.add_argument("--width", "-w", type=int, default=2000, help="Width of the sheet")
-    arger.add_argument("--columns", "-c", type=int, default=4, help="Number of columns")
-    arger.add_argument("--output", "-o", type=Path, default=Path("image_sheet.jpg"), help="Output file")
-    arger.add_argument("--font", dest="font_path", type=Path, help="Label font to use")
-    arger.add_argument("--font-size", dest="font_size", default=10, type=int, help="Label font size")
-    arger.add_argument(
-        "--label",
-        dest="label_format",
-        default="{width}x{height} {size_mb:.1f}MB {filename}",
-        help="Label format. Available tokens: width, height, size_mb, filename",
-    )
-    arger.add_argument("images", nargs="+", type=Path, help="Images or image directory to include in the sheet")
-    if len(argv) == 0:
-        arger.print_help()
-        sys.exit(1)
-    return arger.parse_args(argv)
-
-
-def main():
-    args = parse_args(sys.argv[1:])
-
-    images: list[Path] = args.images
-    if len(images) == 1 and images[0].expanduser().is_dir():
-        images = sorted(it for it in images[0].resolve().glob("*.jpg"))
-
-    sheet = ImageSheet(
-        width=args.width,
-        columns=args.columns,
-        font_path=args.font_path,
-        label_font_size=args.font_size,
-        label_format=args.label_format,
-    )
-
-    sheet_img = sheet.create(images)
-    with args.output.open("wb") as f:
-        f.write(sheet_img.getvalue())
-
-
-if __name__ == "__main__":
-    main()

immichup.py

@@ -24,11 +24,10 @@ from typing import Any, Sequence
 
 import httpx
 
-
 LOG = logging.getLogger("immichup")
 TAG_PORTRAIT = "aspect:portrait"
 TAG_LANDSCAPE = "aspect:landscape"
-STUDIO_RE = re.compile(r"\b@(\S+)\b")
+STUDIO_RE = re.compile(r"@([^_\s]+)")
 
 
 def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
@@ -68,7 +67,8 @@ def parse_filename(image_path: Path) -> tuple[list[str], str | None] | None:
     if not people:
         return None
     match = STUDIO_RE.search(filename)
-    return people, match.group(1) if match else None
+    studio = match.group(1) if match else None
+    return people, studio
 
 
 def read_jpeg_dimensions(path: Path) -> tuple[int, int]:
@@ -216,6 +216,7 @@ def process_image(path: Path, client: ImmichClient | None, dry_run: bool) -> boo
         albums = [person.lower() for person in people]
         tags = list(albums)
         if studio:
+            albums.append(f"@{studio.lower()}")
             tags.append(studio.lower())
         tags.append(TAG_PORTRAIT if width <= height else TAG_LANDSCAPE)
         if dry_run:

immichup_test.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env -S uv run --script
+# /// script
+# dependencies = ["pytest", "httpx>=0.27,<1"]
+# ///
+
+import sys
+from pathlib import Path
+from unittest.mock import Mock, patch
+
+import pytest
+
+import immichup
+
+
+@pytest.mark.parametrize(
+    "filename, expected_studio",
+    [
+        ("Irina Dreyt -- @Mavrin__0007-upscaled.jpg", "Mavrin"),
+        ("Irina Dreyt -- @Mavrin__0155.jpg", "Mavrin"),
+        ("Irina Dreyt -- @MAVRIN__0007-UPSCALED.jpg", "MAVRIN"),
+        ("Irina Dreyt -- @Mavrin -- Shoot__0007-upscaled.jpg", "Mavrin"),
+        ("Irina Dreyt -- @Studio_Name-2.jpg", "Studio"),
+        ("Irina Dreyt -- Shoot__0007-upscaled.jpg", None),
+    ],
+)
+def test_studio_excludes_image_number_and_upscale_suffix(
+    filename: str,
+    expected_studio: str | None,
+) -> None:
+    assert immichup.parse_filename(image_path=Path(filename)) == (["Irina Dreyt"], expected_studio)
+
+
+@pytest.mark.parametrize(
+    "filename",
+    [
+        "Irina Dreyt -- @Mavrin__0007-upscaled.jpg",
+        "Irina Dreyt -- @MAVRIN__0155.jpg",
+    ],
+)
+def test_studio_album_and_tag_use_lowercase_name_without_image_suffix(filename: str) -> None:
+    path = Path(filename)
+    client = Mock(spec=immichup.ImmichClient)
+    with patch.object(immichup, "read_jpeg_dimensions", return_value=(600, 800)):
+        assert immichup.process_image(path=path, client=client, dry_run=False)
+    client.upload_image.assert_called_once_with(
+        path,
+        ["irina dreyt", "@mavrin"],
+        ["irina dreyt", "mavrin", "aspect:portrait"],
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(args=[str(Path(__file__).resolve()), *sys.argv[1:]]))

jd2_hook.py

@@ -1,35 +0,0 @@
-import json
-import logging as logger
-import sys
-from pathlib import Path
-
-import file_renamer
-
-logger.basicConfig(
-    format=f"%(asctime)s {logger.BASIC_FORMAT}",
-    level=logger.DEBUG,
-    handlers=[
-        logger.FileHandler(Path(__file__).with_name("jd2_hook.log"), mode="a"),
-        logger.StreamHandler(),
-    ],
-)
-
-
-def main():
-    try:
-        payload = json.loads(sys.argv[1])
-    except (IndexError, ValueError):
-        logger.error("Expected JSON as first argument")
-        return 1
-
-    for link in payload["links"]:
-        path = link["download_path"]
-        f = Path(path)
-        logger.info("")
-        if f.is_file():
-            file_renamer.fix_dates([f])
-
-
-if __name__ == "__main__":
-    logger.debug(f"Got called with: {sys.argv}")
-    exit(main())

jkup.py

@@ -1,98 +0,0 @@
-#!/usr/bin/env python3.9
-import argparse
-import concurrent.futures
-import dataclasses
-import re
-import sys
-import time
-import typing
-from pathlib import Path
-from typing import Optional
-import logging
-from functools import cache
-
-
-import httpx
-
-
-@dataclasses.dataclass
-class UploadResult:
-    image_url: str
-    thumbnail_url: str
-
-session = httpx.Client(timeout=20)
-
-@cache
-def get_auth_token():
-    logging.info('extracting auth token')
-    res = session.get("https://jerking.empornium.ph/?agree-consent", follow_redirects=True)
-    try:
-        auth_token = re.search(r'name="auth_token" value="([^"]+)"', res.text).group(1)
-    except AttributeError:
-        auth_token = re.search(r'auth_token = "([^"]+)"', res.text).group(1)
-
-    logging.info(f'got {auth_token=}')
-
-    return auth_token
-
-def upload_image(image_path: Path) -> UploadResult:
-    auth_token = get_auth_token()
-
-    logging.info(f'uploading {image_path.name}')
-
-    with image_path.open("rb") as f:
-        res = session.post(
-            "https://jerking.empornium.ph/json",
-            headers={"accept": "application/json"},
-            data={
-                "thumb_width": "400",
-                "thumb_height": "400",
-                "thumb_crop": "false",
-                "medium_width": "600",
-                "medium_crop": "false",
-                "type": "file",
-                "action": "upload",
-                "timestamp": str(round(time.time() * 1000)),
-                "auth_token": auth_token,
-                "nsfw": "1",
-            },
-            files={
-                "source": f,
-            },
-        )
-
-    data = res.json()
-    logging.info(f'got response={data}')
-    image_url = data["image"]["url"]
-    thumbnail_url = data["image"]["display_url"]
-
-    logging.info(f'uploaded {image_path.name}. url={image_url}')
-
-    return UploadResult(image_url, thumbnail_url)
-
-
-def parse_args(argv: Optional[typing.Sequence[str]] = None) -> argparse.Namespace:
-    arger = argparse.ArgumentParser()
-    arger.add_argument('image_paths', nargs='+', action='store', type=Path, help='Path to image to upload')
-    if not argv or len(argv) == 0:
-        arger.print_help()
-        sys.exit(1)
-
-    return arger.parse_args(argv)
-
-
-def main():
-    args = parse_args(sys.argv[1:])
-    image_paths = args.image_paths
-
-    get_auth_token()
-
-    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
-        results = list(pool.map(upload_image, image_paths))
-    for it in results:
-        print(it.image_url)
-
-
-if __name__ == '__main__':
-    logging.basicConfig(level=logging.INFO)
-    main()

launchd_plist.py

@@ -1,84 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import logging
-import plistlib
-import sys
-import typing
-from pathlib import Path
-
-
-def parse_args(argv: list[str]) -> typing.Tuple[argparse.Namespace, list[str]]:
-    def parse_env(env: str) -> typing.Tuple[str, str]:
-        return env.split("=", maxsplit=1)
-
-    def parse_path(path: str) -> Path:
-        return Path(path).expanduser().resolve()
-
-    arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    arger.add_argument(
-        "-e", "--env", dest="env", metavar="ENVVAR", type=parse_env, nargs="+", help="Environment variables as K=V list"
-    )
-    arger.add_argument("-w", "--cwd", dest="cwd", default=Path.cwd(), type=parse_path, help="Working directory")
-    arger.add_argument("-l", "--label", required=True, help="Label")
-    arger.add_argument(
-        "--install",
-        action="store_true",
-        help="Install the plist to ~/Library/LaunchAgents/",
-    )
-
-    if not argv:
-        arger.print_help()
-        sys.exit(1)
-
-    args, unknown = arger.parse_known_args()
-    if unknown and unknown[0] == "--":
-        unknown = unknown[1:]
-
-    if not unknown:
-        arger.print_help()
-        sys.exit(1)
-
-    return args, unknown
-
-
-def generate_plist(label: str, args: list[str], env: dict, cwd: Path) -> str:
-    doc = {
-        "Label": label,
-        "ProgramArguments": args,
-        "EnvironmentVariables": env,
-        "WorkingDirectory": str(cwd),
-        "RunAtLoad": True,
-        "StandardOutPath": str(cwd / f"{label}.stdout.log"),
-        "StandardErrorPath": str(cwd / f"{label}.stderr.log"),
-    }
-    return plistlib.dumps(
-        doc,
-        fmt=plistlib.FMT_XML,
-        sort_keys=True,
-    ).decode()
-
-
-def main():
-    args, unknown = parse_args(sys.argv[1:])
-    print(args, unknown)
-    plist_doc = generate_plist(
-        label=args.label,
-        args=unknown,
-        env=dict(args.env),
-        cwd=args.cwd,
-    )
-    print(plist_doc)
-
-    if args.install:
-        install_dir: Path = Path("~/Library/LaunchAgents/").expanduser()
-        logging.info(f"Installing plist to {install_dir}")
-
-        install_path = install_dir / f"{args.label}.plist"
-        install_path.write_text(plist_doc)
-
-        logging.info(f'Run "launchctl load -w {install_path}" to start the service')
-
-
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.INFO)
-    main()

legacy/aws_codecommit.py

@@ -1,336 +0,0 @@
-import itertools
-import json
-import logging
-import pprint
-import re
-from collections import defaultdict
-from concurrent.futures import ThreadPoolExecutor
-from urllib.parse import quote_plus
-
-import boto3
-import httpx
-
-import urllib.parse
-import subprocess
-import typing
-
-
-def list_repo_tags(*, repository_name: str, region: str, username: str, password: str) -> typing.List[dict]:
-    """
-    Fetches the list of tags in a Codecommit repo using git ls-remote command
-
-    :returns: List of {tag, commit} items
-    """
-    url = "https://{username}:{password}@git-codecommit.{region}.amazonaws.com/v1/repos/{repository_name}".format(
-        username=quote_plus(username),
-        password=quote_plus(password),  # passwords are base64-encoded and might contain "/"
-        region=region,
-        repository_name=repository_name,
-    )
-    result = subprocess.run(["git", "ls-remote", "--refs", "--tags", url], text=True, stdout=subprocess.PIPE)
-
-    tags = []
-    for line in result.stdout.splitlines(keepends=False):
-        # lines are formatted as:
-        # 070161940636bab5d934af2702f4248e5a13eb29	refs/tags/ui_sandbox_acc104
-        # and columns are delimited with "\t"
-        if "refs/tags/" in line:
-            commit_hash, tag_ref = line.split("\t", maxsplit=1)
-            tag = tag_ref.replace("refs/tags/", "")
-            tags.append({"commit": commit_hash, "tag": tag})
-
-    # natural-sort by tag name
-    # flora_prod_1 < ... < flora_prod_9 > flora_prod_10
-    # vs lexical sort would give:
-    # flora_prod_1 < flora_prod_10 < ... < flora_prod_9
-    tags = sorted(tags, key=lambda t: [int(it) if it.isdigit() else it for it in re.split(r"(\d+)", t["tag"])])
-
-    return tags
-
-
-def chunk(it, size):
-    it = iter(it)
-    sentinel = ()
-    return iter(lambda: tuple(itertools.islice(it, size)), sentinel)
-
-
-def summarize_tags(tags: list[dict], take: int = 10) -> list[dict]:
-    re_prefix = re.compile(r"^(\D*)")
-    re_digits = re.compile(r"(\d+)")
-    grouped = defaultdict(list)
-    for t in tags:
-        tag_name = t["tag"]
-        if m := re_prefix.search(tag_name):
-            prefix = m.group(1)
-            grouped[prefix].append(t)
-        else:
-            print("no match", m)
-    flattened = [*grouped.pop("", [])]
-    for g, items in grouped.items():
-        sorted_items = sorted(
-            items, key=lambda t: [int(it) if it.isdigit() else it for it in re_digits.split(t["tag"])], reverse=True
-        )
-        flattened.extend(sorted_items[:take])
-
-    return flattened
-
-
-abdus_creds = {
-    "username": "coder+1-at-400344683105",
-    "password": "4xdnGOtghKiT0M5CjulWf5raWwBSqT0Df2YkIY9Ws68=",
-}
-
-
-def get_commit_content():
-    client = boto3.client("codecommit")
-
-    res = client.get_file(
-        repositoryName="testing",
-        filePath="content.txt1",
-        commitSpecifier="vnext",
-    )
-    print(res)
-
-
-def main():
-    import logging
-    from botocore.endpoint import Endpoint
-
-    logging.basicConfig(level=logging.INFO)
-
-    raw_make_request = Endpoint.make_request
-
-    def intercepted_make_request(*args, **kwargs):
-        aws_response, parsed_response = raw_make_request(*args, **kwargs)
-        logging.info('AWS response: %s', parsed_response)
-        return aws_response, parsed_response
-
-    Endpoint.make_request = intercepted_make_request
-
-    session = boto3.Session(
-        aws_access_key_id="AKIAQKZTNEVLBHDG5Q3X",
-        aws_secret_access_key="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",
-        region_name="eu-central-1",
-    )
-    cc = session.client("rds")
-    _ = cc.describe_db_clusters()
-
-
-if __name__ == "__main__":
-    main()
-    exit()
-    # get_commit_content()
-    # flora b1cd47a221b84b2884b3d24198085a9f-f0dc2e747ff241649715c20d48232d8e
-    # oms 36363cc2f43b4cf5b1965af9f8925a4d-6deff091ce2d4c2483d8834c532a6f36
-    # omnitron 36363cc2f43b4cf5b1965af9f8925a4d-97400227a37940539c55489e1354ffc2
-    all_tags = list_repo_tags(
-        repository_name="b1cd47a221b84b2884b3d24198085a9f-f0dc2e747ff241649715c20d48232d8e",
-        username="sandbox-acc-api-user-at-023193265494",
-        password="sYPWKdhSx4JliXNI4ffRb4qWYBHLxA54nczSJ6CmYCY=",
-        region="eu-central-1",
-    )
-    print(all_tags)
-    exit(0)
-    all_tags.append({"tag": "1asd", "commit": "asdf"})
-    summarized = summarize_tags(all_tags, take=5)
-    commits = {t["commit"] for t in summarized}
-    session = boto3.Session(
-        aws_access_key_id="AKIAQKZTNEVLBHDG5Q3X",
-        aws_secret_access_key="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",
-        region_name="eu-central-1",
-    )
-    client = session.client("codecommit")
-
-    def get_commit_infos(commits: list[str]):
-        res = client.batch_get_commits(
-            repositoryName="36363cc2f43b4cf5b1965af9f8925a4d-97400227a37940539c55489e1354ffc2",
-            commitIds=commits,
-        )
-        return res["commits"]
-
-    with ThreadPoolExecutor() as pool:
-        results = list(pool.map(get_commit_infos, chunk(commits, 100)))
-
-    clone_ssh_url = repo["repositoryMetadata"]["cloneUrlSsh"]
-    print(repo)
-    exit(0)
-# exit()
-from botocore.awsrequest import AWSRequest
-from botocore.signers import RequestSigner
-
-# AWS Version 4 signing example
-
-# DynamoDB API (CreateTable)
-
-# See: http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
-# This version makes a POST request and passes request parameters
-# in the body (payload) of the request. Auth information is passed in
-# an Authorization header.
-import sys, os, base64, datetime, hashlib, hmac
-import requests  # pip install requests
-
-# ************* REQUEST VALUES *************
-method = "POST"
-service = "codecommit"
-host = "codecommit.eu-central-1.amazonaws.com"
-region = "eu-central-1"
-endpoint = "https://codecommit.eu-central-1.amazonaws.com/"
-# POST requests use a content type header. For DynamoDB,
-# the content is JSON.
-content_type = "application/x-amz-json-1.1"
-# DynamoDB requires an x-amz-target header that has this format:
-#     DynamoDB_<API version>.<operationName>
-amz_target = "CodeCommit_20150413.GetReferences"
-
-# Request parameters for CreateTable--passed in a JSON block.
-request_body = {"repositoryName": "36363cc2f43b4cf5b1965af9f8925a4d-6deff091ce2d4c2483d8834c532a6f36"}
-request_parameters = json.dumps(request_body)
-
-
-# Key derivation functions. See:
-# http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python
-def sign(key, msg):
-    return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
-
-
-def getSignatureKey(key, date_stamp, regionName, serviceName):
-    kDate = sign(("AWS4" + key).encode("utf-8"), date_stamp)
-    kRegion = sign(kDate, regionName)
-    kService = sign(kRegion, serviceName)
-    kSigning = sign(kService, "aws4_request")
-    return kSigning
-
-
-# Read AWS access key from env. variables or configuration file. Best practice is NOT
-# to embed credentials in code.
-access_key = os.environ.get("AWS_ACCESS_KEY_ID", "AKIAQKZTNEVLBHDG5Q3X")
-secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY", "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")
-if access_key is None or secret_key is None:
-    print("No access key is available.")
-    sys.exit()
-
-# Create a date for headers and the credential string
-t = datetime.datetime.utcnow()
-amz_date = t.strftime("%Y%m%dT%H%M%SZ")
-date_stamp = t.strftime("%Y%m%d")  # Date w/o time, used in credential scope
-
-# ************* TASK 1: CREATE A CANONICAL REQUEST *************
-# http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
-
-# Step 1 is to define the verb (GET, POST, etc.)--already done.
-
-# Step 2: Create canonical URI--the part of the URI from domain to query
-# string (use '/' if no path)
-canonical_uri = "/"
-
-## Step 3: Create the canonical query string. In this example, request
-# parameters are passed in the body of the request and the query string
-# is blank.
-canonical_querystring = ""
-
-# Step 4: Create the canonical headers. Header names must be trimmed
-# and lowercase, and sorted in code point order from low to high.
-# Note that there is a trailing \n.
-canonical_headers = (
-    "content-type:"
-    + content_type
-    + "\n"
-    + "host:"
-    + host
-    + "\n"
-    + "x-amz-date:"
-    + amz_date
-    + "\n"
-    + "x-amz-target:"
-    + amz_target
-    + "\n"
-)
-
-# Step 5: Create the list of signed headers. This lists the headers
-# in the canonical_headers list, delimited with ";" and in alpha order.
-# Note: The request can include any headers; canonical_headers and
-# signed_headers include those that you want to be included in the
-# hash of the request. "Host" and "x-amz-date" are always required.
-# For DynamoDB, content-type and x-amz-target are also required.
-signed_headers = "content-type;host;x-amz-date;x-amz-target"
-
-# Step 6: Create payload hash. In this example, the payload (body of
-# the request) contains the request parameters.
-payload_hash = hashlib.sha256(request_parameters.encode("utf-8")).hexdigest()
-
-# Step 7: Combine elements to create canonical request
-canonical_request = (
-    method
-    + "\n"
-    + canonical_uri
-    + "\n"
-    + canonical_querystring
-    + "\n"
-    + canonical_headers
-    + "\n"
-    + signed_headers
-    + "\n"
-    + payload_hash
-)
-
-# ************* TASK 2: CREATE THE STRING TO SIGN*************
-# Match the algorithm to the hashing algorithm you use, either SHA-1 or
-# SHA-256 (recommended)
-algorithm = "AWS4-HMAC-SHA256"
-credential_scope = date_stamp + "/" + region + "/" + service + "/" + "aws4_request"
-string_to_sign = (
-    algorithm
-    + "\n"
-    + amz_date
-    + "\n"
-    + credential_scope
-    + "\n"
-    + hashlib.sha256(canonical_request.encode("utf-8")).hexdigest()
-)
-
-# ************* TASK 3: CALCULATE THE SIGNATURE *************
-# Create the signing key using the function defined above.
-signing_key = getSignatureKey(secret_key, date_stamp, region, service)
-
-# Sign the string_to_sign using the signing_key
-signature = hmac.new(signing_key, (string_to_sign).encode("utf-8"), hashlib.sha256).hexdigest()
-
-# ************* TASK 4: ADD SIGNING INFORMATION TO THE REQUEST *************
-# Put the signature information in a header named Authorization.
-authorization_header = (
-    algorithm
-    + " "
-    + "Credential="
-    + access_key
-    + "/"
-    + credential_scope
-    + ", "
-    + "SignedHeaders="
-    + signed_headers
-    + ", "
-    + "Signature="
-    + signature
-)
-
-# For DynamoDB, the request can include any headers, but MUST include "host", "x-amz-date",
-# "x-amz-target", "content-type", and "Authorization". Except for the authorization
-# header, the headers must be included in the canonical_headers and signed_headers values, as
-# noted earlier. Order here is not significant.
-# # Python note: The 'host' header is added automatically by the Python 'requests' library.
-headers = {
-    "User-Agent": "aws-sdk-js/2.627.0 promise",
-    "Content-Type": content_type,
-    "X-Amz-Date": amz_date,
-    "X-Amz-Target": amz_target,
-    "Authorization": authorization_header,
-}
-
-# ************* SEND THE REQUEST *************
-print("\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++")
-print("Request URL = " + endpoint)
-
-r = httpx.post(endpoint, json=request_body, headers=headers)
-
-print("\nRESPONSE++++++++++++++++++++++++++++++++++++")
-print("Response code: %d\n" % r.status_code)
-print(r.text)

legacy/aws_elasticache.py

@@ -1,438 +0,0 @@
-import dataclasses
-import functools
-import itertools
-import logging
-import re
-import time
-import typing
-import uuid
-from typing import Optional
-
-import boto3
-import botocore.exceptions
-from botocore.exceptions import ClientError
-
-logger = logging.getLogger(__name__)
-
-
-def retry(timeout: int, backoff_seconds: int = 10, bubble_errors: typing.List[typing.Type[Exception]] = None):
-    """
-    Tries calling a function {timeout} seconds until it succeeds or gives up and throw an error
-    :param timeout: Timeout in seconds
-    :param backoff_seconds: Wait time between retries
-    :param bubble_errors: List of exceptions to bubble up
-    :return: Wrapped function
-    """
-    bubble_errors = tuple(bubble_errors or [])
-
-    def wrapped(func: typing.Callable):
-        @functools.wraps(func)
-        def inner(*args, **kwargs):
-            waited = 0
-            while True:
-                try:
-                    logger.debug(f"Calling {func}")
-                    return func(*args, **kwargs)
-                except KeyboardInterrupt:
-                    raise
-                except tuple(bubble_errors):
-                    raise
-                except Exception as e:
-                    doze = min(timeout - waited, backoff_seconds)
-                    times = (timeout - waited) // backoff_seconds
-                    logger.debug(f"{func} failed, will try {times} times in {doze} sec")
-                    time.sleep(doze)
-                    waited += backoff_seconds
-                    if waited >= timeout:
-                        raise TimeoutError(f"Couldn't get a successful result from {func} in {timeout} seconds") from e
-
-        return inner
-
-    return wrapped
-
-
-@dataclasses.dataclass
-class RedisInstance:
-    cluster_id: str
-    host: str
-    port: int
-    arn: str
-    user_group_ids: typing.List[str] = dataclasses.field(default_factory=list)
-
-
-def paginate_by_marker(
-    func: callable,
-    list_field: str,
-) -> typing.Iterable:
-    """
-    Simplifies paginating over AWS results with a simpler interface than Paginators.
-    :param func: A function that accepts a parameter named `Marker`
-    :param list_field: Result field to iterate on
-    """
-    marker = ""
-    while True:
-        result = func(Marker=marker)
-
-        items = result.get(list_field, [])
-        yield from items
-
-        if "Marker" not in result:
-            break
-
-        marker = result["Marker"]
-
-
-@dataclasses.dataclass
-class RedisUser:
-    username: str
-    password: str
-    redis_key_prefix: str
-
-
-class CreateUserResult(typing.TypedDict):
-    user_group_id: str
-    users: typing.List[RedisUser]
-    status: str
-
-
-class RedisService:
-    def __init__(self, region: str):
-        self.elasticache = boto3.client("elasticache", region_name=region)
-
-    def create_redis_acl_user(
-        self,
-        replication_group_id: str,
-        users: typing.List[RedisUser],
-    ) -> None:
-        """
-        Creates a Redis ACL user. It will throw an error if the user group attached to the replication group is not ready.
-        This prevents us from quickly creating users. In that case, pass in a list of users instead.
-
-        It takes about 2 minutes until the user is ready.
-        """
-        repl_groups = self.elasticache.describe_replication_groups(ReplicationGroupId=replication_group_id)
-        user_group_id: str = repl_groups['ReplicationGroups'][0]['UserGroupIds'][0]
-        group = self.elasticache.describe_user_groups(UserGroupId=user_group_id)['UserGroups'][0]
-
-        """
-        Limits imposed by AWS:
-        
-        - User groups per replication group = 1
-        - Users per user group = 100
-        - Number of users = 1000
-        - Number of user groups = 100
-        https://docs.aws.amazon.com/AmazonElastiCache/latest/red-ug/Clusters.RBAC.html#Users-groups-to-RGs
-        
-        In short, we can't place more than 100 users on a single Redis instance.   
-        """
-
-        total_users = len(group['UserIds'])
-        if total_users >= 100:
-            raise Exception('Replication group reached the limit of 100 users')
-
-        self._update_or_create_user_group(user_group_id, users)
-
-    def delete_redis_acl_user(self, username: str) -> None:
-        logger.info(f'Deleting user {username}')
-        self.elasticache.delete_user(UserId=username)
-
-        # Even though we just deleted the user, it takes a couple of seconds until AWS updates user group status.
-        # So, we have to wait a bit before we can actually wait and check that the changes have propagated.
-        # Overall, it should take about 2-3 minutes to delete a user.
-
-    def create_redis_acl(
-        self,
-        *,
-        replication_group_id: str,
-        description: str = None,
-        node_type: str,
-        cache_subnet_group_name: str,
-        security_group_id: str,
-        tags: typing.Dict[str, str] = None,
-        users: typing.List[RedisUser],
-    ) -> dict:
-        user_group_id = f"{replication_group_id}-ug"
-        # we have to wait (~1m) until the user group is ready before we can create the ACL
-        self._update_or_create_user_group(user_group_id, users, wait=True)
-
-        cache_param_group = 'redis-acl-with-100-db'
-        self._ensure_cache_parameter_group(cache_param_group)
-
-        logger.info(f"Creating Redis replication group {replication_group_id=}")
-        default_replication_group_kwargs = dict(
-            ReplicasPerNodeGroup=0,
-            Engine="redis",
-            EngineVersion="6.x",
-            TransitEncryptionEnabled=True,  # must be True to use Redis ACL
-            MultiAZEnabled=False,
-            AutomaticFailoverEnabled=False,
-        )
-        repl_result = self.elasticache.create_replication_group(
-            **default_replication_group_kwargs,
-            ReplicationGroupId=replication_group_id,
-            ReplicationGroupDescription=description or f'{replication_group_id} replication group',
-            CacheNodeType=node_type,
-            CacheSubnetGroupName=cache_subnet_group_name,
-            CacheParameterGroupName=cache_param_group,
-            UserGroupIds=[
-                user_group_id,
-            ],
-            SecurityGroupIds=[
-                security_group_id,
-            ],
-            Tags=[{"Key": k, "Value": v} for k, v in (tags or {}).items()],
-        )
-
-        # replication group takes ~8m minutes to be ready
-
-        return {
-            "replication_group_id": repl_result["ReplicationGroup"]["ReplicationGroupId"],
-            'arn': repl_result["ReplicationGroup"]["ARN"],
-            'status': repl_result["ReplicationGroup"]["Status"],
-            'users': users,
-            '_response': repl_result,
-        }
-
-    def _update_or_create_user_group(
-        self, user_group_id: str, users: typing.List[RedisUser], wait: bool = False
-    ) -> None:
-        """
-        Creates or updates a user group with the given users. Waits until the user group is active.
-
-        :param user_group_id:
-        :param users:
-        """
-        # we have to add the `default` user for backwards compatibility
-        existing_users = self.elasticache.describe_users(
-            Filters=[{"Name": "user-id", "Values": [u.username for u in users]}],
-        )["Users"]
-        existing_user_ids = [it["UserId"] for it in existing_users]
-
-        created_user_ids = ["default"]
-        for user in users:
-            if user.username in existing_user_ids:
-                logger.info(f"User {user.username} is already exists")
-                created_user_ids.append(user.username)
-                continue
-
-            redis_acl = f"on +@all -@dangerous ~{user.redis_key_prefix}*"
-            logger.info(f"Creating tenant {user.username=}")
-            user_result = self.elasticache.create_user(
-                UserId=user.username,
-                UserName=user.username,
-                Passwords=[user.password],
-                AccessString=redis_acl,
-                Engine="redis",
-            )
-            created_user_ids.append(user_result["UserId"])
-
-        try:
-            group = self.elasticache.describe_user_groups(UserGroupId=user_group_id)["UserGroups"][0]
-            member_user_ids: typing.List[str] = group["UserIds"]
-            users_to_add = set(created_user_ids) - set(member_user_ids)
-
-            if users_to_add:
-                self.elasticache.modify_user_group(
-                    UserGroupId=user_group_id,
-                    UserIdsToAdd=list(users_to_add),
-                )
-                # this will take some time (~45s) until the changes propagate
-        except self.elasticache.exceptions.UserGroupNotFoundFault:
-            logger.info('User group does not exist, creating')
-            _ = self.elasticache.create_user_group(
-                UserGroupId=user_group_id,
-                Engine="redis",
-                UserIds=created_user_ids,
-            )
-            # user group creation takes around 60s
-            if wait:
-                self._wait_user_group(user_group_id)
-
-    @retry(timeout=60, backoff_seconds=5)
-    def _wait_user_group(self, user_group_id: str) -> None:
-        logger.info(f'Checking status of user group {user_group_id=}')
-        g = self.elasticache.describe_user_groups(UserGroupId=user_group_id)["UserGroups"][0]
-        assert g["Status"] == "active"
-
-    def is_redis_acl_ready(self, replication_group_id: str) -> bool:
-        repl = self.elasticache.describe_replication_groups(ReplicationGroupId=replication_group_id)[
-            'ReplicationGroups'
-        ][0]
-        is_redis_available = repl['Status'] == 'available'
-
-        user_group_id = repl['UserGroupIds'][0]
-        group = self.elasticache.describe_user_groups(UserGroupId=user_group_id)['UserGroups'][0]
-        is_group_available = group['Status'] == 'active'
-
-        return is_group_available and is_redis_available
-
-    def get_redis_acl(self, replication_group_id: str) -> dict:
-        redis = self.elasticache.describe_replication_groups(ReplicationGroupId=replication_group_id)[
-            'ReplicationGroups'
-        ][0]
-        return dict(
-            replication_group_id=redis['ReplicationGroupId'],
-            arn=redis['ARN'],
-            host=redis["NodeGroups"][0]["PrimaryEndpoint"]["Address"],
-            port=redis["NodeGroups"][0]["PrimaryEndpoint"]["Port"],
-        )
-
-    def delete_redis_acl(self, replication_group_id: str) -> None:
-        self.elasticache.delete_replication_group(ReplicationGroupId=replication_group_id)
-
-    def _ensure_cache_parameter_group(self, parameter_group_name: str) -> None:
-        try:
-            _ = self.elasticache.describe_cache_parameter_groups(CacheParameterGroupName=parameter_group_name)[
-                'CacheParameterGroups'
-            ][0]
-            return
-        except self.elasticache.exceptions.CacheParameterGroupNotFoundFault:
-            pass
-
-        logger.info(f"Creating cache parameter group {parameter_group_name=}")
-        _ = self.elasticache.create_cache_parameter_group(
-            CacheParameterGroupName=parameter_group_name,
-            CacheParameterGroupFamily="redis6.x",
-            Description="Redis parameter group for the Redis ACL",
-        )
-        _ = self.elasticache.modify_cache_parameter_group(
-            CacheParameterGroupName=parameter_group_name,
-            ParameterNameValues=[
-                {
-                    "ParameterName": "databases",
-                    "ParameterValue": "100",
-                },
-            ],
-        )
-
-
-def akinon_create_redis(
-    *,
-    k8s_cluster_name: str,
-    region: str,
-    node_type: str,
-    replicas: int,
-    owner_arn: str,
-    app_name: str,
-    role: str,
-):
-    subnet_group_name = f"{k8s_cluster_name}-redis-subg"
-    security_group_name = f"{k8s_cluster_name}-redis-sg"
-
-    # owner arn is formatted like:
-    # arn:aws:iam::412344683105:user/myusername
-    username = owner_arn.split("/")[1]
-    tags = {
-        "CostCenter": f"{username}-{app_name}-{role}-redis",
-    }
-
-    cache_cluster_id = uuid.uuid4().hex
-    r = RedisService(region)
-    return r.create_redis(
-        cluster_id=cache_cluster_id,
-        node_type=node_type,
-        replicas=replicas,
-        security_group_name=security_group_name,
-        subnet_group_name=subnet_group_name,
-        tags=tags,
-    )
-
-
-def akinon_create_redis_with_acl(
-    k8s_cluster_name: str,
-    owner_arn: str,
-    node_type: str,
-    app_name: str,
-    role: str,
-    users: typing.List[RedisUser],
-) -> RedisInstance:
-    cache_cluster_id = uuid.uuid4().hex
-    subnet_group_name = f"{k8s_cluster_name}-redis-subg"
-    security_group_name = f"{k8s_cluster_name}-redis-sg"
-
-    # owner arn is formatted as:
-    # arn:aws:iam::412344683105:user/myusername
-    username = owner_arn.split("/")[1]
-    tags = {
-        "CostCenter": f"{username}-{app_name}-{role}-redis",
-    }
-
-    return r.create_redis_acl(
-        cluster_id=cache_cluster_id,
-        node_type=node_type,
-        subnet_group_name=subnet_group_name,
-        security_group_name=security_group_name,
-        tags=tags,
-        users=users,
-    )
-
-
-def chunk(it, size):
-    it = iter(it)
-    sentinel = ()
-    return iter(lambda: tuple(itertools.islice(it, size)), sentinel)
-
-
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.INFO, format=f"%(asctime)s: {logging.BASIC_FORMAT}")
-    r = RedisService(region="eu-central-1")
-    # res = r.create_redis_acl(
-    #     replication_group_id="zerodev3",
-    #     description="zero redis dev",
-    #     node_type="cache.t3.micro",
-    #     cache_subnet_group_name="dev",
-    #     security_group_id="sg-c11a77ac",
-    #     tags={"CostCenter": "zero123"},
-    #     users=[RedisUser(username="u1", password="zeropassword1234", redis_key_prefix="zero:")],
-    # )
-    # print(res)
-
-    r.create_redis_acl_user(
-        replication_group_id='zerodev3',
-        users=[
-            RedisUser(
-                username='u2',
-                password='testtesttesttest123',
-                redis_key_prefix='test:',
-            )
-        ],
-    )
-    # r.delete_redis_acl_user('test6')
-    while True:
-        if r.is_redis_acl_ready('zerodev3'):
-            logger.info('ready')
-            print(r.get_redis_acl('zerodev3'))
-            break
-        logger.info('not ready')
-        time.sleep(10)
-    exit()
-
-    res = r.create_redis_acl(
-        replication_group_id="zerodev",
-        description="zero redis dev",
-        node_type="cache.t3.micro",
-        cache_subnet_group_name="dev",
-        security_group_id="sg-c11a77ac",
-        tags={"CostCenter": "zero123"},
-        users=[RedisUser(username="zerouser", password="zeropassword1234", redis_key_prefix="zero:")],
-    )
-    print(res)
-    #
-    # redis = r.get_redis_acl(cluster_arn="arn:aws:elasticache:eu-central-1:400344683105:replicationgroup:zero")
-    # r.add_redis_user(
-    #     cluster_id=redis.cluster_id,
-    #     users=[RedisUser(username="zero27", password="zero27password123", redis_key_prefix="zero27:")],
-    # )
-    # print(r.add_redis_user(cluster_id="zero1", users=[RedisUser("zero26", "zero26password123", "zero26:")]))
-    #
-    # print(
-    #     r.create_redis_with_acl(
-    #         cluster_id="zero",
-    #         cluster_description="zero redis",
-    #         node_type="cache.t3.micro",
-    #         subnet_group_name="testing-subnet",
-    #         security_group_name="default",
-    #         tags={"CostCenter": "zero123"},
-    #         users=[RedisUser(username="zerouser", password="zeropassword1234", redis_key_prefix="zero:")],
-    #     )
-    # )

legacy/aws_es.py

@@ -1,238 +0,0 @@
-import dataclasses
-import datetime
-import functools
-import json
-import logging
-import time
-import typing
-import uuid
-
-import boto3
-import botocore
-import requests
-from botocore.client import BaseClient
-from botocore.errorfactory import BaseClientExceptions
-from botocore.exceptions import ClientError
-
-logger = logging.getLogger(__name__)
-
-# fmt: off
-_response_describe_elasticsearch_domain_success = {'ResponseMetadata': {'RequestId': '38d7b555-2699-4868-9139-c47b1c2d70db', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '38d7b555-2699-4868-9139-c47b1c2d70db', 'content-type': 'application/json', 'content-length': '2314', 'date': 'Tue, 11 Jan 2022 07:52:24 GMT'}, 'RetryAttempts': 0}, 'DomainStatus': {'DomainId': '400344683105/es-d8a24850a0ae4b82af26', 'DomainName': 'es-d8a24850a0ae4b82af26', 'ARN': 'arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26', 'Created': True, 'Deleted': False, 'Endpoints': {'vpc': 'vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com'}, 'Processing': False, 'UpgradeProcessing': False, 'ElasticsearchVersion': '7.8', 'ElasticsearchClusterConfig': {'InstanceType': 't2.medium.elasticsearch', 'InstanceCount': 2, 'DedicatedMasterEnabled': False, 'ZoneAwarenessEnabled': True, 'ZoneAwarenessConfig': {'AvailabilityZoneCount': 2}, 'WarmEnabled': False, 'ColdStorageOptions': {'Enabled': False}}, 'EBSOptions': {'EBSEnabled': True, 'VolumeType': 'gp2', 'VolumeSize': 10}, 'AccessPolicies': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"es:*","Resource":"*"}]}', 'SnapshotOptions': {'AutomatedSnapshotStartHour': 0}, 'VPCOptions': {'VPCId': 'vpc-8c28b5e7', 'SubnetIds': ['subnet-004909d95a03c5fcd', 'subnet-b8f596f5'], 'AvailabilityZones': ['eu-central-1a', 'eu-central-1c'], 'SecurityGroupIds': ['sg-c11a77ac']}, 'CognitoOptions': {'Enabled': False}, 'EncryptionAtRestOptions': {'Enabled': False}, 'NodeToNodeEncryptionOptions': {'Enabled': False}, 'AdvancedOptions': {'override_main_response_version': 'false', 'rest.action.multi.allow_explicit_index': 'true'}, 'ServiceSoftwareOptions': {'CurrentVersion': 'R20211203-P2', 'NewVersion': '', 'UpdateAvailable': False, 'Cancellable': False, 'UpdateStatus': 'COMPLETED', 'Description': 'There is no software update available for this domain.', 'AutomatedUpdateDate': datetime.datetime(2021, 12, 14, 3, 38, 38), 'OptionalDeployment': False}, 'DomainEndpointOptions': {'EnforceHTTPS': True, 'TLSSecurityPolicy': 'Policy-Min-TLS-1-0-2019-07', 'CustomEndpointEnabled': False}, 'AdvancedSecurityOptions': {'Enabled': False, 'InternalUserDatabaseEnabled': False}, 'AutoTuneOptions': {'State': 'ENABLE_IN_PROGRESS'}}}
-_response_list_domain_names_success = {'ResponseMetadata': {'RequestId': '25923514-a426-4223-939a-4436f1d6e243', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '25923514-a426-4223-939a-4436f1d6e243', 'content-type': 'application/json', 'content-length': '87', 'date': 'Tue, 11 Jan 2022 07:53:37 GMT'}, 'RetryAttempts': 0}, 'DomainNames': [{'DomainName': 'es-d8a24850a0ae4b82af26', 'EngineType': 'Elasticsearch'}]}
-_response_describe_elasticsearch_domains_success = {'ResponseMetadata': {'RequestId': '61cfe5f4-42e7-4233-bca9-53f0ca62099d', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '61cfe5f4-42e7-4233-bca9-53f0ca62099d', 'content-type': 'application/json', 'content-length': '2320', 'date': 'Tue, 11 Jan 2022 07:55:25 GMT'}, 'RetryAttempts': 0}, 'DomainStatusList': [{'DomainId': '400344683105/es-d8a24850a0ae4b82af26', 'DomainName': 'es-d8a24850a0ae4b82af26', 'ARN': 'arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26', 'Created': True, 'Deleted': False, 'Endpoints': {'vpc': 'vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com'}, 'Processing': False, 'UpgradeProcessing': False, 'ElasticsearchVersion': '7.8', 'ElasticsearchClusterConfig': {'InstanceType': 't2.medium.elasticsearch', 'InstanceCount': 2, 'DedicatedMasterEnabled': False, 'ZoneAwarenessEnabled': True, 'ZoneAwarenessConfig': {'AvailabilityZoneCount': 2}, 'WarmEnabled': False, 'ColdStorageOptions': {'Enabled': False}}, 'EBSOptions': {'EBSEnabled': True, 'VolumeType': 'gp2', 'VolumeSize': 10}, 'AccessPolicies': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"es:*","Resource":"*"}]}', 'SnapshotOptions': {'AutomatedSnapshotStartHour': 0}, 'VPCOptions': {'VPCId': 'vpc-8c28b5e7', 'SubnetIds': ['subnet-004909d95a03c5fcd', 'subnet-b8f596f5'], 'AvailabilityZones': ['eu-central-1a', 'eu-central-1c'], 'SecurityGroupIds': ['sg-c11a77ac']}, 'CognitoOptions': {'Enabled': False}, 'EncryptionAtRestOptions': {'Enabled': False}, 'NodeToNodeEncryptionOptions': {'Enabled': False}, 'AdvancedOptions': {'override_main_response_version': 'false', 'rest.action.multi.allow_explicit_index': 'true'}, 'ServiceSoftwareOptions': {'CurrentVersion': 'R20211203-P2', 'NewVersion': '', 'UpdateAvailable': False, 'Cancellable': False, 'UpdateStatus': 'COMPLETED', 'Description': 'There is no software update available for this domain.', 'AutomatedUpdateDate': datetime.datetime(2021, 12, 14, 3, 38, 38), 'OptionalDeployment': False}, 'DomainEndpointOptions': {'EnforceHTTPS': True, 'TLSSecurityPolicy': 'Policy-Min-TLS-1-0-2019-07', 'CustomEndpointEnabled': False}, 'AdvancedSecurityOptions': {'Enabled': False, 'InternalUserDatabaseEnabled': False}, 'AutoTuneOptions': {'State': 'ENABLE_IN_PROGRESS'}}]}
-_response_delete_elasticsearch_domain_not_found = ClientError({'Error': {'Message': 'Domain not found: asd', 'Code': 'ResourceNotFoundException'}, 'ResponseMetadata': {'RequestId': 'd0669560-77f6-424c-bbc6-fb5878daa494', 'HTTPStatusCode': 409, 'HTTPHeaders': {'x-amzn-requestid': 'd0669560-77f6-424c-bbc6-fb5878daa494', 'x-amzn-errortype': 'ResourceNotFoundException', 'content-type': 'application/json', 'content-length': '35', 'date': 'Tue, 11 Jan 2022 08:06:07 GMT'}, 'RetryAttempts': 0}}, 'DeleteElasticsearchDomain')
-_response_delete_elasticsearch_domain_success = {'ResponseMetadata': {'RequestId': '1e9046f4-c4ee-4fda-8dfe-dd23f0161b7e', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '1e9046f4-c4ee-4fda-8dfe-dd23f0161b7e', 'content-type': 'application/json', 'content-length': '2312', 'date': 'Tue, 11 Jan 2022 08:31:33 GMT'}, 'RetryAttempts': 0}, 'DomainStatus': {'DomainId': '400344683105/es-d8a24850a0ae4b82af26', 'DomainName': 'es-d8a24850a0ae4b82af26', 'ARN': 'arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26', 'Created': True, 'Deleted': True, 'Endpoints': {'vpc': 'vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com'}, 'Processing': True, 'UpgradeProcessing': False, 'ElasticsearchVersion': '7.8', 'ElasticsearchClusterConfig': {'InstanceType': 't2.medium.elasticsearch', 'InstanceCount': 2, 'DedicatedMasterEnabled': False, 'ZoneAwarenessEnabled': True, 'ZoneAwarenessConfig': {'AvailabilityZoneCount': 2}, 'WarmEnabled': False, 'ColdStorageOptions': {'Enabled': False}}, 'EBSOptions': {'EBSEnabled': True, 'VolumeType': 'gp2', 'VolumeSize': 10}, 'AccessPolicies': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"es:*","Resource":"*"}]}', 'SnapshotOptions': {'AutomatedSnapshotStartHour': 0}, 'VPCOptions': {'VPCId': 'vpc-8c28b5e7', 'SubnetIds': ['subnet-004909d95a03c5fcd', 'subnet-b8f596f5'], 'AvailabilityZones': ['eu-central-1a', 'eu-central-1c'], 'SecurityGroupIds': ['sg-c11a77ac']}, 'CognitoOptions': {'Enabled': False}, 'EncryptionAtRestOptions': {'Enabled': False}, 'NodeToNodeEncryptionOptions': {'Enabled': False}, 'AdvancedOptions': {'override_main_response_version': 'false', 'rest.action.multi.allow_explicit_index': 'true'}, 'ServiceSoftwareOptions': {'CurrentVersion': 'R20211203-P2', 'NewVersion': '', 'UpdateAvailable': False, 'Cancellable': False, 'UpdateStatus': 'COMPLETED', 'Description': 'There is no software update available for this domain.', 'AutomatedUpdateDate': datetime.datetime(2021, 12, 14, 3, 38, 38), 'OptionalDeployment': False}, 'DomainEndpointOptions': {'EnforceHTTPS': True, 'TLSSecurityPolicy': 'Policy-Min-TLS-1-0-2019-07', 'CustomEndpointEnabled': False}, 'AdvancedSecurityOptions': {'Enabled': False, 'InternalUserDatabaseEnabled': False}, 'AutoTuneOptions': {'State': 'ENABLE_IN_PROGRESS'}}}
-# fmt: on
-
-
-def retry(timeout: int, backoff_seconds: int = 10, bubble_errors: typing.List[typing.Type[Exception]] = None):
-    """
-    Tries calling a function {timeout} seconds until it succeeds or gives up and throw an error
-    :param timeout: Timeout in seconds
-    :param backoff_seconds: Wait time between retries
-    :param bubble_errors: List of exceptions to bubble up
-    :return: Wrapped function
-    """
-    bubble_errors = tuple(bubble_errors or [])
-
-    def wrapped(func: typing.Callable):
-        @functools.wraps(func)
-        def func_retrier(*args, **kwargs):
-            waited = 0
-            started_at = datetime.datetime.now()
-            while True:
-                try:
-                    logger.debug(f"Calling {func}")
-                    result = func(*args, **kwargs)
-                    logger.debug(f"{func} completed in {(datetime.datetime.now() - started_at)}")
-                    return result
-                except KeyboardInterrupt:
-                    raise
-                except tuple(bubble_errors):
-                    raise
-                except Exception as e:
-                    doze = min(timeout - waited, backoff_seconds)
-                    times = (timeout - waited) // backoff_seconds
-                    logger.debug(f"{func} failed, will try {times} times in {doze} sec", exc_info=True)
-                    time.sleep(doze)
-                    waited += backoff_seconds
-                    if waited >= timeout:
-                        raise TimeoutError(f"Couldn't get a successful result from {func} in {timeout} seconds") from e
-
-        return func_retrier
-
-    return wrapped
-
-
-@dataclasses.dataclass
-class ElasticsearchInstance:
-    domain_name: str
-    endpoint: str
-    arn: str
-
-
-class ElasticsearchService:
-    domain_name_format = "es-{}"
-    default_version = "7.8"
-    valid_versions = {"7.10", "7.9", "7.8", "7.7", "7.4", "7.1", "6.8", "6.7", "6.5", "6.4", "6.3", "6.2", "6.0", "5.6", "5.5"}  # fmt: skip
-
-    def __init__(self, region: str):
-        self.es = boto3.client("es", region_name=region)
-
-    def create_instance(
-        self,
-        node_type: str,
-        total_nodes: int,
-        subnet_group_ids: typing.List[str],
-        security_group_id: str,
-        version: str = None,
-        tags: dict = None,
-        volume_size_gb: int = 30,
-        total_availability_zones: int = 3,
-    ) -> ElasticsearchInstance:
-        if not version:
-            version = self.default_version
-        assert version in self.valid_versions, "Invalid version"
-
-        domain_name = self.domain_name_format.format(uuid.uuid4().hex[:20])
-        logger.info(
-            "Creating a new Elasticsearch domain",
-            extra=dict(version=version, domain_name=domain_name, node_type=node_type, total_nodes=total_nodes),
-        )
-        es_result = self.es.create_elasticsearch_domain(
-            DomainName=domain_name,
-            ElasticsearchVersion=version,
-            ElasticsearchClusterConfig={
-                "InstanceType": node_type,
-                "InstanceCount": total_nodes,
-                "ZoneAwarenessEnabled": True,
-                "ZoneAwarenessConfig": {
-                    "AvailabilityZoneCount": total_availability_zones,
-                },
-            },
-            VPCOptions={
-                "SubnetIds": subnet_group_ids,
-                "SecurityGroupIds": [security_group_id],
-            },
-            EBSOptions={
-                "EBSEnabled": True,
-                "VolumeType": "gp2",
-                "VolumeSize": volume_size_gb,
-            },
-            AccessPolicies=json.dumps(
-                {
-                    "Version": "2012-10-17",
-                    "Statement": [{"Effect": "Allow", "Principal": "*", "Action": "es:*", "Resource": "*"}],
-                }
-            ),
-            DomainEndpointOptions={"EnforceHTTPS": True},
-            TagList=[{"Key": k, "Value": v} for k, v in (tags or {}).items()],
-        )
-        logger.info("Waiting until ES domain becomes available. This will take a while (~15 min)")
-        return self._find_instance(domain_name)
-
-    @retry(timeout=20 * 60, backoff_seconds=30)
-    def _find_instance(self, domain_name: str) -> ElasticsearchInstance:
-        result = self.es.describe_elasticsearch_domain(DomainName=domain_name)["DomainStatus"]
-        return ElasticsearchInstance(endpoint=result["Endpoints"]["vpc"], arn=result["ARN"], domain_name=domain_name)
-
-    def find_domain_name_by_arn(self, arn: str):
-        res = self.es.list_domain_names(EngineType='Elasticsearch')
-        domain_names = [it['DomainName'] for it in res['DomainNames']]
-
-        res = self.es.describe_elasticsearch_domains(DomainNames=domain_names)
-        domain_name_by_arn = {it['ARN']: it['DomainName'] for it in res['DomainStatusList']}
-
-        print(res)
-
-
-def main():
-    username, password = 'bgNNs4EWPTsuQJ24', '3asinFjikT5MgwRz!'
-
-
-"""
-GET _search
-{
-  "query": {
-    "match_all": {}
-  }
-}
-
-###
-PUT /my-index
-PUT /my-index2
-PUT /my-index3
-PUT /my-index3-4
-
-###
-DELETE /my-index*?expand_wildcards=all
-
-###
-DELETE /my-index1
-
-###
-GET /my-index
-GET /_stats/indexing
-###
-GET /_cat/indices?v
-
-###
-
-GET /kibana_sample_data_ecommerce/
-
-###
-GET /_all/_mapping
-
-## delete multiple
-PUT /my-index
-PUT /my-index2
-DELETE /my-index,my-index2
-GET /my-index
-
-###
-GET /_aliases
-
-###
-GET /_stats/
-
-
-user: qdSmWKS8p3VrUyBC
-pwd: un6f2vTrRjoa8orn!
-"""
-
-
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.INFO, format=f"%(asctime)s: {logging.BASIC_FORMAT}")
-    logging.getLogger("botocore").setLevel(logging.INFO)
-
-    # ElasticsearchService('eu-central-1')
-    e = ElasticsearchService("eu-central-1")
-
-    esi = ElasticsearchInstance(
-        endpoint='vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com',
-        arn='arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26',
-        domain_name='es-d8a24850a0ae4b82af26',
-    )
-
-    # e.find_instance_by_arn(esi.arn)
-    # exit()
-    try:
-        es = boto3.client('es')
-        res = es.delete_elasticsearch_domain(DomainName=esi.domain_name)
-        print(res)
-    except ClientError as e:
-        error_code = e.__class__.__name__
-        print(error_code)
-
-    e = ElasticsearchService("eu-central-1")
-    # print(
-    #     e.create_instance(
-    #         node_type="t2.medium.elasticsearch",
-    #         total_nodes=2,
-    #         volume_size_gb=10,
-    #         total_availability_zones=2,
-    #         security_group_id="sg-c11a77ac",
-    #         subnet_group_ids=["subnet-b8f596f5", "subnet-004909d95a03c5fcd"],
-    #         tags={"CostCenter": "my"},
-    #     )
-    # )

legacy/aws_rds.py

@@ -1,401 +0,0 @@
-import dataclasses
-import datetime
-import functools
-import logging
-import time
-import typing
-import uuid
-
-import boto3
-
-# import mysql.connector.connection
-import psycopg2
-import pymysql
-from psycopg2 import sql
-import psycopg2.extensions
-
-logger = logging.getLogger(__name__)
-
-
-@dataclasses.dataclass
-class Credentials:
-    username: str
-    password: str
-
-    @classmethod
-    def new(cls):
-        username_format = "u{}"  # must start with a letter
-        return cls(
-            username=username_format.format(uuid.uuid4().hex),
-            password=uuid.uuid4().hex,
-        )  # make sure it starts with a letter
-
-
-@dataclasses.dataclass
-class DbConnection:
-    endpoint: str
-    port: int
-
-
-@dataclasses.dataclass
-class DbInstance(DbConnection):
-    cluster_id: str
-    arn: str
-    reader_endpoint: typing.Optional[str] = None
-    database_name: typing.Optional[str] = None
-    user: typing.Optional[Credentials] = None
-    master_user: typing.Optional[Credentials] = None
-
-
-def retry(timeout: int, backoff_seconds: int = 10, bubble_errors: typing.List[typing.Type[Exception]] = None):
-    """
-    Tries calling a function {timeout} seconds until it succeeds or gives up and throw an error
-    :param timeout: Timeout in seconds
-    :param backoff_seconds: Wait time between retries
-    :param bubble_errors: List of exceptions to bubble up
-    :return: Wrapped function
-    """
-    bubble_errors = tuple(bubble_errors or [])
-
-    def wrapped(func: typing.Callable):
-        @functools.wraps(func)
-        def func_retrier(*args, **kwargs):
-            waited = 0
-            started_at = datetime.datetime.now()
-            while True:
-                try:
-                    logger.debug(f"Calling {func}")
-                    result = func(*args, **kwargs)
-                    logger.debug(f"{func} completed in {(datetime.datetime.now() - started_at)}")
-                    return result
-                except KeyboardInterrupt:
-                    raise
-                except tuple(bubble_errors):
-                    raise
-                except Exception as e:
-                    doze = min(timeout - waited, backoff_seconds)
-                    times = (timeout - waited) // backoff_seconds
-                    logger.debug(f"{func} failed, will try {times} times in {doze} sec", exc_info=True)
-                    time.sleep(doze)
-                    waited += backoff_seconds
-                    if waited >= timeout:
-                        raise TimeoutError(f"Couldn't get a successful result from {func} in {timeout} seconds") from e
-
-        return func_retrier
-
-    return wrapped
-
-
-def paginate_by_marker(
-    func: callable,
-    list_field: str,
-) -> typing.Iterable:
-    """
-    Simplifies paginating over AWS results with a simpler interface than Paginators.
-    :param func: A function that accepts a parameter named `Marker`
-    :param list_field: Result field to iterate on
-    """
-    marker = ""
-    while True:
-        result = func(Marker=marker)
-
-        items = result.get(list_field, [])
-        yield from items
-
-        if "Marker" not in result:
-            break
-
-        marker = result["Marker"]
-
-
-class PostgresqlService:
-    db_type = "Postgresql"
-    db_engine = "aurora-postgresql"
-    db_engine_version = "10.11"
-    default_master_database_name = "postgres"
-    cluster_id_format = "pg-{}"  # must start with a letter
-    tenant_database_name_format = "db{}"  # must start with a letter
-
-    def __init__(self, region: str):
-        self.rds = boto3.client("rds", region_name=region)
-
-    def create_db(
-        self,
-        total_instances: int,
-        db_subnet_group_name: str,
-        node_type: str,
-        security_group_id: str,
-        engine_version: str = None,
-        backup_retention_days: int = 30,
-        public: bool = False,
-        tags: dict = None,
-    ):
-        if not engine_version:
-            engine_version = self.db_engine_version
-
-        cluster_id = self.cluster_id_format.format(uuid.uuid4().hex)
-
-        master_creds = Credentials.new()
-
-        logger.info(
-            f"Creating {self.db_type} cluster",
-            extra=dict(
-                cluster_id=cluster_id,
-                engine_version=engine_version,
-                db_subnet_group_name=db_subnet_group_name,
-                security_group_id=security_group_id,
-            ),
-        )
-
-        db_result = self.rds.create_db_cluster(
-            DBClusterIdentifier=cluster_id,
-            Engine=self.db_engine,
-            EngineVersion=engine_version,
-            MasterUsername=master_creds.username,
-            MasterUserPassword=master_creds.password,
-            Tags=[{"Key": k, "Value": v} for k, v in (tags or {}).items()],
-            DBSubnetGroupName=db_subnet_group_name,
-            VpcSecurityGroupIds=[security_group_id],
-            BackupRetentionPeriod=backup_retention_days,
-        )["DBCluster"]
-
-        endpoint = db_result["Endpoint"]
-        reader_endpoint = db_result["ReaderEndpoint"]
-        port = db_result["Port"]
-        arn = db_result["DBClusterArn"]
-
-        for i in range(total_instances):
-            instance_id = f"{cluster_id}-Instance-{i}"
-            logger.info(
-                "Creating instances on the cluster",
-                extra=dict(
-                    cluster_id=cluster_id,
-                    instance_id=instance_id,
-                    node_type=node_type,
-                ),
-            )
-            instance_result = self.rds.create_db_instance(
-                DBInstanceIdentifier=instance_id,
-                DBClusterIdentifier=cluster_id,
-                DBInstanceClass=node_type,
-                Engine=self.db_engine,
-                PubliclyAccessible=public,
-                Tags=[{"Key": k, "Value": v} for k, v in (tags or {}).items()],
-            )
-
-        tenant_database_name = self.tenant_database_name_format.format(uuid.uuid4().hex)
-        db = DbInstance(
-            cluster_id=cluster_id,
-            arn=arn,
-            endpoint=endpoint,
-            reader_endpoint=reader_endpoint,
-            port=port,
-            database_name=tenant_database_name,
-            master_user=master_creds,
-        )
-        print(db)  # TODO: remove
-
-        logger.info("Waiting until DB instances are online. This will take a while (~5 min)")
-        logger.info("Connecting DB instance using master credentials", extra=dict(endpoint=db.endpoint))
-        self._check_connection(
-            connection=db,
-            credentials=master_creds,
-        )
-
-        tenant_creds = Credentials.new()
-        logger.info(
-            "Connection successful. Creating a new database and user",
-            extra=dict(
-                endpoint=endpoint,
-                username=tenant_creds.username,
-                database_name=tenant_database_name,
-            ),
-        )
-        self.create_tenant(
-            connection=db,
-            master_user=master_creds,
-            database_name=tenant_database_name,
-            tenant_user=tenant_creds,
-        )
-        db.user = tenant_creds
-
-        return db
-
-    @retry(timeout=10 * 60)
-    def _check_connection(self, connection: DbConnection, credentials: Credentials, database_name: str = "postgres"):
-        psycopg2.connect(
-            host=connection.endpoint,
-            port=connection.port,
-            dbname=database_name,
-            user=credentials.username,
-            password=credentials.password,
-            connect_timeout=15,
-        ).close()
-
-    def _find_instance(self, endpoint: str) -> typing.Optional[DbInstance]:
-        func = functools.partial(
-            self.rds.describe_db_clusters, Filters=[{"Name": "engine", "Values": [self.db_engine]}], MaxRecords=100
-        )
-        cluster = None
-        for it in paginate_by_marker(func, "DBClusters"):
-            if it["Endpoint"] == endpoint:
-                cluster = it
-                break
-        if cluster:
-            return DbInstance(
-                endpoint=cluster["Endpoint"],
-                port=cluster["Port"],
-                cluster_id=cluster["DBClusterIdentifier"],
-                arn=cluster["DBClusterArn"],
-                reader_endpoint=cluster["ReaderEndpoint"],
-            )
-        return None
-
-    def create_tenant(
-        self,
-        connection: DbConnection,
-        master_user: Credentials,
-        tenant_user: Credentials,
-        database_name: typing.Optional[str] = None,
-    ) -> DbInstance:
-        if not database_name:
-            database_name = self.tenant_database_name_format.format(uuid.uuid4().hex)
-        try:
-            con = psycopg2.connect(
-                host=connection.endpoint,
-                port=connection.port,
-                dbname=self.default_master_database_name,
-                user=master_user.username,
-                password=master_user.password,
-                connect_timeout=30,
-            )
-            con.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
-            cur = con.cursor()
-
-            cur.execute(
-                sql.SQL("CREATE DATABASE {}").format(
-                    sql.Identifier(database_name),
-                ),
-            )
-            cur.execute(
-                sql.SQL("CREATE USER {} WITH ENCRYPTED PASSWORD {}").format(
-                    sql.Identifier(tenant_user.username),
-                    sql.Placeholder(),
-                ),
-                [tenant_user.password],
-            )
-            cur.execute(
-                sql.SQL("GRANT ALL PRIVILEGES ON DATABASE {} TO {}").format(
-                    sql.Identifier(database_name),
-                    sql.Identifier(tenant_user.username),
-                )
-            )
-
-            # `with` block would create an implicit transaction
-            con.close()
-
-            db = self._find_instance(connection.endpoint)
-            db.database_name = database_name
-            db.user = tenant_user
-            db.master_user = master_user
-            return db
-        except psycopg2.Error:
-            logging.error(
-                f"Failed to create database and user",
-                exc_info=True,
-                extra=dict(endpoint=connection.endpoint, database_name=database_name),
-            )
-            raise
-
-
-class MysqlService(PostgresqlService):
-    db_type = "MySQL"
-    db_engine = "aurora-mysql"
-    db_engine_version = "5.7.12"
-    default_master_database_name = "mysql"
-    cluster_id_format = "mysql-{}"  # must start with a letter
-
-    @retry(timeout=10 * 60)
-    def _check_connection(self, connection: DbConnection, credentials: Credentials, database_name: str = "postgres"):
-        pymysql.connect(
-            host=connection.endpoint,
-            port=connection.port,
-            user=credentials.username,
-            password=credentials.password,
-            connect_timeout=15,
-        ).close()
-
-    def create_tenant(
-        self,
-        connection: DbConnection,
-        master_user: Credentials,
-        tenant_user: Credentials,
-        database_name: typing.Optional[str] = None,
-    ) -> DbInstance:
-        if not database_name:
-            database_name = self.tenant_database_name_format.format(uuid.uuid4().hex)
-
-        try:
-            with pymysql.connect(
-                host=connection.endpoint,
-                port=connection.port,
-                user=master_user.username,
-                password=master_user.password,
-                connect_timeout=15,
-            ) as con:
-                cur = con.cursor()
-                cur.execute(f"CREATE DATABASE {database_name}")
-                cur.execute(f"CREATE USER {tenant_user.username} IDENTIFIED BY '{tenant_user.password}'")
-                cur.execute(
-                    f"GRANT ALL PRIVILEGES ON {database_name}.* TO {tenant_user.username}@'%' IDENTIFIED BY '{tenant_user.password}'"
-                )
-
-                db = self._find_instance(endpoint=connection.endpoint)
-                db.database_name = database_name
-                db.master_user = master_user
-                db.user = tenant_user
-                return db
-        except pymysql.Error:
-            logger.error(
-                "Failed to create database and user",
-                exc_info=True,
-                extra=dict(endpoint=connection.endpoint, database_name=database_name),
-            )
-            raise
-
-
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.INFO, format=f"%(asctime)s: {logging.BASIC_FORMAT}")
-    logging.getLogger("botocore").setLevel(logging.INFO)
-
-    # m = MysqlService("eu-central-1")
-    # db = m.create_db(
-    #     total_instances=1,
-    #     db_subnet_group_name="mydbsubnet",
-    #     security_group_id="sg-c11a77ac",
-    #     node_type="db.t3.medium",
-    #     tags={"CostCenter": "hello"},
-    #     public=True,
-    # )
-    # print(db)
-
-    # p = PostgresqlService("eu-central-1")
-    # db = p.create_db(
-    #     total_instances=1,
-    #     db_subnet_group_name="mydbsubnet",
-    #     security_group_id="sg-c11a77ac",
-    #     node_type="db.t3.medium",
-    #     tags={"CostCenter": "hello"},
-    #     public=True,
-    # )
-    # p.create_tenant(
-    #     connection=DbConnection(
-    #         endpoint="pg0a085614d5eb4f34affd2a24b1fb18c4.cluster-c8v5rp0ouaey.eu-central-1.rds.amazonaws.com",
-    #         port=5432,
-    #     ),
-    #     master_user=Credentials(
-    #         username="u7d0f9621f5a5419288a4ef7ce9fd5fe", password="ca9473879b794dd3a0b0f0d2a8d4aebb"
-    #     ),
-    #     tenant_user=Credentials.new(),
-    #     database_name="mydb",
-    # )

legacy/aws_redis_acl.py

@@ -1,107 +0,0 @@
-import logging
-import typing
-
-import boto3
-import botocore.client
-
-
-def paginate_by_marker(
-    func: callable,
-    list_field: str,
-) -> typing.Iterable:
-    """
-    Simplifies paginating over AWS results with a simpler interface than Paginators.
-    :param func: A function that accepts a parameter named `Marker`
-    :param list_field: Result field to iterate on
-    """
-    marker = ""
-    while True:
-        result = func(Marker=marker)
-
-        items = result.get(list_field, [])
-        yield from items
-
-        if "Marker" not in result:
-            break
-
-        marker = result["Marker"]
-
-
-logger = logging.getLogger(__name__)
-
-
-class Rediser:
-    @classmethod
-    def new(cls):
-        return cls(boto3.client("elasticache", region_name='eu-central-1'))
-
-    def __init__(self, elasticache: botocore.client.BaseClient):
-        self.elasticache = elasticache
-
-    def _get_redis_acl(
-        self,
-        cluster_id: str = None,
-        status: str = "available",
-    ) -> typing.Optional[dict]:
-        """
-        Finds a Redis replication group by its ID
-
-        :param cluster_id: Replication group ID
-        :param status: Replication status. One of available/starting/modifying/deleting.
-        :return: RedisInstance or throws LookupError error
-        """
-        try:
-            redis = self.elasticache.describe_replication_groups(ReplicationGroupId=cluster_id)["ReplicationGroups"][0]
-            if redis["Status"] != status:
-                raise Exception(f"Found redis with {cluster_id=} but its status={redis['Status']}")
-        except self.elasticache.exceptions.ReplicationGroupNotFoundFault:
-            return None
-
-        return dict(
-            cluster_id=redis["ReplicationGroupId"],
-            host=redis["NodeGroups"][0]["PrimaryEndpoint"]["Address"],
-            port=redis["NodeGroups"][0]["PrimaryEndpoint"]["Port"],
-            arn=redis["ARN"],
-            user_group_ids=redis["UserGroupIds"],
-        )
-
-    def create_redis_acl(
-        self,
-        *,
-        cluster_id: str,
-        cluster_description: typing.Optional[str] = None,
-        node_type: str,
-        subnet_group_name: str,
-        security_group_id: str,
-    ):
-        redis = self._get_redis_acl(cluster_id=cluster_id)
-
-        if not redis:
-            res = self.elasticache.create_replication_group(
-                ReplicationGroupId=cluster_id,
-                ReplicationGroupDescription=cluster_description or cluster_id,
-                ReplicasPerNodeGroup=0,
-                Engine="redis",
-                EngineVersion="6.x",
-                CacheNodeType=node_type,
-                CacheSubnetGroupName=subnet_group_name,
-                TransitEncryptionEnabled=True,  # must be True to use Redis ACL
-                UserGroupIds=[
-                    user_group_id,
-                ],
-                SecurityGroupIds=[
-                    security_group_id,
-                ],
-                Tags=[{"Key": k, "Value": v} for k, v in tags.items()],
-                MultiAZEnabled=False,
-                AutomaticFailoverEnabled=False,
-            )
-
-
-def main():
-    r = Rediser.new()
-    res = r.create_redis_acl(cluster_id='hey', node_type='cache.t2.micro')
-
-
-if __name__ == '__main__':
-    main()

legacy/aws_ses.py

@@ -1,202 +0,0 @@
-import concurrent.futures
-import functools
-import logging
-import pprint
-import time
-import typing
-
-import boto3
-import botocore.client
-
-DnsRecord = typing.TypedDict(
-    "DnsRecord",
-    {
-        "purpose": str,
-        "type": str,
-        "name": str,
-        "value": str,
-        "priority": typing.Optional[str],
-    },
-)
-VerificationStatus = typing.Literal["Pending", "Success", "Failed", "TemporaryFailure", "NotStarted"]
-
-logger = logging.getLogger(__name__)
-
-
-def retry(tries: int = 5, delay: int = 1.5):
-    def wrapper(fn):
-        @functools.wraps(fn)
-        def wrapped(*args, **kwargs):
-            nonlocal tries
-            while tries:
-                try:
-                    return fn(*args, **kwargs)
-                except Exception:
-                    logger.exception(f'{fn} returned error, retrying {tries} more times')
-                time.sleep(delay)
-                tries -= 1
-
-        return wrapped
-
-    return wrapper
-
-
-class AwsSesService:
-    def __init__(self, client: typing.Optional[botocore.client.BaseClient] = None, timeout: int = 5):
-        self.ses = client or boto3.client("ses")
-        self.region = self.ses._client_config.region_name
-        self.timeout = timeout
-
-    # @retry()
-    def configure_domain_identity(
-        self, domain: str, mail_subdomain: typing.Optional[str] = None
-    ) -> typing.List[DnsRecord]:
-        """
-        Creates up mail identities for sending email through a domain.
-
-        DKIM-signed messages help receiving mail servers validate that a message was not forged or altered in transit.
-
-        :param domain: domain name to set up email identity
-        :return: DNS records that need to be set
-        """
-
-        res_id = self.ses.verify_domain_identity(Domain=domain)
-        id_token = res_id["VerificationToken"]
-
-        res_dkim = self.ses.verify_domain_dkim(Domain=domain)
-        dns_records = [
-            {
-                "purpose": "identity",
-                "type": "TXT",
-                "name": domain,
-                "value": id_token,
-            },
-            *[
-                {
-                    "purpose": "dkim",
-                    "type": "CNAME",
-                    "name": f"{dkim_token}._domainkey.{domain}",
-                    "value": f"{dkim_token}.dkim.amazonses.com",
-                }
-                for dkim_token in res_dkim["DkimTokens"]
-            ],
-        ]
-        if mail_subdomain:
-            more_dns = self.configure_from_domain(domain, mail_subdomain)
-            dns_records.extend(more_dns)
-        return dns_records
-
-    @retry()
-    def check_dkim_verification_status(self, domain: str) -> VerificationStatus:
-        """
-        Returns the DKIM verification status for an email identity.
-
-        :param domain: domain used to set up the mail identity
-        :return: one of {Pending, Success, Failed, TemporaryFailure, NotStarted}
-        """
-        res = self.ses.get_identity_dkim_attributes(Identities=[domain])
-        return res["DkimAttributes"][domain]["DkimVerificationStatus"]
-
-    @retry()
-    def check_id_verification_status(self, domain: str) -> VerificationStatus:
-        """
-        Returns the id verification status of an email identity.
-
-        :param domain: domain used to set up the mail identity
-        :return: one of {Pending, Success, Failed, TemporaryFailure, NotStarted}
-        """
-        res = self.ses.get_identity_verification_attributes(Identities=[domain])
-        return res["VerificationAttributes"][domain]["VerificationStatus"]
-
-    def configure_from_domain(self, domain: str, mail_subdomain: str) -> typing.List[DnsRecord]:
-        """
-        Allows using `mail_subdomain` as FROM address when sending emails.
-        Messages sent through Amazon SES will be marked as originating from your domain instead of a subdomain of amazon.com.
-
-        :param domain: mail identity
-        :param mail_subdomain: a subdomain
-        :return: DNS records that need to be set
-        """
-        _ = self.ses.set_identity_mail_from_domain(
-            Identity=domain,
-            MailFromDomain=mail_subdomain,
-            BehaviorOnMXFailure="UseDefaultValue",  # uses $region.amazonses.com if MX records not present on the mail subdomain
-        )
-        return [
-            {
-                "purpose": "mail_from_domain",
-                "type": "MX",
-                "name": mail_subdomain,
-                "value": f"feedback-smtp.{self.region}.amazonses.com",
-                "priority": "10",
-            },
-            {
-                "purpose": "mail_from_domain",
-                "type": "TXT",
-                "name": mail_subdomain,
-                "value": f'"v=spf1 include:amazonses.com ~all"',
-            },
-        ]
-
-    @retry()
-    def check_custom_from_domain_status(self, domain: str) -> VerificationStatus:
-        """
-        Returns the status of custom email FROM request verification.
-
-        :param domain: domain used to set up email identity
-        :return: one of {Pending, Success, Failed, TemporaryFailure}
-        """
-        res = self.ses.get_identity_mail_from_domain_attributes(Identities=[domain])
-        return res["MailFromDomainAttributes"][domain].get("MailFromDomainStatus", 'NotStarted')
-
-    def bulk_check_domain_verification(self, identity_domain: str, timeout: int = 5) -> dict:
-        tasks = {
-            "status_id": functools.partial(self.check_id_verification_status, identity_domain),
-            "status_dkim": functools.partial(self.check_dkim_verification_status, identity_domain),
-            "status_custom": functools.partial(self.check_custom_from_domain_status, identity_domain),
-        }
-        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
-            futures = [pool.submit(t) for _, t in tasks.items()]
-            results = dict(zip(tasks.keys(), [f.result(timeout=timeout) for f in futures]))
-
-        return {
-            **results,
-            "identity_domain": identity_domain,
-        }
-
-    @retry()
-    def bulk_configure_email_domains(
-        self,
-        identity_domain: str,
-        mail_subdomain: typing.Optional[str] = None,
-        timeout: int = 10,
-    ) -> dict:
-        tasks = {
-            "dns": functools.partial(self.configure_domain_identity, identity_domain, mail_subdomain),
-        }
-
-        # send all requests in parallel
-        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
-            futures = [pool.submit(t) for _, t in tasks.items()]
-            results = dict(zip(tasks.keys(), [f.result(timeout=timeout) for f in futures]))
-        return {
-            **results,
-            "identity_domain": identity_domain,
-            "mail_subdomain": mail_subdomain,
-        }
-
-
-if __name__ == "__main__":
-    ses = AwsSesService()
-    logging.basicConfig(level=logging.INFO)
-    # print(ses.bulk_configure_email_domains('abdus100.dev', 'mail.abdus100.dev'))
-    # print(ses.check_custom_from_domain_status('abdus101.dev'))
-
-    tasks = [
-        functools.partial(ses.bulk_check_domain_verification, d) for d in ['abdus.dev', 'abdus1.dev', 'abdus2.dev']
-    ] * 3
-
-    with concurrent.futures.ThreadPoolExecutor(max_workers=20) as pool:
-        futures = [pool.submit(t) for t in tasks]
-        results = [f.result(timeout=5) for f in futures]
-    pprint.pp(results)

legacy/aws_ses_test.py

@@ -1,133 +0,0 @@
-from unittest import mock
-
-import pytest
-
-from aws_ses import AwsSesService
-
-# fmt: off
-_response_verify_domain_dkim = {'DkimTokens': ['token1', 'token2', 'token3'], 'ResponseMetadata': {'RequestId': '6e4da72e-3063-41da-9434-dae7ef19c78e', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:27:40 GMT', 'content-type': 'text/xml', 'content-length': '469', 'connection': 'keep-alive', 'x-amzn-requestid': '6e4da72e-3063-41da-9434-dae7ef19c78e'}, 'RetryAttempts': 0}}
-_response_verify_domain_identity = {'VerificationToken': 'token', 'ResponseMetadata': {'RequestId': 'b733069c-115f-4457-b6cb-14e44f29d17f', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:27:12 GMT', 'content-type': 'text/xml', 'content-length': '370', 'connection': 'keep-alive', 'x-amzn-requestid': 'b733069c-115f-4457-b6cb-14e44f29d17f'}, 'RetryAttempts': 0}}
-_response_get_identity_dkim_attributes = {'DkimAttributes': {'example.com': {'DkimEnabled': True, 'DkimVerificationStatus': 'Success', 'DkimTokens': ['bgobmdqs2vbxr6ygwbmg753knfu6tos7', 'm6o6w6twpmwzwe3igpxxo7qsvwp3hlgj', 'zqfz4ekdelp7m2u2phsyad3h5dsfle4v']}}, 'ResponseMetadata': {'RequestId': 'ba466be1-79bc-499b-9766-ca1acd7d1c08', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 16:49:53 GMT', 'content-type': 'text/xml', 'content-length': '833', 'connection': 'keep-alive', 'x-amzn-requestid': 'ba466be1-79bc-499b-9766-ca1acd7d1c08'}, 'RetryAttempts': 0}}
-_response_get_identity_dkim_attributes_pending = {'DkimAttributes': {'example.com': {'DkimEnabled': True, 'DkimVerificationStatus': 'Pending', 'DkimTokens': ['bgobmdqs2vbxr6ygwbmg753knfu6tos7', 'm6o6w6twpmwzwe3igpxxo7qsvwp3hlgj', 'zqfz4ekdelp7m2u2phsyad3h5dsfle4v']}}, 'ResponseMetadata': {'RequestId': 'b24732d9-01e1-4266-8b34-550a9ecf634d', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:30:03 GMT', 'content-type': 'text/xml', 'content-length': '833', 'connection': 'keep-alive', 'x-amzn-requestid': 'b24732d9-01e1-4266-8b34-550a9ecf634d'}, 'RetryAttempts': 0}}
-_response_get_identity_verification_attributes = {'VerificationAttributes': {'example.com': {'VerificationStatus': 'Success', 'VerificationToken': 'XzMvLQqOPdC0iJ02YTdjqWBJr7FTqhGtAsEcfpUwfoA='}}, 'ResponseMetadata': {'RequestId': '022c3e6e-a6e8-426f-bf4d-149f71ca7913', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 16:56:19 GMT', 'content-type': 'text/xml', 'content-length': '637', 'connection': 'keep-alive', 'x-amzn-requestid': '022c3e6e-a6e8-426f-bf4d-149f71ca7913'}, 'RetryAttempts': 0}}
-_response_get_identity_verification_attributes_pending = {'VerificationAttributes': {'example.com': {'VerificationStatus': 'Pending', 'VerificationToken': 'XzMvLQqOPdC0iJ02YTdjqWBJr7FTqhGtAsEcfpUwfoA='}}, 'ResponseMetadata': {'RequestId': '022c3e6e-a6e8-426f-bf4d-149f71ca7913', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 16:56:19 GMT', 'content-type': 'text/xml', 'content-length': '637', 'connection': 'keep-alive', 'x-amzn-requestid': '022c3e6e-a6e8-426f-bf4d-149f71ca7913'}, 'RetryAttempts': 0}}
-_response_set_identity_mail_from_domain = {'ResponseMetadata': {'RequestId': 'c04acd33-40be-46cd-9960-c9e51397dc48', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:03:50 GMT', 'content-type': 'text/xml', 'content-length': '266', 'connection': 'keep-alive', 'x-amzn-requestid': 'c04acd33-40be-46cd-9960-c9e51397dc48'}, 'RetryAttempts': 0}}
-_response_get_identity_mail_from_domain_attributes = {'MailFromDomainAttributes': {'example.com': {'MailFromDomain': 'email.abdus.dev', 'MailFromDomainStatus': 'Success', 'BehaviorOnMXFailure': 'UseDefaultValue'}}, 'ResponseMetadata': {'RequestId': '63049966-c062-47f4-8238-145bd533d880', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:37:06 GMT', 'content-type': 'text/xml', 'content-length': '687', 'connection': 'keep-alive', 'x-amzn-requestid': '63049966-c062-47f4-8238-145bd533d880'}, 'RetryAttempts': 0}}
-_response_get_identity_mail_from_domain_attributes_pending = {'MailFromDomainAttributes': {'example.com': {'MailFromDomain': 'email.abdus.dev', 'MailFromDomainStatus': 'Pending', 'BehaviorOnMXFailure': 'UseDefaultValue'}}, 'ResponseMetadata': {'RequestId': '63049966-c062-47f4-8238-145bd533d880', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Mon, 03 Jan 2022 17:37:06 GMT', 'content-type': 'text/xml', 'content-length': '687', 'connection': 'keep-alive', 'x-amzn-requestid': '63049966-c062-47f4-8238-145bd533d880'}, 'RetryAttempts': 0}}
-# fmt: on
-
-
-@pytest.fixture()
-def mock_ses() -> mock.MagicMock:
-    with mock.patch("boto3.client") as m:
-        m_ses = m.return_value
-        yield m_ses
-
-
-def test_init_domain_verification(mock_ses):
-    m_verify_id = mock_ses.verify_domain_identity
-    m_verify_dkim = mock_ses.verify_domain_dkim
-    m_verify_id.return_value = _response_verify_domain_identity
-    m_verify_dkim.return_value = _response_verify_domain_dkim
-    domain = "example.com"
-
-    dns = AwsSesService(mock_ses).configure_domain_identity(domain)
-
-    m_verify_id.assert_called_once_with(Domain=domain)
-    m_verify_dkim.assert_called_once_with(Domain=domain)
-    assert dns == [
-        {
-            "type": "TXT",
-            "name": domain,
-            "value": "token",
-        },
-        {
-            "type": "CNAME",
-            "name": f"token1._domainkey.{domain}",
-            "value": f"token1.dkim.amazonses.com",
-        },
-        {
-            "type": "CNAME",
-            "name": f"token2._domainkey.{domain}",
-            "value": f"token2.dkim.amazonses.com",
-        },
-        {
-            "type": "CNAME",
-            "name": f"token3._domainkey.{domain}",
-            "value": f"token3.dkim.amazonses.com",
-        },
-    ]
-
-
-def test_get_dkim_status(mock_ses):
-    m = mock_ses.get_identity_dkim_attributes
-    m.return_value = _response_get_identity_dkim_attributes_pending
-    domain = "example.com"
-
-    status = AwsSesService(mock_ses).check_dkim_verification_status(domain)
-
-    assert status == "Pending"
-    m.assert_called_once_with(Identities=[domain])
-
-    # ===
-
-    m.reset_mock()
-    m.return_value = _response_get_identity_dkim_attributes
-
-    status = AwsSesService(mock_ses).check_dkim_verification_status(domain)
-    assert status == "Success"
-
-
-def test_get_id_status(mock_ses):
-    m = mock_ses.get_identity_verification_attributes
-    m.return_value = _response_get_identity_verification_attributes
-
-    domain = "example.com"
-    status = AwsSesService(mock_ses).check_id_verification_status(domain)
-
-    assert status == "Success"
-    m.assert_called_once_with(Identities=[domain])
-
-
-def test_set_custom_from_domain(mock_ses):
-    mock_ses._client_config.region_name = "fake-region"
-    m = mock_ses.set_identity_mail_from_domain
-    m.return_value = _response_set_identity_mail_from_domain
-
-    dns = AwsSesService(mock_ses).configure_from_domain("example.com", "mail.example.com")
-
-    assert dns == [
-        {
-            "type": "MX",
-            "name": "mail.example.com",
-            "value": f"feedback-smtp.fake-region.amazonses.com",
-            "priority": "10",
-        },
-        {"type": "TXT", "name": "mail.example.com", "value": f'"v=spf1 include:amazonses.com ~all"'},
-    ]
-    m.assert_called_once_with(
-        Identity="example.com",
-        MailFromDomain="mail.example.com",
-        BehaviorOnMXFailure="UseDefaultValue",
-    )
-
-
-def test_get_mail_from_status(mock_ses):
-    m = mock_ses.get_identity_mail_from_domain_attributes
-    m.return_value = _response_get_identity_mail_from_domain_attributes_pending
-
-    domain = "example.com"
-    status = AwsSesService(mock_ses).check_custom_from_domain_status(domain)
-
-    assert status == "Pending"
-    m.assert_called_once_with(Identities=[domain])
-
-    # ===
-
-    m.reset_mock()
-    m.return_value = _response_get_identity_mail_from_domain_attributes
-
-    status = AwsSesService(mock_ses).check_custom_from_domain_status(domain)
-
-    assert status == "Success"
-    m.assert_called_once_with(Identities=[domain])

legacy/hetzner_kube.py

@@ -1,237 +0,0 @@
-#!/usr/bin/env python3.9
-import base64
-import json
-import logging
-import pprint
-import subprocess
-import time
-import webbrowser
-from pathlib import Path
-
-import httpx
-import typer
-
-hetzner_token = 'zdlkmAF9EswvOIEGvWgutULYTmUpa3c44YcKZOB0jymQfgvra7kRVJxuLiFTouM9'
-client = httpx.Client(base_url='https://api.hetzner.cloud/v1', headers={'Authorization': f'Bearer {hetzner_token}'})
-
-
-datacenters = [
-    {'id': 2, 'name': 'nbg1-dc3', 'description': 'Nuremberg 1 DC 3'},
-    {'id': 3, 'name': 'hel1-dc2', 'description': 'Helsinki 1 DC 2'},
-    {'id': 4, 'name': 'fsn1-dc14', 'description': 'Falkenstein 1 DC14'},
-    {'id': 5, 'name': 'ash-dc1', 'description': 'Ashburn DC1'},
-]
-
-server_types = {
-    'cx11': {'cores': 1, 'memory': 2.0, 'disk': 20},
-    'cx21': {'cores': 2, 'memory': 4.0, 'disk': 40},
-    'cx31': {'cores': 2, 'memory': 8.0, 'disk': 80},
-    'cx41': {'cores': 4, 'memory': 16.0, 'disk': 160},
-    'cx51': {'cores': 8, 'memory': 32.0, 'disk': 240},
-    'ccx11': {'cores': 2, 'memory': 8.0, 'disk': 80},
-    'ccx21': {'cores': 4, 'memory': 16.0, 'disk': 160},
-    'ccx31': {'cores': 8, 'memory': 32.0, 'disk': 240},
-    'ccx41': {'cores': 16, 'memory': 64.0, 'disk': 360},
-    'ccx51': {'cores': 32, 'memory': 128.0, 'disk': 600},
-    'cpx11': {'cores': 2, 'memory': 2.0, 'disk': 40},
-    'cpx21': {'cores': 3, 'memory': 4.0, 'disk': 80},
-    'cpx31': {'cores': 4, 'memory': 8.0, 'disk': 160},
-    'cpx41': {'cores': 8, 'memory': 16.0, 'disk': 240},
-    'cpx51': {'cores': 16, 'memory': 32.0, 'disk': 360},
-    'ccx12': {'cores': 2, 'memory': 8.0, 'disk': 80},
-    'ccx22': {'cores': 4, 'memory': 16.0, 'disk': 160},
-    'ccx32': {'cores': 8, 'memory': 32.0, 'disk': 240},
-    'ccx42': {'cores': 16, 'memory': 64.0, 'disk': 360},
-    'ccx52': {'cores': 32, 'memory': 128.0, 'disk': 600},
-    'ccx62': {'cores': 48, 'memory': 192.0, 'disk': 960},
-}
-
-
-def create_server(name: str, datacenter: str):
-    payload = {
-        "datacenter": datacenter,
-        "image": "ubuntu-20.04",
-        "name": name,
-        "server_type": "cpx21",
-        "ssh_keys": ["entropy"],
-    }
-
-    res = client.post('/servers', json=payload)
-    res.raise_for_status()
-
-    data = res.json()
-    server = data['server']
-    pprint.pprint(server)
-
-    return {
-        'id': server['id'],
-        'name': server['name'],
-        'ip': server['public_net']['ipv4']['ip'],
-    }
-
-
-def find_server(name: str) -> dict:
-    res = client.get('/servers')
-    res.raise_for_status()
-
-    data = res.json()
-
-    server = [it for it in data['servers'] if it['name'] == name][0]
-    return {
-        'id': server['id'],
-        'name': server['name'],
-        'ip': server['public_net']['ipv4']['ip'],
-    }
-
-
-def delete_server(name: str) -> None:
-    server = find_server(name)
-    logging.info(f'deleting server={name} with id={server["id"]}')
-
-    res = client.delete(f'/servers/{server["id"]}')
-    res.raise_for_status()
-
-    logging.info(f'deleted server={name}')
-
-
-def copy_to_clipboard(text: str) -> None:
-    subprocess.Popen(['pbcopy'], stdin=subprocess.PIPE).communicate(input=text.encode())
-
-
-class Server:
-    ssh_options = ['-o', 'StrictHostKeyChecking=no']
-
-    def __init__(self, ip_address: str, name: str = None):
-        self.ip = ip_address
-        self.ssh_host = f'root@{self.ip}'
-        self.name = name
-
-    @classmethod
-    def from_hetzner(cls, server: dict):
-        return cls(server['ip'], name=server['name'])
-
-    def run_command(self, cmd: str, return_output: bool = False) -> str:
-
-        logging.debug(f'running command={cmd} on {self.ssh_host}')
-        extra_args = {}
-        if return_output:
-            extra_args['stdout'] = subprocess.PIPE
-        p = subprocess.run(['ssh', *self.ssh_options, self.ssh_host, '--', cmd], text=True, **extra_args)
-        p.check_returncode()
-        return p.stdout
-
-    def install_package(self, packages: str) -> str:
-        return self.run_command(f'apt-get install -y {packages}')
-
-    def install_shell(self):
-        self.install_package('fish')
-        self.run_command('chsh -s /usr/bin/fish')
-
-    def install_microk8s(self):
-        self.install_package('snapd')
-        self.run_command('snap install microk8s --classic')
-        self.run_command('ufw allow in on cni0 && ufw allow out on cni0 && ufw default allow routed')
-        self.run_command('microk8s enable dashboard dns rbac ingress storage prometheus registry')
-
-    def get_microk8s_token(self):
-        raw_json = self.run_command('microk8s kubectl -n kube-system get secret -o json', return_output=True)
-        secrets = json.loads(raw_json)['items']
-
-        default_token_secret = [it for it in secrets if it['metadata']['name'].startswith('default-token')][0]
-        token_base64 = default_token_secret['data']['token']
-
-        token = base64.decodebytes(token_base64.encode()).decode()
-        return token
-
-    def show_microk8s_access_info(self):
-        logging.info('getting default service account token')
-        token = self.get_microk8s_token()
-        print(token)
-        logging.info('copying token to clipboard')
-        copy_to_clipboard(token)
-
-        logging.info('getting dashboard ip')
-        proxy = self.get_microk8s_dashboard_proxy()
-        print(proxy['ssh'])
-        copy_to_clipboard(proxy['ssh'])
-
-        logging.info('opening url in the browser')
-        print(proxy['url'])
-        webbrowser.open_new_tab(proxy['url'])
-
-    def get_microk8s_dashboard_proxy(self):
-        cluster_ip = self.run_command(
-            'microk8s kubectl get -n kube-system -o json service/kubernetes-dashboard -o jsonpath="{.spec.clusterIP}"',
-            return_output=True,
-        ).strip()
-        port, local_port = 443, 18433
-
-        return {
-            'ssh': f'ssh -vNL {local_port}:{cluster_ip}:{port} {self.ssh_host}',
-            'url': f'https://localhost:{local_port}',
-        }
-
-    def write_kubeconfig(self, filename: str):
-        logging.info(f'writing kubeconfig to {filename}')
-        kubeconfig = self.run_command('microk8s config', return_output=True)
-        file_path = Path(filename).expanduser()
-        file_path.parent.mkdir(parents=True, exist_ok=True)
-        file_path.write_text(kubeconfig)
-
-    def install_micro_editor(self):
-        logging.info('installing micro editor')
-        self.run_command('curl https://getmic.ro | bash && mv ./micro /usr/bin/')
-
-    def setup(self):
-        self.run_command('apt-get update')
-
-        self.install_shell()
-        self.install_micro_editor()
-        self.install_microk8s()
-
-        logging.info('waiting until pods are online')
-        time.sleep(10)
-
-        self.show_microk8s_access_info()
-        self.write_kubeconfig(f'~/Desktop/kubeconfig-{self.name}.yml')
-
-
-cli = typer.Typer(invoke_without_command=False)
-
-
-@cli.command()
-def create(
-    name: str,
-    datacenter: str = typer.Option('nbg1-dc3'),
-):
-    create_server(name=name, datacenter=datacenter)
-
-
-@cli.command()
-def destroy(
-    name: str,
-):
-    delete_server(name=name)
-
-
-@cli.command()
-def setup(
-    name: str,
-):
-    server = find_server(name)
-    server = Server.from_hetzner(server)
-    server.setup()
-
-
-@cli.command()
-def show_k8s(
-    name: str,
-):
-    server = Server.from_hetzner(find_server(name))
-    server.show_microk8s_access_info()
-    server.write_kubeconfig(f'~/Desktop/kubeconfig-{server.name}.yml')
-
-
-if __name__ == '__main__':
-    logging.basicConfig(level=logging.INFO)
-    cli()

macos_wallpaper.py

@@ -1,72 +0,0 @@
-import contextlib
-import dataclasses
-import plistlib
-import sqlite3
-import subprocess
-from pathlib import Path
-from typing import Optional
-
-DB_PATH = Path("~/Library/Application Support/Dock/desktoppicture.db").expanduser()
-
-
-@dataclasses.dataclass
-class SpaceInfo:
-    uuid: str
-    image_path: Optional[Path]
-    data_id: Optional[Path]
-
-
-@contextlib.contextmanager
-def get_connection() -> sqlite3.Connection:
-    con = sqlite3.connect(str(DB_PATH))
-    con.row_factory = sqlite3.Row
-    yield con
-    con.close()
-
-
-def get_current_wallpapers() -> list[SpaceInfo]:
-    sql = """
-    select distinct s.space_uuid, pr.key, d.rowid, d.value from preferences pr
-    left join payload d on d.rowid = pr.data_id
-    left join pictures p on p.rowid = pr.picture_id
-    inner join spaces s on s.rowid = p.space_id
-    where key = 1
-    
-    order by space_uuid, key
-    """
-    spaces = list_spaces()
-
-    with get_connection() as con:
-        cur = con.execute(sql)
-
-        space_props = {}
-        for i, row in enumerate(cur):
-            space_uuid = row["space_uuid"]
-            data_id = row["rowid"]
-            image_path = row["value"]
-
-            if image_path:
-                image_path = Path(image_path).expanduser()
-            space_props[space_uuid] = SpaceInfo(uuid=space_uuid, image_path=image_path, data_id=data_id)
-    return [space_props.get(id, SpaceInfo(uuid="", image_path=None, data_id=None)) for id in spaces]
-
-
-def read_plist(plist_path):
-    with plist_path.open("rb") as f:
-        return plistlib.load(f)
-
-
-def list_spaces():
-    spaces_plist_path = Path("~/Library/Preferences/com.apple.spaces.plist").expanduser()
-    data = read_plist(spaces_plist_path)
-    spaces = data["SpacesDisplayConfiguration"]["Management Data"]["Monitors"][0]["Spaces"]
-    return [s["uuid"] for s in spaces]
-
-
-def set_wallpaper(image_path: Path):
-    script = f'tell application "Finder" to set desktop picture to "{image_path.expanduser().resolve()}" as POSIX file'
-    subprocess.run(["osascript", "-e", script], check=True)
-
-
-if __name__ == "__main__":
-    pass

prompter/prompter.html

@@ -1,523 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="UTF-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <title>Prompter</title>
-  <!-- Persist plugin must load before Alpine core -->
-  <script defer src="https://cdn.jsdelivr.net/npm/@alpinejs/persist@3.x.x/dist/cdn.min.js"></script>
-  <script defer src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"></script>
-  <style>
-    *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
-
-    body {
-      display: flex;
-      height: 100vh;
-      overflow: hidden;
-      font-family: system-ui, -apple-system, sans-serif;
-      background: #0f0f0f;
-      color: #e0e0e0;
-    }
-
-    /* ── Image panel (left 50%) ──────────────────────────────────────────── */
-    .image-panel {
-      width: 50%;
-      height: 100vh;
-      display: flex;
-      flex-direction: column;
-      align-items: center;
-      justify-content: center;
-      background: #111;
-      padding: 20px;
-      gap: 10px;
-      border-right: 1px solid #1e1e1e;
-    }
-
-    .image-panel img {
-      max-width: 100%;
-      max-height: calc(100vh - 72px);
-      object-fit: contain;
-    }
-
-    .image-meta {
-      display: flex;
-      flex-direction: column;
-      align-items: center;
-      gap: 4px;
-    }
-
-    .image-filename {
-      font-size: 12px;
-      color: #666;
-      word-break: break-all;
-      text-align: center;
-      max-width: 100%;
-    }
-
-    .image-counter {
-      display: flex;
-      align-items: center;
-      gap: 4px;
-      font-size: 12px;
-      color: #555;
-    }
-
-    .image-counter input[type="number"] {
-      background: transparent;
-      border: none;
-      border-bottom: 1px solid #444;
-      color: #999;
-      font-family: inherit;
-      font-size: 12px;
-      outline: none;
-      padding: 0 2px;
-      text-align: right;
-      width: 3.5ch;
-      -moz-appearance: textfield;
-    }
-
-    .image-counter input[type="number"]:focus {
-      border-bottom-color: #4a6cf7;
-      color: #e0e0e0;
-    }
-
-    .image-counter input::-webkit-outer-spin-button,
-    .image-counter input::-webkit-inner-spin-button { -webkit-appearance: none; }
-
-    /* ── Options panel (right 50%) ───────────────────────────────────────── */
-    .options-panel {
-      width: 50%;
-      height: 100vh;
-      display: flex;
-      flex-direction: column;
-      overflow: hidden;
-    }
-
-    .panel-top,
-    .panel-bottom {
-      flex: 1 1 0;
-      min-height: 0;
-      overflow-y: auto;
-      padding: 24px;
-      display: flex;
-      flex-direction: column;
-      gap: 8px;
-    }
-
-    .panel-bottom {
-      border-top: 1px solid #1e1e1e;
-    }
-
-    .section-title {
-      font-size: 11px;
-      color: #555;
-      text-transform: uppercase;
-      letter-spacing: 0.08em;
-      margin-bottom: 4px;
-    }
-
-    .option {
-      border: 1px solid #2a2a2a;
-      border-radius: 8px;
-      padding: 12px 14px;
-      cursor: pointer;
-      transition: border-color 0.1s;
-    }
-
-    .option:hover { border-color: #3a3a3a; }
-    .option.active { border-color: #4a6cf7; background: rgba(74, 108, 247, 0.06); }
-
-    .option-meta {
-      display: flex;
-      align-items: center;
-      gap: 8px;
-      margin-bottom: 10px;
-      font-size: 12px;
-      color: #666;
-    }
-
-    .saved-badge { color: #4caf50; margin-left: auto; font-size: 11px; }
-
-    .kbd {
-      display: inline-block;
-      background: #1e1e1e;
-      border: 1px solid #333;
-      border-radius: 3px;
-      padding: 1px 6px;
-      font-size: 11px;
-      font-family: monospace;
-      color: #888;
-      white-space: nowrap;
-    }
-
-    textarea {
-      width: 100%;
-      background: #1a1a1a;
-      border: 1px solid #333;
-      border-radius: 6px;
-      color: #e0e0e0;
-      font-size: 14px;
-      line-height: 1.5;
-      padding: 10px;
-      resize: vertical;
-      min-height: 90px;
-      font-family: inherit;
-      outline: none;
-      transition: border-color 0.1s;
-    }
-
-    textarea:focus { border-color: #4a6cf7; }
-
-    .btn-save {
-      margin-top: 10px;
-      padding: 8px 18px;
-      background: #4a6cf7;
-      color: #fff;
-      border: none;
-      border-radius: 6px;
-      font-size: 14px;
-      cursor: pointer;
-      transition: background 0.1s;
-    }
-
-    .btn-save:hover { background: #3b5de8; }
-
-    .history-section { margin-top: 8px; }
-    .history-item { margin-top: 8px; }
-
-    .history-text {
-      font-size: 13px;
-      color: #bbb;
-      white-space: pre-wrap;
-      word-break: break-word;
-      line-height: 1.4;
-    }
-
-    .blocks-grid {
-      display: grid;
-      grid-template-columns: 1fr 1fr;
-      gap: 6px;
-      margin-top: 4px;
-    }
-
-    .block-pill {
-      background: #1e1e1e;
-      border: 1px solid #333;
-      border-radius: 8px;
-      color: #ccc;
-      cursor: pointer;
-      font-family: inherit;
-      font-size: 12px;
-      line-height: 1.4;
-      padding: 6px 10px;
-      text-align: left;
-      transition: border-color 0.1s, background 0.1s;
-      white-space: pre-wrap;
-      word-break: break-word;
-    }
-
-    .block-pill:hover {
-      background: #2a2a2a;
-      border-color: #4a6cf7;
-      color: #e0e0e0;
-    }
-  </style>
-</head>
-<body x-data="prompter" @keydown.window="handleKey($event)">
-
-  <!-- ── Left: image ───────────────────────────────────────────────────── -->
-  <div class="image-panel">
-    <img :src="`/image/${currentIndex}`" :alt="currentImage" />
-    <div class="image-meta">
-      <div class="image-filename" x-text="currentImage.split('/').pop()"></div>
-      <div class="image-counter">
-        <input
-          type="number"
-          min="1"
-          :max="images.length"
-          :value="currentIndex + 1"
-          @change="jumpTo($event.target.valueAsNumber - 1)"
-          @keydown.stop
-          @click.stop
-        />
-        <span>/ <span x-text="images.length"></span></span>
-      </div>
-    </div>
-  </div>
-
-  <!-- ── Right: options ────────────────────────────────────────────────── -->
-  <div class="options-panel">
-
-    <!-- ── Top half: prompt + history ──────────────────────────────────── -->
-    <div class="panel-top">
-      <div class="section-title">Prompt</div>
-
-      <!-- Custom prompt -->
-      <div class="option" :class="{ active: selectedOption === 'custom' }" @click="selectCustom()">
-        <div class="option-meta">
-          <span class="kbd">alt+1</span>
-          <span>Custom prompt</span>
-          <span class="saved-badge" x-show="selections[currentImage]">&#10003;&nbsp;saved</span>
-        </div>
-        <textarea
-          x-ref="textarea"
-          x-model="customPrompt"
-          @focus="selectedOption = 'custom'"
-          placeholder="Enter a prompt&#8230;"
-          rows="5"
-        ></textarea>
-        <button class="btn-save" @click.stop="saveAndNext()">Save &amp; Next</button>
-      </div>
-
-      <!-- History -->
-      <div class="history-section" x-show="promptHistory.length > 0">
-        <div class="section-title">Previously used</div>
-        <template x-for="(prompt, i) in promptHistory" :key="prompt">
-          <div
-            class="option history-item"
-            :class="{ active: selectedOption === i }"
-            @click="selectHistory(i)"
-          >
-            <div class="option-meta">
-              <span class="kbd" x-text="`alt+${i + 2}`"></span>
-            </div>
-            <div class="history-text" x-text="prompt"></div>
-          </div>
-        </template>
-      </div>
-    </div>
-
-    <!-- ── Bottom half: building blocks ────────────────────────────────── -->
-    <div class="panel-bottom" x-show="buildingBlocks.length > 0">
-      <div class="section-title">Building blocks</div>
-      <div class="blocks-grid">
-        <template x-for="block in buildingBlocks" :key="block">
-          <button class="block-pill" @click="appendBlock(block)" x-text="block"></button>
-        </template>
-      </div>
-    </div>
-
-  </div>
-
-  <script>
-    /**
-     * @typedef {{ images: string[], canned_prompts: string[], building_blocks: string[], existing_prompts: Record<string, string> }} ApiImages
-     */
-
-    document.addEventListener('alpine:init', () => {
-      Alpine.data('prompter', () => ({
-
-        /** @type {string[]} Ordered list of absolute image paths from the server */
-        images: [],
-
-        /** @type {number} Zero-based index of the currently displayed image */
-        currentIndex: 0,
-
-        /** @type {Record<string, string>} Absolute image path → saved prompt text */
-        selections: {},
-
-        /** @type {string} Current value of the custom-prompt textarea */
-        customPrompt: '',
-
-        /** @type {'custom' | number | null} Which option row is highlighted */
-        selectedOption: null,
-
-        /** @type {string[]} Reusable text fragments loaded from the server */
-        buildingBlocks: [],
-
-        /**
-         * Prompt history persisted to localStorage via Alpine Persist.
-         * Ordered most-recent-first; no duplicates.
-         * @type {string[]}
-         */
-        promptHistory: Alpine.$persist([]).as('prompter_history'),
-
-        /**
-         * Absolute path of the image currently on screen.
-         * @returns {string}
-         */
-        get currentImage() {
-          return this.images[this.currentIndex] ?? '';
-        },
-
-        /**
-         * Fetch the image list from the server, merge canned prompts into
-         * history, and restore any previously saved prompt for the first image.
-         */
-        async init() {
-          /** @type {ApiImages} */
-          const data = await fetch('/api/images').then(r => r.json());
-
-          this.images = data.images;
-          this.selections = { ...data.existing_prompts };
-          this.buildingBlocks = data.building_blocks ?? [];
-
-          // Append canned prompts not already present in the persisted history.
-          const inHistory = new Set(this.promptHistory);
-          for (const p of data.canned_prompts) {
-            if (!inHistory.has(p)) {
-              this.promptHistory.push(p);
-            }
-          }
-
-          // Restore a previously saved prompt for the first image (if any).
-          const saved = this.selections[this.currentImage];
-          if (saved) {
-            this.customPrompt = saved;
-            this.selectedOption = 'custom';
-          }
-        },
-
-        /**
-         * Append a building block to the textarea, separated by a blank line.
-         * If the textarea is empty the block becomes the entire content.
-         * @param {string} block
-         */
-        appendBlock(block) {
-          const cur = this.customPrompt.trimEnd();
-          this.customPrompt = cur ? cur + '\n' + block : block;
-          this.selectedOption = 'custom';
-          this.$nextTick(() => this.$refs.textarea.focus());
-        },
-
-        /** Highlight the custom-prompt option and focus the textarea. */
-        selectCustom() {
-          this.selectedOption = 'custom';
-          this.$nextTick(() => this.$refs.textarea.focus());
-        },
-
-        /**
-         * Copy a history entry into the textarea and focus it.
-         * @param {number} i - Index into promptHistory
-         */
-        selectHistory(i) {
-          this.customPrompt = this.promptHistory[i];
-          this.selectedOption = 'custom';
-          this.$nextTick(() => this.$refs.textarea.focus());
-        },
-
-        /**
-         * POST the current textarea content to the server and update local
-         * state. Does nothing if the textarea is blank.
-         */
-        async saveCurrentIfNeeded() {
-          const prompt = this.customPrompt.trim();
-          if (!prompt) return;
-
-          const filePath = this.currentImage;
-          this.selections[filePath] = prompt;
-          this._addToHistory(prompt);
-
-          await fetch('/api/save', {
-            method: 'POST',
-            headers: { 'Content-Type': 'application/json' },
-            body: JSON.stringify({ file_path: filePath, prompt }),
-          });
-        },
-
-        /**
-         * Prepend a prompt to history, removing any earlier duplicate entry.
-         * @param {string} prompt
-         */
-        _addToHistory(prompt) {
-          this.promptHistory = [
-            prompt,
-            ...this.promptHistory.filter(p => p !== prompt),
-          ];
-        },
-
-        /**
-         * Populate the textarea with the saved prompt for the current image,
-         * or clear it if no prompt has been saved yet.
-         */
-        _loadCurrentSelection() {
-          const saved = this.selections[this.currentImage];
-          this.customPrompt = saved ?? '';
-          this.selectedOption = saved ? 'custom' : null;
-        },
-
-        /** Save the current prompt (if any) then advance to the next image. */
-        async navigateNext() {
-          if (this.currentIndex >= this.images.length - 1) return;
-          await this.saveCurrentIfNeeded();
-          this.currentIndex++;
-          this._loadCurrentSelection();
-        },
-
-        /** Save the current prompt (if any) then go back to the previous image. */
-        async navigatePrev() {
-          if (this.currentIndex <= 0) return;
-          await this.saveCurrentIfNeeded();
-          this.currentIndex--;
-          this._loadCurrentSelection();
-        },
-
-        /** Save the current prompt (if any) and advance to the next image. */
-        async saveAndNext() {
-          await this.saveCurrentIfNeeded();
-          if (this.currentIndex < this.images.length - 1) {
-            this.currentIndex++;
-            this._loadCurrentSelection();
-          }
-        },
-
-        /**
-         * Save the current prompt (if any) then jump to a specific index.
-         * Out-of-range values are clamped silently.
-         * @param {number} idx - Zero-based target index
-         */
-        async jumpTo(idx) {
-          const target = Math.max(0, Math.min(idx, this.images.length - 1));
-          if (target === this.currentIndex) return;
-          await this.saveCurrentIfNeeded();
-          this.currentIndex = target;
-          this._loadCurrentSelection();
-        },
-
-        /**
-         * Global keyboard handler (attached to window via @keydown.window).
-         *
-         * Shortcuts:
-         *   Alt+1        → focus the custom-prompt textarea
-         *   Alt+N (N≥2)  → load history[N-2] into the textarea and focus it
-         *   → / k        → next image  (k ignored while textarea is focused)
-         *   ← / j        → prev image  (j ignored while textarea is focused)
-         *
-         * Arrow keys navigate from anywhere, including inside the textarea,
-         * and auto-save the current prompt if non-empty.
-         *
-         * @param {KeyboardEvent} e
-         */
-        handleKey(e) {
-          const inTextarea =
-            this.$refs.textarea &&
-            document.activeElement === this.$refs.textarea;
-          if (inTextarea) return;
-
-          if (e.altKey) {
-            const m = e.code.match(/^Digit(\d)$/);
-            if (m) {
-              const n = parseInt(m[1], 10);
-              e.preventDefault();
-              if (n === 1) {
-                this.selectCustom();
-              } else if (n >= 2 && n - 2 < this.promptHistory.length) {
-                this.selectHistory(n - 2);
-              }
-              return;
-            }
-          }
-
-          if (e.key === 'ArrowRight' || e.key === 'k') {
-            e.preventDefault();
-            this.navigateNext();
-          } else if (e.key === 'ArrowLeft' || e.key === 'j') {
-            e.preventDefault();
-            this.navigatePrev();
-          }
-        },
-      }));
-    });
-  </script>
-</body>
-</html>

prompter/prompter.py

@@ -1,205 +0,0 @@
-#!/usr/bin/env -S uv run --script
-# /// script
-# requires-python = ">=3.11"
-# dependencies = [
-#     "bottle",
-# ]
-# ///
-"""Browse images one-by-one and assign text prompts via a local web UI.
-
-Saves results to a JSONL file: {"filename": "...", "prompt": "..."}
-Images navigated past without a prompt produce no output line.
-"""
-
-import argparse
-import json
-import socket
-import sys
-import threading
-import time
-import webbrowser
-from pathlib import Path
-
-import bottle
-
-# ---------------------------------------------------------------------------
-# Configuration
-# ---------------------------------------------------------------------------
-
-CANNED_PROMPTS: list[str] = [
-    "photorealistic. skin must be flawless",
-]
-"""Pre-populated prompt history shown in the UI on first launch."""
-
-BUILDING_BLOCKS: list[str] = [
-    "make her prettier",
-    "age her by 5 years and make her look like a 25 year old",
-    "keep the dark skin",
-    "wet skin",
-    "skin covered with baby oil, shiny skin",
-    "soaking wet skin and hair, wet clothes clinging to her body",
-    "long, flowy hair",
-    "japanese",
-    "greek",
-    "no extra or missing fingers. each hand must have 5 fingers with the same hand pose as the original image",
-    # expression
-    "direct her gaze at the camera",
-    "give her a serious and seductive look",
-    "give her a playful expression",
-    "slightly parted lips",
-    "french kiss, eyes closed, tongue out",
-    "closed eyes",
-    "lower her eyelids and slightly part her lips in a seductive fashion",
-    "exaggerated expression",
-    "soft dramatic lighting",
-    # outfit
-    "replace the outfit with glossy latex",
-    "remove the tan lines, she doesn't wear white bra",
-]
-"""Palette of text snippets the user can append to any prompt."""
-
-VALID_EXTENSIONS: frozenset[str] = frozenset({".webp", ".jpg", ".jpeg", ".png"})
-
-HTML: str = (Path(__file__).parent / "prompter.html").read_text(encoding="utf-8")
-
-# ---------------------------------------------------------------------------
-# App
-# ---------------------------------------------------------------------------
-
-
-def find_free_port() -> int:
-    """Bind to port 0 and return the OS-assigned port number."""
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        s.bind(("127.0.0.1", 0))
-        return s.getsockname()[1]
-
-
-class PrompterApp:
-    def __init__(self, images: list[str], output_path: Path) -> None:
-        self.images = images
-        self.output_path = output_path
-        self.app = bottle.Bottle()
-        self.app.route("/")(self.index)
-        self.app.route("/api/images")(self.api_images)
-        self.app.route("/image/<idx:int>")(self.serve_image)
-        self.app.route("/api/save", method="POST")(self.api_save)
-
-    def index(self) -> str:
-        return HTML
-
-    def api_images(self) -> str:
-        """Return image list, canned prompts, building blocks, and existing prompts."""
-        stem_to_path: dict[str, str] = {Path(p).stem: p for p in self.images}
-        existing_by_stem = self._load_existing_prompts()
-        existing_prompts = {
-            stem_to_path[stem]: prompt
-            for stem, prompt in existing_by_stem.items()
-            if stem in stem_to_path
-        }
-        bottle.response.content_type = "application/json"
-        return json.dumps(
-            {
-                "images": self.images,
-                "canned_prompts": CANNED_PROMPTS,
-                "building_blocks": BUILDING_BLOCKS,
-                "existing_prompts": existing_prompts,
-            }
-        )
-
-    def serve_image(self, idx: int) -> bottle.HTTPResponse:
-        if idx < 0 or idx >= len(self.images):
-            bottle.abort(404, "Image not found")
-        path = Path(self.images[idx])
-        return bottle.static_file(path.name, root=str(path.parent))
-
-    def api_save(self) -> str:
-        data: dict = bottle.request.json or {}
-        file_path: str = data.get("file_path", "")
-        prompt: str = data.get("prompt", "")
-        if file_path and prompt:
-            self._save_entry(file_path, prompt)
-        bottle.response.content_type = "application/json"
-        return json.dumps({"ok": True})
-
-    def _load_existing_prompts(self) -> dict[str, str]:
-        """Read the output JSONL and return a mapping of stem → prompt.
-
-        When a filename appears multiple times the last entry wins, so re-running
-        the tool and overwriting a previous prompt works naturally.
-        """
-        if not self.output_path.exists():
-            return {}
-        prompts: dict[str, str] = {}
-        with open(self.output_path, encoding="utf-8") as f:
-            for line in f:
-                line = line.strip()
-                if not line:
-                    continue
-                try:
-                    entry = json.loads(line)
-                    if "filename" in entry and "prompt" in entry:
-                        prompts[entry["filename"]] = entry["prompt"]
-                except json.JSONDecodeError:
-                    pass
-        return prompts
-
-    def _save_entry(self, file_path: str, prompt: str) -> None:
-        """Append one {filename, prompt} record to the JSONL output file."""
-        with open(self.output_path, "a", encoding="utf-8") as f:
-            f.write(
-                json.dumps({"filename": Path(file_path).stem, "prompt": prompt}) + "\n"
-            )
-
-    def run(self, port: int) -> None:
-        bottle.run(self.app, host="localhost", port=port, quiet=True)
-
-
-# ---------------------------------------------------------------------------
-# Entry point
-# ---------------------------------------------------------------------------
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(
-        description="Browse images and assign text prompts via a web UI.",
-    )
-    parser.add_argument("images", nargs="+", help="Image paths to browse")
-    parser.add_argument(
-        "-o",
-        "--output-path",
-        default="prompts.jsonl",
-        metavar="FILE",
-        help="Output JSONL file path (default: prompts.jsonl)",
-    )
-    args = parser.parse_args()
-
-    images = [
-        str(Path(p).resolve())
-        for p in args.images
-        if Path(p).suffix.lower() in VALID_EXTENSIONS
-    ]
-
-    if not images:
-        exts = ", ".join(sorted(VALID_EXTENSIONS))
-        print(
-            f"Error: no valid images found. Expected extensions: {exts}",
-            file=sys.stderr,
-        )
-        sys.exit(1)
-
-    print(f"Loaded {len(images)} image(s).")
-
-    port = find_free_port()
-    url = f"http://localhost:{port}"
-    print(f"Starting server at {url}")
-
-    threading.Thread(
-        target=lambda: (time.sleep(0.8), webbrowser.open(url)),
-        daemon=True,
-    ).start()
-
-    PrompterApp(images, Path(args.output_path)).run(port)
-
-
-if __name__ == "__main__":
-    main()

reset_router.py

@@ -1,97 +0,0 @@
-#!/usr/bin/env -S uv run --script
-# /// script
-# dependencies = [
-#   "playwright>=1.45.0",
-# ]
-# ///
-
-from __future__ import annotations
-
-import argparse
-import sys
-import time
-
-from playwright.sync_api import Playwright, TimeoutError as PlaywrightTimeoutError, sync_playwright
-
-ROUTER_URL = "http://192.168.1.1"
-ROUTER_USERNAME = "abdus"
-ROUTER_PASSWORD = "xAsametk50"
-
-
-def reboot_router(playwright: Playwright, headed: bool, timeout_ms: int) -> None:
-    browser = playwright.chromium.launch(headless=not headed)
-    context = browser.new_context(ignore_https_errors=True)
-    page = context.new_page()
-    page.set_default_timeout(timeout_ms)
-
-    try:
-        page.goto(f"{ROUTER_URL}/login", wait_until="domcontentloaded")
-
-        username_box = page.get_by_role("textbox", name="User Name")
-        password_box = page.get_by_role("textbox", name="Password")
-
-        username_box.click()
-        username_box.press("ControlOrMeta+a")
-        username_box.press("Backspace")
-        username_box.type(ROUTER_USERNAME, delay=80)
-
-        password_box.click()
-        password_box.press("ControlOrMeta+a")
-        password_box.press("Backspace")
-        password_box.type(ROUTER_PASSWORD, delay=140)
-        page.get_by_role("textbox", name="Password").press("Enter")
-
-        page.wait_for_load_state("domcontentloaded")
-
-        try:
-            page.get_by_text("The username or password is not correct", exact=False).wait_for(
-                state="visible", timeout=1500
-            )
-            raise RuntimeError("Router rejected configured credentials")
-        except PlaywrightTimeoutError:
-            pass
-
-        menu_trigger = page.locator("#h_menu_list").first
-        if menu_trigger.count() and menu_trigger.is_visible():
-            menu_trigger.click()
-
-        restart_icon = page.locator("#navbar_reboot .icon-menu-restart, #navbar_reboot").first
-        restart_icon.wait_for(state="visible")
-
-        with page.expect_response(lambda r: "/cgi-bin/Reboot" in r.url, timeout=timeout_ms):
-            restart_icon.click()
-            page.get_by_role("button", name="OK").click()
-
-        time.sleep(5)
-
-        print("Router reboot request sent.")
-    finally:
-        context.close()
-        browser.close()
-
-
-def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Log in and reboot router")
-    parser.add_argument("--headed", action="store_true", help="Run with visible browser")
-    parser.add_argument(
-        "--timeout",
-        type=int,
-        default=60000,
-        help="Timeout in milliseconds (default: 60000)",
-    )
-    return parser.parse_args()
-
-
-def main() -> int:
-    args = parse_args()
-    try:
-        with sync_playwright() as playwright:
-            reboot_router(playwright, headed=args.headed, timeout_ms=args.timeout)
-        return 0
-    except Exception as exc:
-        print(f"Error: {exc}", file=sys.stderr)
-        return 1
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

rewrite_srt.py

@@ -0,0 +1,364 @@
+#!/usr/bin/env python3
+import argparse
+import logging
+import re
+import textwrap
+from dataclasses import dataclass
+from pathlib import Path
+
+# lowercase tokens (en, tr) whose trailing period does not end a sentence
+ABBREVIATIONS = {"mr", "mrs", "ms", "dr", "prof", "e.g", "i.e", "vs", "etc", "doç", "av", "sn", "vb", "bkz"}
+# Finds candidate sentence boundaries: end punctuation, optional closers, then whitespace.
+# Matches in "He left.  Then" -> ".  " and in 'He said "no." Then' -> '." '.
+SENTENCE_END = re.compile(
+    r"""
+    [.!?…]+         # one or more end marks: "." "?!" "..." or the ellipsis character
+    [\"'”’)\]]*     # closing quotes or brackets that belong to the finished sentence
+    \s+             # whitespace after it; one or more, so double spaces work too
+    """,
+    re.VERBOSE,
+)
+# Matches an SRT time line such as "00:01:02,345 --> 00:01:04,000" and captures the 8 time fields.
+TIMELINE_PATTERN = re.compile(
+    r"""
+    (\d+):(\d{2}):(\d{2})[,.](\d{3})    # start: hours, minutes, seconds, milliseconds (comma or dot)
+    \s*-->\s*                           # arrow, with optional spaces around it
+    (\d+):(\d{2}):(\d{2})[,.](\d{3})    # end, same layout
+    """,
+    re.VERBOSE,
+)
+# Matches a whole cue that is only a sound description or music, e.g. "[Music]", "(laughing)", "♪ la la ♪".
+NONSPEECH_PATTERN = re.compile(
+    r"""
+    ^(
+        \[[^\]]*\]      # [ ... ]
+      | \([^)]*\)       # ( ... )
+      | ♪[^♪]*♪?       # ♪ ... ♪, closing note optional
+      | ♫[^♫]*♫?       # ♫ ... ♫, closing note optional
+    )$
+    """,
+    re.VERBOSE,
+)
+# Matches inline markup to strip: HTML tags like <i> and </i>, and ASS/SSA overrides like {\an8}.
+MARKUP_PATTERN = re.compile(
+    r"""
+    <[^>]+>         # < ... >
+  | \{[^}]*\}       # { ... }
+    """,
+    re.VERBOSE,
+)
+MIN_CUE_DURATION = 0.2
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class Cue:
+    """A subtitle cue or piece of one; times in seconds. `dialogue` marks cues holding several '- ' speaker lines."""
+
+    start: float
+    end: float
+    text: str
+    dialogue: bool = False
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Re-time and re-segment an SRT file mechanically: merge by sentence, split long cues, wrap lines, enforce reading speed.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("subtitle", type=Path, help="Input SRT file")
+    parser.add_argument("--output", "-o", type=Path, help="Output SRT path (default: <name>.rewritten.srt)")
+    parser.add_argument("--max-line-length", type=int, default=42, help="Max characters per line")
+    parser.add_argument("--max-lines", type=int, default=2, help="Max lines per cue")
+    parser.add_argument("--max-cps", type=float, default=17.0, help="Max characters per second before a cue's end is extended")
+    parser.add_argument("--min-duration", type=float, default=1.0, help="Cues shorter than this (seconds) are merged into a neighbour")
+    parser.add_argument("--max-duration", type=float, default=7.0, help="Cues longer than this (seconds) are split")
+    parser.add_argument("--min-gap", type=float, default=0.04, help="Minimum gap between cues in seconds")
+    parser.add_argument("--drop-nonspeech", action="store_true", help="Remove cues that are only [..], (..) or music notes")
+    return parser.parse_args()
+
+
+def parse_timestamp(hours: str, minutes: str, seconds: str, milliseconds: str) -> float:
+    """Convert the four captured time fields of TIMELINE_PATTERN to seconds."""
+    return int(hours) * 3600 + int(minutes) * 60 + int(seconds) + int(milliseconds) / 1000
+
+
+def parse_srt(content: str) -> list[Cue]:
+    """Parse SRT text into cues sorted by start time.
+
+    Tolerates CRLF line endings and a missing index line. Text lines are flattened into one line and markup is
+    stripped. A cue with two or more lines starting with '- ' is flagged as dialogue. Empty cues are dropped.
+    """
+    cues = []
+    # \n\s*\n = a blank line (possibly containing spaces) separates SRT blocks
+    for block in re.split(r"\n\s*\n", content.replace("\r\n", "\n").replace("\r", "\n").strip()):
+        lines = block.split("\n")
+        match = next((found for line in lines[:3] if (found := TIMELINE_PATTERN.search(line))), None)
+        if match is None:
+            raise ValueError(f"Invalid SRT block: {block[:80]!r}")
+        position = next(position for position, line in enumerate(lines[:3]) if TIMELINE_PATTERN.search(line))
+        body = [line.strip() for line in lines[position + 1 :] if line.strip()]
+        # \s+ -> " " collapses runs of whitespace (including the gaps left by removed markup) to one space
+        text = re.sub(r"\s+", " ", MARKUP_PATTERN.sub("", " ".join(body))).strip()
+        if text:
+            cues.append(
+                Cue(
+                    start=parse_timestamp(*match.group(1, 2, 3, 4)),
+                    end=parse_timestamp(*match.group(5, 6, 7, 8)),
+                    text=text,
+                    dialogue=sum(line.startswith("- ") for line in body) >= 2,
+                )
+            )
+    if not cues:
+        raise ValueError("SRT contains no cues")
+    cues.sort(key=lambda cue: cue.start)
+    return cues
+
+
+def format_timestamp(seconds: float) -> str:
+    """Format seconds as an SRT timestamp, HH:MM:SS,mmm."""
+    total_milliseconds = round(seconds * 1000)
+    hours, remainder = divmod(total_milliseconds, 3_600_000)
+    minutes, remainder = divmod(remainder, 60_000)
+    secs, milliseconds = divmod(remainder, 1000)
+    return f"{hours:02d}:{minutes:02d}:{secs:02d},{milliseconds:03d}"
+
+
+def split_sentences(text: str) -> list[str]:
+    """Split text into sentences at SENTENCE_END matches.
+
+    A boundary is kept only if the next character starts a sentence (uppercase, digit, opening quote/bracket or a
+    '- ' speaker dash) and the word before a period is not a known abbreviation or a single-letter initial.
+    Decimals such as "3.5" never match because no whitespace follows the period.
+    """
+    sentences = []
+    start = 0
+    for match in SENTENCE_END.finditer(text):
+        following = text[match.end() : match.end() + 1]
+        starts_sentence = following.isupper() or following.isdigit() or following in "\"'“‘([" or text.startswith("- ", match.end())
+        if not following or not starts_sentence:
+            continue
+        punctuation = match.group().strip()
+        # (\S+)$ = the last whitespace-delimited word before the punctuation, e.g. "Dr" in "... Dr."
+        word = re.search(r"(\S+)$", text[start : match.start()])
+        token = word.group(1).lower().lstrip("\"'“‘([") if word else ""
+        if punctuation.startswith(".") and (token in ABBREVIATIONS or (len(token) == 1 and token.isalpha())):
+            continue
+        sentences.append(text[start : match.start() + len(match.group().rstrip())].strip())
+        start = match.end()
+    sentences.append(text[start:].strip())
+    return [sentence for sentence in sentences if sentence]
+
+
+def wrap_text(text: str, width: int, max_lines: int) -> list[str] | None:
+    """Wrap text to at most `max_lines` lines of at most `width` characters, preferring balanced lines.
+
+    Text of the form "- a - b" is split into one line per speaker when each fits. Returns None if no wrapping fits.
+    """
+    if len(text) <= width:
+        return [text]
+    if text.startswith("- ") and " - " in text:
+        # \s+(?=- ) = whitespace followed by a dash-space (lookahead, not consumed): splits before each speaker dash
+        speakers = re.split(r"\s+(?=- )", text)
+        if len(speakers) <= max_lines and all(len(line) <= width for line in speakers):
+            return speakers
+    for count in range(2, max_lines + 1):
+        lines = textwrap.wrap(text, width=-(-len(text) // count), break_long_words=False, break_on_hyphens=False)
+        if len(lines) <= max_lines and all(len(line) <= width for line in lines):
+            return lines
+    lines = textwrap.wrap(text, width=width, break_long_words=False, break_on_hyphens=False)
+    return lines if len(lines) <= max_lines and all(len(line) <= width for line in lines) else None
+
+
+def wrap_dialogue(text: str, width: int) -> list[str]:
+    """Wrap a dialogue cue, one or more lines per speaker. The number of lines is not limited."""
+    lines = []
+    # \s+(?=- ) = whitespace followed by a dash-space (lookahead, not consumed): splits before each speaker dash
+    for speaker in re.split(r"\s+(?=- )", text):
+        lines.extend(textwrap.wrap(speaker, width=width, break_long_words=False, break_on_hyphens=False))
+    return lines
+
+
+def time_at(spans: list[tuple[int, Cue]], offset: int, is_end: bool) -> float:
+    """Map a character offset in the joined text to a time, interpolating linearly inside the cue that holds it.
+
+    `spans` are (offset of cue text in the joined text, cue). For an end offset the cue boundary belongs to the
+    earlier cue, for a start offset to the later one. Offsets in the joining gap snap to the nearest cue edge.
+    """
+    for span_offset, cue in spans:
+        length = len(cue.text)
+        inside = span_offset < offset <= span_offset + length if is_end else span_offset <= offset < span_offset + length
+        if inside:
+            return cue.start + (cue.end - cue.start) * (offset - span_offset) / length
+    span_offset, cue = min(spans, key=lambda span: min(abs(offset - span[0]), abs(offset - span[0] - len(span[1].text))))
+    return cue.start if offset <= span_offset else cue.end
+
+
+def sentences_with_times(cues: list[Cue]) -> list[Cue]:
+    """Join the cues' text, split it into sentences and give each sentence a start and end time.
+
+    A sentence spanning several cues starts where its first character is spoken and ends where its last one is.
+    """
+    joined = " ".join(cue.text for cue in cues)
+    spans = []
+    offset = 0
+    for cue in cues:
+        spans.append((offset, cue))
+        offset += len(cue.text) + 1
+    pieces = []
+    cursor = 0
+    for sentence in split_sentences(text=joined):
+        position = joined.index(sentence, cursor)
+        cursor = position + len(sentence)
+        pieces.append(
+            Cue(
+                start=time_at(spans=spans, offset=position, is_end=False),
+                end=time_at(spans=spans, offset=cursor, is_end=True),
+                text=sentence,
+            )
+        )
+    return pieces
+
+
+def fits(text: str, duration: float, args: argparse.Namespace) -> bool:
+    """Whether one cue can hold `text`: within --max-duration and wrappable into --max-lines lines."""
+    return duration <= args.max_duration and wrap_text(text=text, width=args.max_line_length, max_lines=args.max_lines) is not None
+
+
+def merge_short(pieces: list[Cue], args: argparse.Namespace) -> list[Cue]:
+    """Merge a piece into the previous one when either is shorter than --min-duration and the result still fits."""
+    merged: list[Cue] = []
+    for piece in pieces:
+        if merged:
+            previous = merged[-1]
+            short = piece.end - piece.start < args.min_duration or previous.end - previous.start < args.min_duration
+            text = f"{previous.text} {piece.text}"
+            if short and fits(text=text, duration=piece.end - previous.start, args=args):
+                previous.end = piece.end
+                previous.text = text
+                continue
+        merged.append(piece)
+    return merged
+
+
+def find_split(text: str) -> int | None:
+    """Pick the index at which to cut `text` in two, or None if it is a single word.
+
+    Prefers the start of the word after a comma, semicolon or colon if it is within 25% of the length from the
+    middle; otherwise the word start nearest the middle.
+    """
+    middle = len(text) / 2
+    # \s+ = any whitespace run; match.end() is where the next word starts (interior only, not at either edge)
+    word_starts = [match.end() for match in re.finditer(r"\s+", text) if 0 < match.start() and match.end() < len(text)]
+    if not word_starts:
+        return None
+    # [,;:]\s+ = a clause mark followed by whitespace; match.end() is the start of the next clause
+    clause_starts = [match.end() for match in re.finditer(r"[,;:]\s+", text) if match.end() < len(text)]
+    if clause_starts:
+        best = min(clause_starts, key=lambda index: abs(index - middle))
+        if abs(best - middle) <= len(text) * 0.25:
+            return best
+    return min(word_starts, key=lambda index: abs(index - middle))
+
+
+def split_piece(piece: Cue, args: argparse.Namespace) -> list[Cue]:
+    """Recursively split a piece until every part fits, dividing its time in proportion to character length."""
+    if fits(text=piece.text, duration=piece.end - piece.start, args=args):
+        return [piece]
+    index = find_split(text=piece.text)
+    if index is None:
+        return [piece]
+    left, right = piece.text[:index].strip(), piece.text[index:].strip()
+    middle = piece.start + (piece.end - piece.start) * len(left) / (len(left) + len(right))
+    return split_piece(piece=Cue(start=piece.start, end=middle, text=left), args=args) + split_piece(
+        piece=Cue(start=middle, end=piece.end, text=right), args=args
+    )
+
+
+def retime(pieces: list[Cue], args: argparse.Namespace) -> list[int]:
+    """Fix timing in place: remove overlaps, keep --min-gap, and extend a cue's end to meet --max-cps.
+
+    An end is only extended into free time before the next cue and never beyond --max-duration.
+    Returns the 1-based numbers of cues still above --max-cps.
+    """
+    over_budget = []
+    for index, piece in enumerate(pieces):
+        if index and piece.start < pieces[index - 1].end:
+            piece.start = pieces[index - 1].end
+        limit = pieces[index + 1].start - args.min_gap if index + 1 < len(pieces) else float("inf")
+        wanted = piece.start + len(piece.text) / args.max_cps
+        ceiling = min(piece.start + args.max_duration, limit)
+        if wanted > piece.end:
+            piece.end = max(piece.end, min(wanted, ceiling))
+        if piece.end > limit:
+            piece.end = max(limit, piece.start + MIN_CUE_DURATION)
+        if piece.end <= piece.start:
+            piece.end = piece.start + MIN_CUE_DURATION
+        if len(piece.text) / (piece.end - piece.start) > args.max_cps + 0.01:
+            over_budget.append(index + 1)
+    return over_budget
+
+
+def rewrite(cues: list[Cue], args: argparse.Namespace) -> tuple[list[Cue], list[int]]:
+    """Run the full pipeline. Dialogue and non-speech cues act as barriers and pass through unchanged;
+    the speech between them is merged into sentences, short pieces are merged, long ones split, then retimed.
+
+    Returns (pieces, numbers of cues still above --max-cps).
+    """
+    pieces: list[Cue] = []
+    run: list[Cue] = []
+
+    def flush() -> None:
+        if run:
+            pieces.extend(
+                part
+                for piece in merge_short(pieces=sentences_with_times(cues=run), args=args)
+                for part in split_piece(piece=piece, args=args)
+            )
+            run.clear()
+
+    for cue in cues:
+        if cue.dialogue or NONSPEECH_PATTERN.match(cue.text):
+            flush()
+            if cue.dialogue or not args.drop_nonspeech:
+                pieces.append(Cue(start=cue.start, end=cue.end, text=cue.text, dialogue=cue.dialogue))
+        else:
+            run.append(cue)
+    flush()
+    return pieces, retime(pieces=pieces, args=args)
+
+
+def build_srt(pieces: list[Cue], args: argparse.Namespace) -> str:
+    """Render the pieces as SRT, wrapping each to --max-line-length (dialogue cues one speaker per line)."""
+    blocks = []
+    for index, piece in enumerate(pieces, start=1):
+        if piece.dialogue:
+            lines = wrap_dialogue(text=piece.text, width=args.max_line_length)
+        else:
+            lines = wrap_text(text=piece.text, width=args.max_line_length, max_lines=args.max_lines) or [piece.text]
+        blocks.append(f"{index}\n{format_timestamp(piece.start)} --> {format_timestamp(piece.end)}\n" + "\n".join(lines) + "\n")
+    return "\n".join(blocks)
+
+
+def main() -> None:
+    """Read the SRT, rewrite it and write the result next to it, refusing to overwrite the input."""
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s")
+    args = parse_args()
+    if min(args.max_line_length, args.max_lines) < 1 or args.max_cps <= 0:
+        raise SystemExit("--max-line-length, --max-lines and --max-cps must be positive")
+    output = args.output or args.subtitle.with_name(f"{args.subtitle.stem}.rewritten.srt")
+    if output.resolve() == args.subtitle.resolve():
+        raise SystemExit("Refusing to overwrite the input file")
+
+    cues = parse_srt(content=args.subtitle.read_text(encoding="utf-8-sig"))
+    pieces, over_budget = rewrite(cues=cues, args=args)
+    output.write_text(build_srt(pieces=pieces, args=args), encoding="utf-8")
+    logger.info("Wrote %d cues (from %d) to %s", len(pieces), len(cues), output)
+    if over_budget:
+        logger.warning("%d cue(s) still above %.1f chars/s (text is not shortened): %s", len(over_budget), args.max_cps, over_budget[:20])
+
+
+if __name__ == "__main__":
+    main()

router_monitor.py

@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+
+import argparse
+import base64
+import csv
+import json
+import logging
+import re
+import ssl
+import subprocess
+import time
+import urllib.request
+from datetime import datetime
+from pathlib import Path
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s")
+logger = logging.getLogger(__name__)
+
+ROUTER_HOST = "192.168.1.1"
+ROUTER_USERNAME = "abdus"
+ROUTER_PASSWORD = "xAsametk50"
+
+COLUMNS = [
+    "timestamp",
+    "router_uptime_s",
+    "dsl_state",
+    "dsl_uptime_s",
+    "down_kbps",
+    "up_kbps",
+    "snr_down_db",
+    "snr_up_db",
+    "attn_down_db",
+    "attn_up_db",
+    "dsl_errors",
+    "wan_ip",
+    "ping_router_ms",
+    "ping_isp_gateway_ms",
+    "ping_internet_ms",
+]
+UNIT_SECONDS = {"day": 86400, "hour": 3600, "min": 60, "sec": 1}
+SLOW_PING_MS = 150
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Log DSL, PPPoE and ping health of the Zyxel router to a CSV to tell ISP from device drops",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("--output", type=Path, default=Path.cwd() / "router_monitor.csv")
+    parser.add_argument("--interval", type=float, default=30.0, help="Seconds between samples")
+    parser.add_argument("--internet-target", default="8.8.8.8")
+    parser.add_argument("--timeout", type=float, default=10.0, help="Router request timeout in seconds")
+    return parser.parse_args()
+
+
+def build_opener() -> urllib.request.OpenerDirector:
+    # Self-signed certificate; over HTTPS the router accepts plain JSON login.
+    context = ssl.create_default_context()
+    context.check_hostname = False
+    context.verify_mode = ssl.CERT_NONE
+    return urllib.request.build_opener(
+        urllib.request.HTTPSHandler(context=context),
+        urllib.request.HTTPCookieProcessor(),
+    )
+
+
+def login(opener: urllib.request.OpenerDirector, base_url: str, timeout: float) -> str:
+    request = urllib.request.Request(
+        url=f"{base_url}/UserLogin",
+        data=json.dumps(
+            {
+                "Input_Account": ROUTER_USERNAME,
+                "Input_Passwd": base64.b64encode(ROUTER_PASSWORD.encode()).decode(),
+                "currLang": "en",
+                "RememberPassword": 0,
+                "SHA512_password": False,
+            }
+        ).encode(),
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    with opener.open(request, timeout=timeout) as response:
+        body = json.loads(response.read().decode())
+    if body.get("result") != "ZCFG_SUCCESS":
+        raise RuntimeError(f"Login rejected: {body}")
+    return body.get("sessionkey", "")
+
+
+def get_json(opener: urllib.request.OpenerDirector, url: str, timeout: float) -> object:
+    with opener.open(url, timeout=timeout) as response:
+        return json.loads(response.read().decode(errors="replace"))
+
+
+def parse_duration(text: str) -> int:
+    return sum(int(number) * UNIT_SECONDS[unit] for number, unit in re.findall(r"(\d+)\s*(day|hour|min|sec)", text))
+
+
+def find_number(pattern: str, text: str) -> float | None:
+    match = re.search(pattern, text)
+    return float(match.group(1)) if match else None
+
+
+def parse_dsl(text: str) -> dict:
+    state = re.search(r"Line State:\s*(\S+)", text)
+    uptime = re.search(r"DSL uptime\s*:(.*)", text)
+    errors = sum(int(count) for count in re.findall(r"(?:CRC|FEC|HEC) error \w+: (\d+)", text))
+    return {
+        "dsl_state": state.group(1) if state else "",
+        "dsl_uptime_s": parse_duration(text=uptime.group(1)) if uptime else "",
+        "down_kbps": find_number(pattern=r"near-end fast channel bit rate: (\d+)", text=text),
+        "up_kbps": find_number(pattern=r"far-end fast channel bit rate: (\d+)", text=text),
+        "snr_down_db": find_number(pattern=r"noise margin downstream: ([\d.]+)", text=text),
+        "snr_up_db": find_number(pattern=r"noise margin upstream: ([\d.]+)", text=text),
+        "attn_down_db": find_number(pattern=r"attenuation downstream: ([\d.]+)", text=text),
+        "attn_up_db": find_number(pattern=r"attenuation upstream: ([\d.]+)", text=text),
+        "dsl_errors": errors,
+    }
+
+
+def read_router(opener: urllib.request.OpenerDirector, base_url: str, session_key: str, timeout: float) -> dict:
+    def fetch(path: str) -> object:
+        return get_json(opener=opener, url=f"{base_url}{path}&sessionkey={session_key}", timeout=timeout)
+
+    dsl = fetch("/cgi-bin/xDSLStatistics_handle?line=0")
+    status = fetch("/cgi-bin/DAL?oid=status")
+    wan = fetch("/cgi-bin/DAL?oid=wan")
+
+    sample = parse_dsl(text=dsl[0]["result"])
+    sample["router_uptime_s"] = status["Object"][0]["DeviceInfo"]["UpTime"]
+    vdsl = next((entry for entry in wan["Object"] if entry.get("Name") == "VDSL"), {})
+    sample["wan_ip"] = vdsl.get("IPAddress", "")
+    sample["isp_gateway"] = vdsl.get("GatewayIPAddress", "")
+    return sample
+
+
+def ping(host: str) -> float | None:
+    if not host:
+        return None
+    result = subprocess.run(
+        ["ping", "-c", "1", "-W", "2000", host],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    return find_number(pattern=r"time=([\d.]+)", text=result.stdout)
+
+
+def is_slow(sample: dict) -> bool:
+    return (sample.get("ping_internet_ms") or 0) > SLOW_PING_MS
+
+
+def describe(sample: dict) -> str:
+    def ms(key: str) -> str:
+        value = sample.get(key)
+        return "LOST" if value is None else f"{value:.0f}ms"
+
+    return (
+        f"router {ms('ping_router_ms')} | internet {ms('ping_internet_ms')} | "
+        f"dsl {sample.get('dsl_state') or '?'} up {sample.get('dsl_uptime_s', '?')}s | "
+        f"errors {sample.get('dsl_errors', '?')} | wan {sample.get('wan_ip') or '-'}"
+    )
+
+
+def log_changes(previous: dict, current: dict) -> None:
+    if not previous:
+        return
+    if current.get("router_uptime_s") != "" and previous.get("router_uptime_s") != "":
+        if current["router_uptime_s"] < previous["router_uptime_s"]:
+            logger.warning("ROUTER REBOOTED (uptime reset)")
+    if current.get("dsl_uptime_s") != "" and previous.get("dsl_uptime_s") != "":
+        if current["dsl_uptime_s"] < previous["dsl_uptime_s"]:
+            logger.warning("DSL LINE RESYNCED (line side / ISP)")
+    if current.get("dsl_state") != previous.get("dsl_state"):
+        logger.warning("DSL state %s -> %s", previous.get("dsl_state"), current.get("dsl_state"))
+    if current.get("wan_ip") != previous.get("wan_ip"):
+        logger.warning("WAN IP %s -> %s (PPPoE session changed)", previous.get("wan_ip"), current.get("wan_ip"))
+    for key, label in (("ping_router_ms", "router"), ("ping_internet_ms", "internet")):
+        if (previous.get(key) is None) != (current.get(key) is None):
+            logger.warning("Ping to %s %s", label, "LOST" if current.get(key) is None else "RESTORED")
+    if is_slow(sample=current) != is_slow(sample=previous):
+        if is_slow(sample=current):
+            logger.warning("Internet ping SLOW: %.0f ms", current["ping_internet_ms"])
+        else:
+            logger.warning("Internet ping back to normal")
+
+
+def main() -> None:
+    args = parse_args()
+    base_url = f"https://{ROUTER_HOST}"
+    opener = build_opener()
+    session_key = ""
+    previous: dict = {}
+    new_file = not args.output.exists()
+
+    with args.output.open(mode="a", newline="") as handle:
+        writer = csv.DictWriter(handle, fieldnames=COLUMNS, extrasaction="ignore")
+        if new_file:
+            writer.writeheader()
+        logger.info("Logging to %s every %.0fs", args.output, args.interval)
+
+        while True:
+            started = time.monotonic()
+            sample = {"timestamp": datetime.now().isoformat(timespec="seconds")}
+            try:
+                if not session_key:
+                    session_key = login(opener=opener, base_url=base_url, timeout=args.timeout)
+                sample.update(
+                    read_router(opener=opener, base_url=base_url, session_key=session_key, timeout=args.timeout)
+                )
+            except Exception as exc:
+                logger.warning("Router query failed: %s", exc)
+                session_key = ""
+
+            sample["ping_router_ms"] = ping(host=ROUTER_HOST)
+            sample["ping_isp_gateway_ms"] = ping(host=sample.get("isp_gateway", ""))
+            sample["ping_internet_ms"] = ping(host=args.internet_target)
+
+            log_changes(previous=previous, current=sample)
+            writer.writerow(sample)
+            handle.flush()
+            logger.info(describe(sample=sample))
+            previous = sample
+            time.sleep(max(0.0, args.interval - (time.monotonic() - started)))
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        logger.info("Stopped")

router_refresh_ip.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+
+import argparse
+import base64
+import json
+import logging
+import ssl
+import time
+import urllib.error
+import urllib.request
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s")
+logger = logging.getLogger(__name__)
+
+ROUTER_HOST = "192.168.1.1"
+ROUTER_USERNAME = "abdus"
+ROUTER_PASSWORD = "xAsametk50"
+PUBLIC_IP_URL = "https://api.ipify.org"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Drop and re-establish the router's PPPoE session to request a new CGNAT public IP, without rebooting",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("--host", default=ROUTER_HOST)
+    parser.add_argument("--timeout", type=float, default=15.0, help="Router request timeout in seconds")
+    parser.add_argument("--wait-after-disconnect", type=float, default=5.0, help="Seconds to wait before reconnecting")
+    parser.add_argument("--max-wait", type=float, default=60.0, help="Seconds to wait for the session to come back up")
+    parser.add_argument("--skip-public-ip-check", action="store_true", help="Do not call an external service to compare public IPs")
+    parser.add_argument("--dry-run", action="store_true", help="Log in and find the active PPPoE session, but do not disconnect it")
+    return parser.parse_args()
+
+
+class RouterSession:
+    """Logs in to the router on enter and always logs out on exit, so a crash never leaks a session.
+
+    The router has a limit on concurrent logins and returns "Maxium number of login account has
+    reached" once it's hit; the only way out of that is waiting for idle sessions to time out.
+    """
+
+    def __init__(self, host: str, timeout: float) -> None:
+        self.base_url = f"https://{host}"
+        self.timeout = timeout
+        self.session_key = ""
+        # The router uses a self-signed certificate. Over HTTPS the UI skips its RSA/AES login layer.
+        context = ssl.create_default_context()
+        context.check_hostname = False
+        context.verify_mode = ssl.CERT_NONE
+        self.opener = urllib.request.build_opener(
+            urllib.request.HTTPSHandler(context=context),
+            urllib.request.HTTPCookieProcessor(),
+        )
+
+    def request(self, path: str, body: dict | None, method: str) -> dict:
+        separator = "&" if "?" in path else "?"
+        suffix = f"{separator}sessionkey={self.session_key}" if self.session_key else ""
+        headers = {"Content-Type": "application/json"}
+        if self.session_key and method != "GET":
+            # Since the ABSB.5 firmware, state-changing requests (POST/PUT/DELETE) are only
+            # accepted with the session key repeated in this header; the ?sessionkey= query
+            # param alone (still required for GETs) gets a bare 403 on everything else.
+            headers["CSRFToken"] = self.session_key
+        request = urllib.request.Request(
+            url=f"{self.base_url}{path}{suffix}",
+            data=json.dumps(body).encode() if body is not None else None,
+            headers=headers,
+            method=method,
+        )
+        with self.opener.open(request, timeout=self.timeout) as response:
+            raw = response.read().decode()
+        return json.loads(raw) if raw.strip() else {}
+
+    def login(self) -> None:
+        body = self.request(
+            path="/UserLogin",
+            body={
+                "Input_Account": ROUTER_USERNAME,
+                "Input_Passwd": base64.b64encode(ROUTER_PASSWORD.encode()).decode(),
+                "currLang": "en",
+                "RememberPassword": 0,
+                "SHA512_password": False,
+            },
+            method="POST",
+        )
+        if body.get("result") != "ZCFG_SUCCESS":
+            raise RuntimeError(f"Login rejected: {body}")
+        self.session_key = body.get("sessionkey", "")
+        logger.info("Logged in as %s", ROUTER_USERNAME)
+
+    def logout(self) -> None:
+        if not self.session_key:
+            return
+        try:
+            self.request(path="/cgi-bin/UserLogout", body={}, method="POST")
+            logger.info("Logged out, session released")
+        except Exception as exc:
+            logger.warning("Logout failed, the router may hold this session until it times out: %s", exc)
+        finally:
+            self.session_key = ""
+
+    def get_status(self) -> dict:
+        return self.request(path="/cgi-bin/DAL?oid=status", body=None, method="GET")
+
+    def wan_action(self, wanppp_idx: int, action: str) -> None:
+        # Disconnect/Connect tear down or rebuild the PPPoE link, so the router often doesn't get
+        # an HTTP response back in time (or at all, if the link it was answering over just dropped).
+        # That doesn't mean the action failed: treat a timeout as "sent, outcome unknown" and let
+        # the caller confirm the real state by polling, instead of raising and giving up.
+        try:
+            reply = self.request(path="/cgi-bin/DAL?oid=status", body={"action": action, "wanpppIdx": wanppp_idx}, method="PUT")
+        except (TimeoutError, urllib.error.URLError, ConnectionError) as exc:
+            logger.info("%s request did not get a response (%s), checking the actual state instead", action, exc)
+            return
+        if reply.get("result") not in (None, "ZCFG_SUCCESS"):
+            raise RuntimeError(f"Router rejected {action}: {reply}")
+
+    def __enter__(self) -> "RouterSession":
+        self.login()
+        return self
+
+    def __exit__(self, exc_type: object, exc_value: object, traceback: object) -> None:
+        self.logout()
+
+
+def find_active_pppoe(status: dict) -> dict:
+    candidates = [
+        entry
+        for entry in status["Object"][0]["WanLanInfo"]
+        if entry.get("Encapsulation") == "PPPoE" and entry.get("Status") == "Up"
+    ]
+    if not candidates:
+        raise RuntimeError("No active PPPoE WAN session found to refresh")
+    if len(candidates) > 1:
+        logger.warning("Multiple active PPPoE sessions found, using the first: %s", candidates[0].get("Name"))
+    return candidates[0]
+
+
+def wan_ip_of(entry: dict) -> str | None:
+    return next(iter(entry.get("IPv4Address") or []), {}).get("IPAddress")
+
+
+def find_wan_entry(status: dict, wanppp_idx: int) -> dict | None:
+    return next(
+        (e for e in status["Object"][0]["WanLanInfo"] if e.get("wanpppIdx") == wanppp_idx and e.get("Encapsulation") == "PPPoE"),
+        None,
+    )
+
+
+def poll_until_up(session: RouterSession, wanppp_idx: int, timeout: float) -> dict | None:
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        try:
+            entry = find_wan_entry(status=session.get_status(), wanppp_idx=wanppp_idx)
+            if entry and entry.get("Status") == "Up" and wan_ip_of(entry=entry):
+                return entry
+        except (urllib.error.URLError, json.JSONDecodeError, ConnectionError, KeyError, IndexError) as exc:
+            logger.info("Still reconnecting: %s", exc)
+        time.sleep(2.0)
+    return None
+
+
+def ensure_reconnected(session: RouterSession, wanppp_idx: int, max_wait: float, connect_retries: int = 2) -> dict:
+    """Poll for the link to come back; if it's stuck at N/A (the router's own auto-reconnect can
+    fail to kick in), nudge it with another Connect call instead of leaving it down."""
+    per_attempt = max(max_wait / (connect_retries + 1), 15.0)
+    for attempt in range(connect_retries + 1):
+        entry = poll_until_up(session=session, wanppp_idx=wanppp_idx, timeout=per_attempt)
+        if entry:
+            return entry
+        if attempt < connect_retries:
+            logger.warning("Not back up yet, retrying Connect (%d/%d)...", attempt + 1, connect_retries)
+            session.wan_action(wanppp_idx=wanppp_idx, action="Connect")
+    raise TimeoutError(f"WAN session did not come back up within {max_wait:.0f}s, even after retrying Connect")
+
+
+def fetch_public_ip(timeout: float) -> str | None:
+    try:
+        with urllib.request.urlopen(PUBLIC_IP_URL, timeout=timeout) as response:
+            return response.read().decode().strip()
+    except Exception as exc:
+        logger.info("Could not fetch public IP from %s: %s", PUBLIC_IP_URL, exc)
+        return None
+
+
+def main() -> None:
+    args = parse_args()
+    public_ip_before = None if args.skip_public_ip_check else fetch_public_ip(timeout=args.timeout)
+    if public_ip_before:
+        logger.info("Public IP before: %s", public_ip_before)
+
+    with RouterSession(host=args.host, timeout=args.timeout) as session:
+        wan = find_active_pppoe(status=session.get_status())
+        wanppp_idx = wan["wanpppIdx"]
+        logger.info("Active PPPoE session: %s (wanpppIdx=%s, current WAN IP=%s)", wan.get("Name"), wanppp_idx, wan_ip_of(entry=wan))
+
+        if args.dry_run:
+            logger.info("Dry run, not disconnecting")
+            return
+
+        logger.info("Disconnecting PPPoE session...")
+        session.wan_action(wanppp_idx=wanppp_idx, action="Disconnect")
+
+        time.sleep(args.wait_after_disconnect)
+
+        logger.info("Reconnecting PPPoE session...")
+        session.wan_action(wanppp_idx=wanppp_idx, action="Connect")
+
+        entry = ensure_reconnected(session=session, wanppp_idx=wanppp_idx, max_wait=args.max_wait)
+        logger.info("Reconnected. WAN IP: %s", wan_ip_of(entry=entry))
+
+        if public_ip_before:
+            public_ip_after = fetch_public_ip(timeout=args.timeout)
+            if public_ip_after:
+                logger.info("Public IP after: %s", public_ip_after)
+                if public_ip_after == public_ip_before:
+                    logger.warning("Public IP did not change (ISP returned the same CGNAT address)")
+                else:
+                    logger.info("Public IP changed: %s -> %s", public_ip_before, public_ip_after)
+
+
+if __name__ == "__main__":
+    main()

router_reset.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+
+import argparse
+import base64
+import json
+import logging
+import ssl
+import urllib.parse
+import urllib.request
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s")
+logger = logging.getLogger(__name__)
+
+ROUTER_HOST = "192.168.1.1"
+ROUTER_USERNAME = "abdus"
+ROUTER_PASSWORD = "xAsametk50"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Log in to the Zyxel router over HTTPS and reboot it",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("--host", default=ROUTER_HOST)
+    parser.add_argument("--username", default=ROUTER_USERNAME)
+    parser.add_argument("--timeout", type=float, default=15.0, help="Request timeout in seconds")
+    parser.add_argument("--dry-run", action="store_true", help="Log in but do not reboot")
+    return parser.parse_args()
+
+
+def build_opener() -> urllib.request.OpenerDirector:
+    # The router uses a self-signed certificate. Over HTTPS the UI skips its RSA/AES login layer.
+    context = ssl.create_default_context()
+    context.check_hostname = False
+    context.verify_mode = ssl.CERT_NONE
+    return urllib.request.build_opener(
+        urllib.request.HTTPSHandler(context=context),
+        urllib.request.HTTPCookieProcessor(),
+    )
+
+
+def post(opener: urllib.request.OpenerDirector, url: str, body: dict, timeout: float) -> dict:
+    request = urllib.request.Request(
+        url=url,
+        data=json.dumps(body).encode(),
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    with opener.open(request, timeout=timeout) as response:
+        raw = response.read().decode()
+    return json.loads(raw) if raw.strip() else {}
+
+
+def login(opener: urllib.request.OpenerDirector, base_url: str, username: str, timeout: float) -> str:
+    body = post(
+        opener=opener,
+        url=f"{base_url}/UserLogin",
+        body={
+            "Input_Account": username,
+            "Input_Passwd": base64.b64encode(ROUTER_PASSWORD.encode()).decode(),
+            "currLang": "en",
+            "RememberPassword": 0,
+            "SHA512_password": False,
+        },
+        timeout=timeout,
+    )
+    if body.get("result") != "ZCFG_SUCCESS":
+        raise RuntimeError(f"Login rejected: {body}")
+    return body.get("sessionkey", "")
+
+
+def reboot(opener: urllib.request.OpenerDirector, base_url: str, session_key: str, timeout: float) -> None:
+    query = f"?{urllib.parse.urlencode({'sessionkey': session_key})}" if session_key else ""
+    body = post(opener=opener, url=f"{base_url}/cgi-bin/Reboot{query}", body={}, timeout=timeout)
+    logger.info("Reboot response: %s", body)
+
+
+def main() -> None:
+    args = parse_args()
+    base_url = f"https://{args.host}"
+    opener = build_opener()
+
+    session_key = login(opener=opener, base_url=base_url, username=args.username, timeout=args.timeout)
+    logger.info("Logged in as %s", args.username)
+    if args.dry_run:
+        logger.info("Dry run, not rebooting")
+        return
+    reboot(opener=opener, base_url=base_url, session_key=session_key, timeout=args.timeout)
+    logger.info("Reboot request sent, router is back in about a minute")
+
+
+if __name__ == "__main__":
+    main()

snot/snot.html

@@ -162,7 +162,12 @@
 <div x-data="app()" class="gapped" @keydown.window="onKeydown">
     <form @submit.prevent="search" class="search-form">
         <div class="row">
-            <input class="input" type="text" x-ref="query" x-model="form.query" placeholder="Search videos" style="width: 300px" />
+            <input class="input" type="text" x-ref="query" x-model="form.query" list="search-history" placeholder="Search videos" style="width: 300px" />
+            <datalist id="search-history">
+                <template x-for="query in searchHistory" :key="query">
+                    <option :value="query"></option>
+                </template>
+            </datalist>
             <input class="button" type="submit" />
         </div>
     </form>
@@ -225,8 +230,38 @@
 
 <script defer src="./alpine@3.x.x.cdn.min.js"></script>
 <script>
+    const searchHistory = {
+        /** @returns {string[]} The most recent searches. */
+        get history() {
+            try {
+                const history = JSON.parse(localStorage.getItem("snot.searchHistory") || "[]");
+                return Array.isArray(history) ? history.filter((query) => typeof query === "string" && query.trim()).slice(0, 40) : [];
+            } catch (error) {
+                console.warn("Could not load search history", error);
+                return [];
+            }
+        },
+        /**
+         * @param {string} value
+         * @returns {string[]} The updated searches.
+         */
+        add(value) {
+            const query = value.trim();
+            if (!query) return this.history;
+            const history = [query, ...this.history.filter((item) => item !== query)].slice(0, 40);
+            try {
+                localStorage.setItem("snot.searchHistory", JSON.stringify(history));
+            } catch (error) {
+                console.warn("Could not save search history", error);
+            }
+            return history;
+        },
+    };
+
+    /** @returns {object} The Alpine application state and methods. */
     const app = () => ({
         results: [],
+        searchHistory: [],
         errorMessage: "",
         form: {
             query: "",
@@ -236,9 +271,17 @@
             minSizeMB: 0,
         },
         sort: { by: null, dir: "asc" },
+        /**
+         * @param {string} url
+         * @returns {string} The Alfred trigger URL.
+         */
         buildAlfredURL(url) {
             return `alfred://runtrigger/dev.abdus.alfred.media/video/?argument=${encodeURIComponent(url)}`;
         },
+        /**
+         * @param {string} field The result field to sort by.
+         * @returns {void}
+         */
         toggleSort(field) {
             if (this.sort.by === field) {
                 this.sort.dir = this.sort.dir === "asc" ? "desc" : "asc";
@@ -247,6 +290,10 @@
                 this.sort.dir = "asc";
             }
         },
+        /**
+         * @param {KeyboardEvent} e
+         * @returns {Promise<void>}
+         */
         async onKeydown(e) {
             if (e.target.tagName === "INPUT" || e.target.tagName === "TEXTAREA") {
                 return;
@@ -256,6 +303,7 @@
                 this.$refs.filterInput.focus();
             }
         },
+        /** @returns {object[]} The filtered and sorted results. */
         get visibleResults() {
             const isEmpty = !this.filter.query && this.filter.minSizeMB === 0;
             let rows = isEmpty
@@ -284,6 +332,10 @@
             { key: "file_path", value: (it) => it.file_path, label: "File Path" },
             { key: "created_at", value: (it) => (it.created_at ? it.created_at.slice(0, 10) : ""), label: "Created At", sortable: true },
         ],
+        /**
+         * @param {string} actor
+         * @returns {string} The actor filter token.
+         */
         actorToken(actor) {
             const escaped = actor.replaceAll('"', '\\"');
             if (escaped.includes(" ")) {
@@ -291,11 +343,20 @@
             }
             return `actor:${escaped}`;
         },
+        /**
+         * @param {string} actor
+         * @returns {boolean} Whether the actor is already in the query.
+         */
         hasActorFilter(actor) {
             const escaped = actor.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replaceAll('"', '\\"');
             const pattern = new RegExp(`(^|\\s|\\|)actor:(?:"${escaped}"|${escaped})(?=\\s|\\||$)`, "i");
             return pattern.test(this.form.query);
         },
+        /**
+         * @param {string} actor
+         * @param {MouseEvent} event
+         * @returns {void}
+         */
         addActorFilter(actor, event) {
             if (!actor) {
                 return;
@@ -315,12 +376,15 @@
             this.form.query = this.form.query ? `${this.form.query} ${token}` : token;
             this.search();
         },
+        /** @returns {Promise<void>} */
         async init() {
+            this.searchHistory = searchHistory.history;
             if (window.ENV?.query) {
                 this.form.query = window.ENV?.query ?? "";
                 await this.search();
             }
         },
+        /** @returns {string[] | null} Playable URLs, if available. */
         get playableURLs() {
             const playable = this.visibleResults.some((item) => item.file_path.startsWith("/mnt/box/files"));
             if (!playable) {
@@ -332,6 +396,7 @@
                 .sort(() => Math.random() - 0.5);
             return urls;
         },
+        /** @returns {string | null} The playlist trigger URL, if available. */
         get playlistLink() {
             const urls = this.playableURLs;
             if (!urls) {
@@ -341,6 +406,7 @@
             return `alfred://runtrigger/dev.abdus.alfred.media/mpv/?argument=${playlistArg}`;
         },
 
+        /** @returns {string | null} The grid trigger URL, if available. */
         get gridLink() {
             const urls = this.playableURLs;
             if (!urls) {
@@ -350,7 +416,10 @@
             return `alfred://runtrigger/dev.abdus.alfred.media/gridvid/?argument=${playlistArg}`;
         },
 
+        /** @returns {Promise<void>} */
         async search() {
+            this.errorMessage = "";
+            const query = this.form.query.trim();
             const response = await fetch("/search", {
                 method: "POST",
                 headers: { "Content-Type": "application/json" },
@@ -362,6 +431,9 @@
                 return;
             }
             this.results = await response.json();
+            if (query) {
+                this.searchHistory = searchHistory.add(query);
+            }
             window.app.setTitle(`s·m·u·t - ${this.results.length} results for "${this.form.query}"`);
         },
     });

snot/snot.py

@@ -31,9 +31,10 @@ import click
 import httpx
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s")
+logging.getLogger("httpx").setLevel(logging.WARNING)
 
 HandlerFunc = Callable[["Request"], "Response"]
-client = httpx.Client(base_url="http://klein:18543", timeout=30)
+client = httpx.Client(base_url="http://klein.talpa-carat.ts.net:18543", timeout=30)
 
 
 @dataclasses.dataclass
@@ -551,11 +552,14 @@ def fetch_videos(params: dict[str, str]) -> list[dict[str, Any]]:
     rows: list[dict[str, Any]] = []
     total: int | None = None
     while total is None or len(rows) < total:
+        try:
             response = client.get(
                 "/videos",
                 params={**params, "limit": "1000", "offset": str(len(rows))},
                 headers={"Prefer": "count=exact"} if total is None else None,
             )
+        except httpx.ConnectError as e:
+            raise RuntimeError(f"Cannot reach database at {client.base_url} ({e}). Is the server running and reachable?") from e
         response.raise_for_status()
         page = response.json()
         if total is None:
@@ -744,6 +748,19 @@ def scan(paths: list[Path]):
     scan_videos(video_paths=list(paths))
 
 
+@cli.command()
+def purge() -> None:
+    """Remove box entries not seen in the last 24 hours."""
+    cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(hours=24)
+    response = client.delete(
+        "/videos",
+        params={"category": "eq.box", "last_seen_at": f"lt.{cutoff.isoformat()}"},
+        headers={"Prefer": "return=representation"},
+    )
+    response.raise_for_status()
+    logging.info(f"Purged {len(response.json())} box entries")
+
+
 @cli.command("mark-deletion")
 @click.argument("paths", nargs=-1, type=Path)
 def mark_deletion(paths: list[Path]):

translate_srt.py

@@ -0,0 +1,256 @@
+#!/usr/bin/env -S uv run --script
+# /// script
+# dependencies = ["httpx"]
+# ///
+import argparse
+import functools
+import json
+import logging
+import os
+import re
+import time
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from pathlib import Path
+
+import httpx
+
+RETRY_STATUS_CODES = {429, 500, 502, 503, 504, 524, 529}
+CONTEXT_CUES = 2
+MAX_VALIDATION_ATTEMPTS = 3
+MAX_REQUEST_ATTEMPTS = 5
+# Matches the start of an SRT time line such as "00:01:02,345 --> 00:01:04,000".
+TIMELINE_PATTERN = re.compile(
+    r"""
+    ^\d{2}:\d{2}:\d{2}[,.]\d{3}     # start: HH:MM:SS,mmm (comma or dot before the milliseconds)
+    \s+-->\s+                       # arrow, with whitespace around it
+    \d{2}:\d{2}:\d{2}[,.]\d{3}      # end, same layout
+    """,
+    re.VERBOSE,
+)
+
+PROMPT = """You are a professional subtitle translator. Translate every item in "items" into {language}.
+
+Rules:
+1. Return only one JSON object, no Markdown, no explanation: {{"translations":[{{"index":1,"text":"..."}}]}}
+2. "translations" must contain exactly one entry per item, with the same "index" values. Never merge, split, skip or reorder items.
+3. Translate each item at its own index even if word order differs; never move meaning to a neighbouring index.
+4. Keep translations natural and concise, suitable for subtitles. Keep names, numbers, negation and tone accurate without removing profanity.
+5. Use "context_before" and "context_after" only to resolve references and keep terminology consistent. Do not translate them.
+6. Do not add notes. Keep line breaks only where the source has them.
+
+Input:
+{payload}"""
+
+
+@dataclass
+class Cue:
+    """One SRT cue: 1-based index, the original time line (kept verbatim) and its text."""
+
+    index: int
+    timeline: str
+    text: str
+
+
+logger = logging.getLogger(__name__)
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Translate an SRT file with an OpenRouter chat model, keeping cues 1:1.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("subtitle", type=Path, help="Input SRT file")
+    parser.add_argument("--translate", "-t", required=True, metavar="LANG", help="Target language, e.g. Turkish")
+    parser.add_argument("--model", "-m", default="google/gemini-3.5-flash-lite", help="OpenRouter chat model id")
+    parser.add_argument(
+        "--api-key",
+        default=os.getenv("OPENROUTER_API_KEY"),
+        help="OpenRouter API key (default: $OPENROUTER_API_KEY%(default).0s)",  # %(default).0s keeps the key out of --help
+    )
+    parser.add_argument("--output", "-o", type=Path, help="Output SRT path (default: <name>.<lang>.srt)")
+    parser.add_argument("--batch-size", type=int, default=20, help="Cues per request")
+    parser.add_argument("--workers", type=int, default=8, help="Parallel requests")
+    return parser.parse_args()
+
+
+def parse_srt(content: str) -> list[Cue]:
+    """Parse SRT text into cues, tolerating CRLF line endings and a missing index line.
+
+    Cues are renumbered from 1; the time line is kept as written. Raises ValueError on a malformed block.
+    """
+    cues = []
+    # \n\s*\n = a blank line (possibly containing spaces) separates SRT blocks
+    for block in re.split(r"\n\s*\n", content.replace("\r\n", "\n").replace("\r", "\n").strip()):
+        lines = block.split("\n")
+        timeline_position = next((position for position, line in enumerate(lines[:3]) if TIMELINE_PATTERN.match(line.strip())), None)
+        if timeline_position is None:
+            raise ValueError(f"Invalid SRT block: {block[:80]!r}")
+        text = "\n".join(lines[timeline_position + 1 :]).strip()
+        cues.append(Cue(index=len(cues) + 1, timeline=lines[timeline_position].strip(), text=text))
+    if not cues:
+        raise ValueError("SRT contains no cues")
+    return cues
+
+
+def retry(
+    attempts: int = MAX_REQUEST_ATTEMPTS,
+    exceptions: tuple[type[Exception], ...] = (httpx.TransportError, httpx.HTTPStatusError),
+) -> Callable:
+    """Decorator factory: call the function up to `attempts` times, sleeping 2**attempt seconds between tries.
+
+    Only `exceptions` trigger a retry; the last one is re-raised when attempts run out.
+    """
+
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            for attempt in range(1, attempts + 1):
+                try:
+                    return func(*args, **kwargs)
+                except exceptions as error:
+                    if attempt == attempts:
+                        raise
+                    logger.warning("%s failed (%s), retry %d/%d", func.__name__, error, attempt, attempts)
+                    time.sleep(2**attempt)
+
+        return wrapper
+
+    return decorator
+
+
+@retry()
+def post_request(client: httpx.Client, payload: dict) -> httpx.Response:
+    """POST to the chat completions endpoint. Retryable statuses (429/5xx) raise so `@retry` repeats them;
+    other statuses are returned for the caller to check."""
+    response = client.post("/chat/completions", json=payload)
+    if response.status_code in RETRY_STATUS_CODES:
+        response.raise_for_status()
+    return response
+
+
+def parse_translations(response_text: str, expected: int) -> list[str]:
+    """Extract the {"translations": [{"index", "text"}]} object from a model reply.
+
+    Takes everything from the first "{" to the last "}" so Markdown fences around the JSON are ignored.
+    Returns the texts ordered by index. Raises ValueError unless there is exactly one non-empty entry per index.
+    """
+    start, end = response_text.find("{"), response_text.rfind("}")
+    if start < 0 or end <= start:
+        raise ValueError("no JSON object in response")
+    items = json.loads(response_text[start : end + 1]).get("translations")
+    if not isinstance(items, list) or len(items) != expected:
+        raise ValueError(f"expected {expected} translations, got {len(items) if isinstance(items, list) else 'none'}")
+    result: list[str | None] = [None] * expected
+    for item in items:
+        position = item.get("index")
+        text = str(item.get("text", "")).strip()
+        if not isinstance(position, int) or not 1 <= position <= expected or result[position - 1] is not None:
+            raise ValueError(f"invalid or duplicate index {position!r}")
+        if not text:
+            raise ValueError(f"empty translation at index {position}")
+        result[position - 1] = text
+    return [text for text in result if text is not None]
+
+
+def translate_batch(
+    client: httpx.Client,
+    cues: list[Cue],
+    before: list[Cue],
+    after: list[Cue],
+    language: str,
+    model: str,
+) -> list[str]:
+    """Translate one batch of cues, with `before`/`after` cues sent only as context.
+
+    Invalid replies (wrong count, bad indices, empty text) are retried up to MAX_VALIDATION_ATTEMPTS times with
+    the validation error appended to the prompt. Returns one translation per cue, in order.
+    """
+    payload_text = json.dumps(
+        {
+            "context_before": [cue.text for cue in before],
+            "items": [{"index": position, "text": cue.text} for position, cue in enumerate(cues, start=1)],
+            "context_after": [cue.text for cue in after],
+        },
+        ensure_ascii=False,
+    )
+    prompt = PROMPT.format(language=language, payload=payload_text)
+    last_error = ""
+    for attempt in range(1, MAX_VALIDATION_ATTEMPTS + 1):
+        request_prompt = prompt
+        if last_error:
+            request_prompt += f"\n\nYour previous answer was invalid: {last_error}. Return the complete corrected JSON."
+        response = post_request(
+            client=client,
+            payload={
+                "model": model,
+                "temperature": 0.2,
+                "response_format": {"type": "json_object"},
+                "messages": [{"role": "user", "content": request_prompt}],
+            },
+        )
+        response.raise_for_status()
+        try:
+            content = response.json()["choices"][0]["message"]["content"] or ""
+            return parse_translations(response_text=content, expected=len(cues))
+        except (ValueError, KeyError, IndexError, AttributeError) as error:
+            last_error = str(error)
+            logger.warning("Invalid response (%s), attempt %d/%d", last_error, attempt, MAX_VALIDATION_ATTEMPTS)
+    raise RuntimeError(f"Batch starting at cue {cues[0].index} failed: {last_error}")
+
+
+def build_srt(cues: list[Cue], translations: list[str]) -> str:
+    """Render the translations as SRT using the original indices and time lines."""
+    return "\n".join(f"{cue.index}\n{cue.timeline}\n{text}\n" for cue, text in zip(cues, translations))
+
+
+def main() -> None:
+    """Translate the SRT in batches with surrounding context and write it with the original timing."""
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s")
+    args = parse_args()
+    if not args.api_key:
+        raise SystemExit("Provide --api-key or set OPENROUTER_API_KEY")
+    if args.batch_size < 1:
+        raise SystemExit("--batch-size must be at least 1")
+    if args.workers < 1:
+        raise SystemExit("--workers must be at least 1")
+
+    cues = parse_srt(content=args.subtitle.read_text(encoding="utf-8-sig"))
+    # [^A-Za-z0-9_-]+ = any run of characters outside letters, digits, "_" and "-"; becomes one "_" for the filename
+    slug = re.sub(r"[^A-Za-z0-9_-]+", "_", args.translate).strip("_").lower()
+    output = args.output or args.subtitle.with_name(f"{args.subtitle.stem}.{slug}.srt")
+    translations: list[str] = []
+
+    with httpx.Client(
+        base_url="https://openrouter.ai/api/v1",
+        headers={"Authorization": f"Bearer {args.api_key}"},
+        timeout=300,
+    ) as client:
+        ranges = [(start, min(start + args.batch_size, len(cues))) for start in range(0, len(cues), args.batch_size)]
+
+        def translate_range(bounds: tuple[int, int]) -> list[str]:
+            """Translate cues[start:end] with ±CONTEXT_CUES neighbours as context."""
+            start, end = bounds
+            logger.info("Translating cues %d-%d of %d with %s", start + 1, end, len(cues), args.model)
+            return translate_batch(
+                client=client,
+                cues=cues[start:end],
+                before=cues[max(0, start - CONTEXT_CUES) : start],
+                after=cues[end : end + CONTEXT_CUES],
+                language=args.translate,
+                model=args.model,
+            )
+
+        # map() yields results in input order, so the translations stay aligned with the cues
+        with ThreadPoolExecutor(max_workers=args.workers) as executor:
+            for batch_translations in executor.map(translate_range, ranges):
+                translations.extend(batch_translations)
+
+    output.write_text(build_srt(cues=cues, translations=translations), encoding="utf-8")
+    logger.info("Wrote %s", output)
+
+
+if __name__ == "__main__":
+    main()

video_to_srt.py

@@ -0,0 +1,468 @@
+#!/usr/bin/env -S uv run --script
+# /// script
+# dependencies = ["httpx"]
+# ///
+import argparse
+import base64
+import functools
+import logging
+import os
+import re
+import subprocess
+import tempfile
+import textwrap
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+
+import httpx
+
+RETRY_STATUS_CODES = {429, 500, 502, 503, 504, 524, 529}
+MAX_ATTEMPTS = 5
+MIN_SPLIT_DURATION = 1.0
+MIN_CHUNK_DURATION = 0.1
+DIALOGUE_LINE_LENGTH = 42
+MAX_DIALOGUE_GAP = 1.0
+MAX_DIALOGUE_DURATION = 7.0
+LINE_LENGTH = 42
+MAX_LINES = 2
+# lowercase tokens (en, tr) whose trailing period does not end a sentence
+ABBREVIATIONS = {"mr", "mrs", "ms", "dr", "prof", "e.g", "i.e", "vs", "etc", "doç", "av", "sn", "vb", "bkz"}
+# Finds candidate sentence boundaries: end punctuation, optional closers, then whitespace.
+# Matches in "He left.  Then" -> ".  " and in 'He said "no." Then' -> '." '.
+SENTENCE_END = re.compile(
+    r"""
+    [.!?…]+         # one or more end marks: "." "?!" "..." or the ellipsis character
+    [\"'”’)\]]*     # closing quotes or brackets that belong to the finished sentence
+    \s+             # whitespace after it; one or more, so double spaces work too
+    """,
+    re.VERBOSE,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class Segment:
+    """A timed piece of transcript; times are in seconds, speaker is a diarization index if known."""
+
+    start: float
+    end: float
+    text: str
+    speaker: int | None = None
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Transcribe a video into an SRT file using an OpenRouter speech-to-text model.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("video", type=Path, help="Input video (or audio) file")
+    parser.add_argument("--output", "-o", type=Path, help="Output SRT path (default: <video>.srt)")
+    parser.add_argument("--model", "-m", default="elevenlabs/scribe-v2", help="OpenRouter STT model id (must support diarization unless --no-diarize)")
+    parser.add_argument(
+        "--api-key",
+        default=os.getenv("OPENROUTER_API_KEY"),
+        help="OpenRouter API key (default: $OPENROUTER_API_KEY%(default).0s)",  # %(default).0s keeps the key out of --help
+    )
+    parser.add_argument("--language", "-l", help="ISO-639-1 source language, e.g. en or de (auto-detect if omitted)")
+    parser.add_argument("--chunk-minutes", type=float, default=60.0, help="Audio chunk length in minutes (of sped-up audio)")
+    parser.add_argument(
+        "--speed-up",
+        type=float,
+        default=1.0,
+        help="Speed the audio up (1.0-2.0) before transcribing to reduce cost; timestamps are scaled back and multi-sentence cues are split",
+    )
+    parser.add_argument(
+        "--diarize",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="Label speakers (needs a supporting model, e.g. elevenlabs/scribe-v2); short speaker changes become '- ' dialogue cues",
+    )
+    return parser.parse_args()
+
+
+def run_ffmpeg_extract(video: Path, directory: Path, chunk_seconds: int, speed_up: float) -> list[Path]:
+    """Extract mono 16 kHz mp3 audio into `directory` as chunks of `chunk_seconds` (measured after speed-up).
+
+    Returns the chunk paths in order. `speed_up` > 1 applies a single ffmpeg `atempo` filter.
+    """
+    filter_arguments = ["-filter:a", f"atempo={speed_up}"] if speed_up != 1.0 else []
+    subprocess.run(
+        [
+            "ffmpeg",
+            "-hide_banner",
+            "-loglevel",
+            "error",
+            "-i",
+            str(video),
+            "-vn",
+            "-ac",
+            "1",
+            "-ar",
+            "16000",
+            *filter_arguments,
+            "-c:a",
+            "libmp3lame",
+            "-b:a",
+            "48k",
+            "-f",
+            "segment",
+            "-segment_time",
+            str(chunk_seconds),
+            "-reset_timestamps",
+            "1",
+            str(directory / "chunk_%04d.mp3"),
+        ],
+        check=True,
+    )
+    chunks = sorted(directory.glob("chunk_*.mp3"))
+    if not chunks:
+        raise RuntimeError(f"ffmpeg produced no audio from {video}")
+    return chunks
+
+
+def get_duration(path: Path) -> float:
+    """Return the media duration in seconds via ffprobe, or 0.0 if it is unreadable (e.g. an empty trailing chunk)."""
+    result = subprocess.run(
+        ["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", str(path)],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    try:
+        return max(float(result.stdout.strip()), 0.0)
+    except ValueError:
+        return 0.0
+
+
+def retry(
+    attempts: int = MAX_ATTEMPTS,
+    exceptions: tuple[type[Exception], ...] = (httpx.TransportError, httpx.HTTPStatusError),
+) -> Callable:
+    """Decorator factory: call the function up to `attempts` times, sleeping 2**attempt seconds between tries.
+
+    Only `exceptions` trigger a retry; the last one is re-raised when attempts run out.
+    """
+
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            for attempt in range(1, attempts + 1):
+                try:
+                    return func(*args, **kwargs)
+                except exceptions as error:
+                    if attempt == attempts:
+                        raise
+                    logger.warning("%s failed (%s), retry %d/%d", func.__name__, error, attempt, attempts)
+                    time.sleep(2**attempt)
+
+        return wrapper
+
+    return decorator
+
+
+@retry()
+def post_request(client: httpx.Client, payload: dict) -> httpx.Response:
+    """POST to the transcription endpoint. Retryable statuses (429/5xx) raise so `@retry` repeats them;
+    other statuses, including 400, are returned for the caller to handle."""
+    response = client.post("/audio/transcriptions", json=payload)
+    if response.status_code in RETRY_STATUS_CODES:
+        response.raise_for_status()
+    return response
+
+
+def transcribe_chunk(
+    client: httpx.Client,
+    path: Path,
+    duration: float,
+    model: str,
+    language: str | None,
+    diarize: bool,
+) -> tuple[list[Segment], bool]:
+    """Transcribe one audio chunk. Times in the result are relative to the chunk, in sped-up audio time.
+
+    Returns (segments, has_timestamps). If the model rejects `verbose_json` the call is repeated as plain
+    JSON and a single segment covering the whole chunk is returned with has_timestamps=False.
+    With `diarize`, speaker turns are built from word-level labels, falling back to segment-level labels;
+    a rejected diarize request exits instead of falling back.
+    """
+    payload: dict = {
+        "model": model,
+        "input_audio": {"data": base64.b64encode(path.read_bytes()).decode(), "format": "mp3"},
+        "response_format": "verbose_json",
+        "timestamp_granularities": ["segment", "word"] if diarize else ["segment"],
+    }
+    if language:
+        payload["language"] = language
+    if diarize:
+        payload["diarize"] = True
+    response = post_request(client=client, payload=payload)
+    if response.status_code == 400 and diarize:
+        raise SystemExit(f"Model rejected diarization: {response.text[:300]}")
+    if response.status_code == 400:
+        logger.warning("Model rejected verbose_json (%s); falling back to a single cue per chunk", response.text[:200])
+        payload.pop("response_format")
+        payload.pop("timestamp_granularities")
+        response = post_request(client=client, payload=payload)
+    response.raise_for_status()
+    data = response.json()
+    if not data.get("segments") and not data.get("words") and not str(data.get("text", "")).strip():
+        raise RuntimeError(f"Empty transcription for {path.name} ({path.stat().st_size / 1e6:.1f} MB); response: {str(data)[:300]}")
+    if diarize:
+        turns = turns_from_words(words=data.get("words") or [])
+        if turns:
+            return turns, True
+    segments = [
+        Segment(
+            start=float(segment["start"]),
+            end=float(segment["end"]),
+            text=str(segment["text"]).strip(),
+            speaker=segment.get("speaker") if diarize else None,
+        )
+        for segment in data.get("segments") or []
+        if str(segment.get("text", "")).strip()
+    ]
+    if diarize and not any(segment.speaker is not None for segment in segments):
+        logger.warning("No speaker labels in the response; the model may not support diarization")
+    if segments:
+        return segments, True
+    text = str(data.get("text", "")).strip()
+    return ([Segment(start=0.0, end=duration, text=text)] if text else []), False
+
+
+def turns_from_words(words: list[dict]) -> list[Segment]:
+    """Group consecutive words with the same `speaker` into one Segment per speaker turn.
+
+    Words without a speaker and `audio_event` entries (e.g. laughter) are skipped.
+    """
+    turns: list[Segment] = []
+    for word in words:
+        text = str(word.get("word", "")).strip()
+        speaker = word.get("speaker")
+        if not text or speaker is None or word.get("type") == "audio_event":
+            continue
+        if turns and turns[-1].speaker == speaker:
+            turns[-1].end = float(word["end"])
+            turns[-1].text += f" {text}"
+        else:
+            turns.append(Segment(start=float(word["start"]), end=float(word["end"]), text=text, speaker=speaker))
+    for turn in turns:
+        # Words are joined with spaces, so punctuation tokens end up detached ("Hi ,"); glue them back:
+        # \s+ = the stray whitespace, ([,.!?;:…]) = the punctuation mark, kept via \1.
+        turn.text = re.sub(r"\s+([,.!?;:…])", r"\1", turn.text)
+    return turns
+
+
+def group_dialogue(cues: list[Segment]) -> list[tuple[float, float, str]]:
+    """Turn segments into SRT cues, pairing adjacent turns by different speakers into '- a' / '- b' cues.
+
+    A pair is formed only if each line is at most DIALOGUE_LINE_LENGTH characters, the gap between them is at
+    most MAX_DIALOGUE_GAP seconds and the pair spans at most MAX_DIALOGUE_DURATION seconds.
+    Everything else becomes its own cue. Returns (start, end, text) tuples.
+    """
+    grouped = []
+    index = 0
+    while index < len(cues):
+        cue = cues[index]
+        following = cues[index + 1] if index + 1 < len(cues) else None
+        if (
+            cue.speaker is not None
+            and following is not None
+            and following.speaker not in (None, cue.speaker)
+            and max(len(cue.text), len(following.text)) <= DIALOGUE_LINE_LENGTH
+            and following.start - cue.end <= MAX_DIALOGUE_GAP
+            and following.end - cue.start <= MAX_DIALOGUE_DURATION
+        ):
+            grouped.append((cue.start, following.end, f"- {cue.text}\n- {following.text}"))
+            index += 2
+        else:
+            grouped.append((cue.start, cue.end, cue.text))
+            index += 1
+    return grouped
+
+
+def split_sentences(text: str) -> list[str]:
+    """Split text into sentences at SENTENCE_END matches.
+
+    A boundary is kept only if the next character starts a sentence (uppercase, digit or opening quote/bracket)
+    and the word before a period is not a known abbreviation or a single-letter initial.
+    Decimals such as "3.5" never match because no whitespace follows the period.
+    """
+    sentences = []
+    start = 0
+    for match in SENTENCE_END.finditer(text):
+        following = text[match.end() : match.end() + 1]
+        if not following or not (following.isupper() or following.isdigit() or following in "\"'“‘(["):
+            continue
+        punctuation = match.group().strip()
+        # (\S+)$ = the last whitespace-delimited word before the punctuation, e.g. "Dr" in "... Dr."
+        word = re.search(r"(\S+)$", text[start : match.start()])
+        token = word.group(1).lower().lstrip("\"'“‘([") if word else ""
+        if punctuation.startswith(".") and (token in ABBREVIATIONS or (len(token) == 1 and token.isalpha())):
+            continue
+        sentences.append(text[start : match.start() + len(match.group().rstrip())].strip())
+        start = match.end()
+    sentences.append(text[start:].strip())
+    return [sentence for sentence in sentences if sentence]
+
+
+def split_cue(start: float, end: float, text: str) -> list[tuple[float, float, str]]:
+    """Split a cue into one cue per sentence, dividing [start, end] in proportion to character length.
+
+    Pieces shorter than MIN_SPLIT_DURATION are merged into their neighbour. The pieces are contiguous.
+    Returns (start, end, text) tuples; a single-sentence cue is returned unchanged.
+    """
+    sentences = split_sentences(text=text)
+    if len(sentences) < 2:
+        return [(start, end, text)]
+    total = sum(len(sentence) for sentence in sentences)
+    pieces: list[list] = []
+    cursor = start
+    for sentence in sentences:
+        piece_end = cursor + (end - start) * len(sentence) / total
+        pieces.append([cursor, piece_end, sentence])
+        cursor = piece_end
+    pieces[-1][1] = end
+    merged: list[list] = []
+    for piece in pieces:
+        if merged and (piece[1] - piece[0] < MIN_SPLIT_DURATION or merged[-1][1] - merged[-1][0] < MIN_SPLIT_DURATION):
+            merged[-1][1] = piece[1]
+            merged[-1][2] += f" {piece[2]}"
+        else:
+            merged.append(piece)
+    return [(piece_start, piece_end, piece_text) for piece_start, piece_end, piece_text in merged]
+
+
+def wrap_text(text: str) -> str:
+    """Wrap text into balanced lines of at most LINE_LENGTH characters (e.g. 60 chars -> 2 lines of 30)."""
+    if len(text) <= LINE_LENGTH:
+        return text
+    lines_needed = -(-len(text) // LINE_LENGTH)  # ceiling division
+    # widen the target width until the words fit in the needed number of lines
+    for width in range(-(-len(text) // lines_needed), LINE_LENGTH + 1):
+        lines = textwrap.wrap(text, width=width)
+        if len(lines) <= lines_needed:
+            break
+    return "\n".join(lines)
+
+
+def split_long(start: float, end: float, text: str) -> list[tuple[float, float, str]]:
+    """Split a cue longer than MAX_LINES lines into several cues, dividing [start, end] by character length.
+
+    Splits near the middle, preferring a position right after a comma, semicolon, colon or sentence end
+    (within the middle half of the text), else any space. Repeats until every piece fits.
+    """
+    if len(text) <= LINE_LENGTH * MAX_LINES:
+        return [(start, end, text)]
+    middle = len(text) / 2
+    spaces = [match.start() for match in re.finditer(" ", text)]
+    if not spaces:
+        return [(start, end, text)]
+    # a space preceded by , ; : . ! ? is a clause break
+    clause_breaks = [position for position in spaces if text[position - 1] in ",;:.!?" and abs(position - middle) <= len(text) / 4]
+    position = min(clause_breaks or spaces, key=lambda candidate: abs(candidate - middle))
+    left, right = text[:position].strip(), text[position:].strip()
+    boundary = start + (end - start) * len(left) / (len(left) + len(right))
+    return split_long(start=start, end=boundary, text=left) + split_long(start=boundary, end=end, text=right)
+
+
+def format_cues(cues: list[tuple[float, float, str]]) -> list[tuple[float, float, str]]:
+    """Split single-speaker cues that are too long and wrap their text into lines; dialogue cues pass through."""
+    formatted = []
+    for start, end, text in cues:
+        if text.startswith("- "):
+            formatted.append((start, end, text))
+        else:
+            formatted.extend(
+                (piece_start, piece_end, wrap_text(text=piece_text)) for piece_start, piece_end, piece_text in split_long(start=start, end=end, text=text)
+            )
+    return formatted
+
+
+def format_timestamp(seconds: float) -> str:
+    """Format seconds as an SRT timestamp, HH:MM:SS,mmm."""
+    total_milliseconds = round(seconds * 1000)
+    hours, remainder = divmod(total_milliseconds, 3_600_000)
+    minutes, remainder = divmod(remainder, 60_000)
+    secs, milliseconds = divmod(remainder, 1000)
+    return f"{hours:02d}:{minutes:02d}:{secs:02d},{milliseconds:03d}"
+
+
+def build_srt(cues: list[tuple[float, float, str]]) -> str:
+    """Render (start, end, text) cues as SRT, keeping them non-overlapping with a minimum length of 0.3 s."""
+    blocks = []
+    previous_end = 0.0
+    for index, (start, end, text) in enumerate(cues, start=1):
+        start = max(start, previous_end)
+        end = max(end, start + 0.3)
+        previous_end = end
+        blocks.append(f"{index}\n{format_timestamp(start)} --> {format_timestamp(end)}\n{text}\n")
+    return "\n".join(blocks)
+
+
+def main() -> None:
+    """Extract audio, transcribe it chunk by chunk, rescale timestamps by the speed-up and write the SRT."""
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s")
+    args = parse_args()
+    if not args.api_key:
+        raise SystemExit("Provide --api-key or set OPENROUTER_API_KEY")
+    if not args.video.is_file():
+        raise SystemExit(f"Input not found: {args.video}")
+    if not 1.0 <= args.speed_up <= 2.0:
+        raise SystemExit("--speed-up must be between 1.0 and 2.0")
+
+    output = args.output or args.video.with_suffix(".srt")
+    cues: list[Segment] = []
+
+    with (
+        tempfile.TemporaryDirectory() as temp_directory,
+        httpx.Client(
+            base_url="https://openrouter.ai/api/v1",
+            headers={"Authorization": f"Bearer {args.api_key}"},
+            timeout=300,
+        ) as client,
+    ):
+        chunks = run_ffmpeg_extract(
+            video=args.video,
+            directory=Path(temp_directory),
+            chunk_seconds=int(args.chunk_minutes * 60),
+            speed_up=args.speed_up,
+        )
+        logger.info("Extracted %d audio chunk(s)", len(chunks))
+        offset = 0.0
+        for index, chunk in enumerate(chunks, start=1):
+            duration = get_duration(path=chunk)
+            if duration < MIN_CHUNK_DURATION:
+                logger.info("Skipping empty chunk %d/%d", index, len(chunks))
+                continue
+            logger.info("Transcribing chunk %d/%d with %s", index, len(chunks), args.model)
+            segments, has_timestamps = transcribe_chunk(
+                client=client,
+                path=chunk,
+                duration=duration,
+                model=args.model,
+                language=args.language,
+                diarize=args.diarize,
+            )
+            for segment in segments:
+                start = segment.start * args.speed_up + offset
+                end = segment.end * args.speed_up + offset
+                if args.speed_up > 1.0 or args.diarize or not has_timestamps:
+                    pieces = split_cue(start=start, end=end, text=segment.text)
+                else:
+                    pieces = [(start, end, segment.text)]
+                cues.extend(Segment(start=piece[0], end=piece[1], text=piece[2], speaker=segment.speaker) for piece in pieces)
+            offset += duration * args.speed_up
+
+    if not cues:
+        raise SystemExit("Transcription returned no text")
+    srt_cues = format_cues(cues=group_dialogue(cues=cues))
+    output.write_text(build_srt(cues=srt_cues), encoding="utf-8")
+    logger.info("Wrote %d cues to %s", len(srt_cues), output)
+
+
+if __name__ == "__main__":
+    main()

youtube.html

@@ -1,365 +0,0 @@
-<!doctype html>
-<html lang="en">
-    <head>
-        <meta charset="UTF-8" />
-        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-        <title>YouTube Video Summarizer</title>
-        <style>
-            :root {
-                color-scheme: light dark;
-                --bg: #fff;
-                --bg-alt: #f5f5f5;
-                --text: #000;
-                --text-muted: #666;
-                --border: #ddd;
-                --accent: #4285f4;
-            }
-
-            @media (prefers-color-scheme: dark) {
-                :root {
-                    --bg: #1a1a1a;
-                    --bg-alt: #2a2a2a;
-                    --text: #fff;
-                    --text-muted: #aaa;
-                    --border: #444;
-                    --accent: #5a9fff;
-                }
-            }
-
-            *,
-            *::before,
-            *::after {
-                box-sizing: border-box;
-            }
-
-            body {
-                font-family: "JetBrains Mono", monospace;
-                line-height: 1.6;
-                font-size: 16px;
-                padding: 2rem;
-                max-width: 1200px;
-                margin: 0 auto;
-                background: var(--bg);
-                color: var(--text);
-                transition:
-                    background 0.2s,
-                    color 0.2s;
-            }
-
-            .container {
-                padding: 0;
-            }
-
-            .form-group {
-                display: none;
-            }
-
-            .loading {
-                text-align: center;
-                color: var(--text-muted);
-                margin: 2rem 0;
-            }
-
-            .spinner {
-                display: inline-block;
-                width: 1rem;
-                height: 1rem;
-                border: 3px solid var(--bg-alt);
-                border-top: 3px solid var(--accent);
-                border-radius: 50%;
-                animation: spin 1s linear infinite;
-                margin-right: 0.5rem;
-            }
-
-            @keyframes spin {
-                0% {
-                    transform: rotate(0deg);
-                }
-                100% {
-                    transform: rotate(360deg);
-                }
-            }
-
-            #summary {
-                margin-top: 0;
-                padding: 0;
-            }
-
-            #summary :is(ul, ol) {
-                margin: 1rem 0;
-                padding-left: 1.5rem;
-            }
-
-            #summary :is(ul, ol) :is(ul, ol) {
-                padding-left: 1.5rem;
-                margin: 0.5rem 0;
-            }
-
-            #summary li {
-                margin-bottom: 0.5rem;
-            }
-
-            #summary blockquote {
-                border-left: 3px solid var(--border);
-                padding-left: 1rem;
-                margin: 1rem 0;
-                color: var(--text-muted);
-                font-style: italic;
-            }
-
-            #summary h2,
-            #summary h3 {
-                margin: 1.5rem 0 0.5rem 0;
-            }
-
-            .error {
-                background: #c33;
-                color: #fff;
-                padding: 1rem;
-                border-radius: 4px;
-                margin: 1rem 0;
-            }
-        </style>
-    </head>
-    <body>
-        <div class="container">
-            <div class="form-group">
-                <input type="text" id="videoUrl" placeholder="https://www.youtube.com/watch?v=..." />
-                <textarea id="video" placeholder="Video transcript"></textarea>
-            </div>
-
-            <button id="summarizeBtn" style="display: none"></button>
-
-            <div id="loading" class="loading" style="display: none">
-                <div class="spinner"></div>
-                Generating summary...
-            </div>
-
-            <div id="error" class="error" style="display: none"></div>
-
-            <div id="summary"></div>
-        </div>
-
-        <script src="https://cdn.jsdelivr.net/npm/markdown-it@14.1.0/dist/markdown-it.min.js"></script>
-        <script>
-            const videoUrlInput = document.getElementById("videoUrl");
-            const textInput = document.getElementById("text");
-            const summarizeBtn = document.getElementById("summarizeBtn");
-            const loadingDiv = document.getElementById("loading");
-            const errorDiv = document.getElementById("error");
-            const summaryDiv = document.getElementById("summary");
-
-            // Pre-fill with YouTube URL if available and auto-submit
-            if (window.env?.YOUTUBE_URL || window.env?.YOUTUBE_TRANSCRIPT) {
-                if (window.env?.YOUTUBE_URL) videoUrlInput.value = window.env.YOUTUBE_URL;
-                if (document.readyState === "loading") {
-                    document.addEventListener("DOMContentLoaded", () => summarizeBtn.click());
-                } else {
-                    summarizeBtn.click();
-                }
-            }
-
-            summarizeBtn.addEventListener("click", async () => {
-                const url = videoUrlInput.value.trim();
-                const transcript = window.env?.YOUTUBE_TRANSCRIPT;
-
-                if (!transcript && !url) {
-                    showError("Please enter a YouTube URL");
-                    return;
-                }
-
-                if (!transcript) {
-                    // Validate YouTube URL
-                    const youtubeUrlPattern = /^https?:\/\/(www\.)?youtube\.com\/watch\?v=.+$|^https?:\/\/youtu\.be\/.+$/;
-                    if (!youtubeUrlPattern.test(url)) {
-                        showError("Please enter a valid YouTube URL");
-                        return;
-                    }
-                }
-
-                await summarize(url, transcript);
-            });
-
-            function showError(message) {
-                errorDiv.textContent = message;
-                errorDiv.style.display = "block";
-            }
-
-            function hideError() {
-                errorDiv.style.display = "none";
-                errorDiv.textContent = "";
-            }
-
-            async function summarize(url, transcript) {
-                hideError();
-                summaryDiv.innerHTML = "";
-                loadingDiv.style.display = "block";
-                summarizeBtn.disabled = true;
-
-                const prompt = `
-                    Give me keypoints of this video transcript, with enough context, also with quotes from the content.
-                    Only the summary please.
-                    Skip promotions.
-                    Do not censor profanity.
-                    Formatted in valid markdown without any wrappers or code blocks.
-                    `;
-
-                let fullResponse = "";
-                const abortController = new AbortController();
-
-                try {
-                    await chatWithGemini({
-                        prompt: prompt,
-                        messages: [{ role: "user", content: transcript ?? url }],
-                        model: "gemini-3-flash-preview",
-                        onContent: (text) => {
-                            fullResponse += text;
-                            summaryDiv.innerHTML = markdownToHtml(fullResponse);
-                        },
-                        onError: (err) => {
-                            showError(`Error: ${err.message}`);
-                            loadingDiv.style.display = "none";
-                            summarizeBtn.disabled = false;
-                        },
-                        signal: abortController.signal,
-                    });
-
-                    loadingDiv.style.display = "none";
-                } catch (err) {
-                    showError(`Error: ${err.message}`);
-                    loadingDiv.style.display = "none";
-                } finally {
-                    summarizeBtn.disabled = false;
-                }
-            }
-
-            const md = markdownit({ html: false });
-            function markdownToHtml(markdown) {
-                return md.render(markdown);
-            }
-
-            async function chatWithGemini({ prompt, onContent, onError, messages, signal, apiKey = window.env.GEMINI_API_KEY, model }) {
-                const contents = messages.map((msg, i, arr) => {
-                    const role = msg.role === "assistant" ? "model" : "user";
-                    let parts = [{ text: msg.content }];
-
-                    // Add YouTube URLs as file data for the final user message
-                    const isLastMessage = i === arr.length - 1;
-                    if (isLastMessage && role === "user") {
-                        const youtubeUrlPattern = /https?:\/\/(www\.)?youtube\.com\/watch\?v=[\w-]+|https?:\/\/youtu\.be\/[\w-]+/g;
-                        const youtubeUrls = msg.content.match(youtubeUrlPattern) || [];
-                        const fileParts = youtubeUrls.map((url) => ({ file_data: { file_uri: url } }));
-                        parts = [...parts, ...fileParts];
-                    }
-
-                    return {
-                        role,
-                        parts,
-                    };
-                });
-
-                try {
-                    const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${model}:streamGenerateContent?key=${apiKey}`, {
-                        method: "POST",
-                        headers: { "Content-Type": "application/json" },
-                        body: JSON.stringify({
-                            ...(prompt ? { system_instruction: { parts: [{ text: prompt }] } } : {}),
-                            contents,
-                            generationConfig: {
-                                thinkingConfig: { thinkingLevel: "low" },
-                            },
-                        }),
-                        signal,
-                    });
-
-                    if (!response.ok) {
-                        const text = await response.text();
-                        await onError?.(new Error(`Gemini API Error ${response.status}: ${text}`));
-                        return;
-                    }
-
-                    const reader = response.body.getReader();
-                    const decoder = new TextDecoder();
-                    let buffer = "";
-
-                    while (true) {
-                        const { done, value } = await reader.read();
-                        if (done) break;
-
-                        buffer += decoder.decode(value, { stream: true });
-
-                        let pos = 0;
-                        while (pos < buffer.length) {
-                            // Find start of JSON object
-                            const start = buffer.indexOf("{", pos);
-                            if (start === -1) break;
-
-                            // Find matching closing brace
-                            let depth = 0;
-                            let inString = false;
-                            let escaped = false;
-                            let end = -1;
-
-                            for (let i = start; i < buffer.length; i++) {
-                                const ch = buffer[i];
-
-                                if (escaped) {
-                                    escaped = false;
-                                    continue;
-                                }
-
-                                if (ch === "\\") {
-                                    escaped = true;
-                                    continue;
-                                }
-
-                                if (ch === '"') {
-                                    inString = !inString;
-                                    continue;
-                                }
-
-                                if (!inString) {
-                                    if (ch === "{") depth++;
-                                    else if (ch === "}") {
-                                        depth--;
-                                        if (depth === 0) {
-                                            end = i;
-                                            break;
-                                        }
-                                    }
-                                }
-                            }
-
-                            if (end === -1) {
-                                // Incomplete JSON, keep it in buffer
-                                buffer = buffer.slice(start);
-                                break;
-                            }
-
-                            // Parse and process JSON
-                            try {
-                                const json = JSON.parse(buffer.slice(start, end + 1));
-                                const parts = json.candidates?.[0]?.content?.parts || [];
-                                const text = parts
-                                    .filter((p) => p.text)
-                                    .map((p) => p.text)
-                                    .join("");
-
-                                if (text) await onContent(text);
-                            } catch (e) {
-                                // Ignore parse errors
-                            }
-
-                            pos = end + 1;
-                        }
-
-                        // Keep only unparsed remainder
-                        buffer = buffer.slice(pos);
-                    }
-                } catch (err) {
-                    if (err.name !== "AbortError") await onError?.(err);
-                }
-            }
-        </script>
-    </body>
-</html>

youtube_to_srt.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env -S uv run --script
+# /// script
+# dependencies = ["yt-dlp"]
+# ///
+import argparse
+import logging
+import re
+import subprocess
+import tempfile
+from pathlib import Path
+
+import yt_dlp
+
+VIDEO_TO_SRT = Path(__file__).resolve().with_name("video_to_srt.py")
+MAX_FILENAME_BYTES = 240
+
+logger = logging.getLogger(__name__)
+
+
+def parse_args() -> tuple[argparse.Namespace, list[str]]:
+    """Parse command-line arguments; unknown options are returned to be forwarded to video_to_srt.py."""
+    parser = argparse.ArgumentParser(
+        description="Download a YouTube video's audio and transcribe it with video_to_srt.py. "
+        "Options not listed here (--model, --speed-up, --language, --chunk-minutes, --api-key, --no-diarize, ...) are passed through.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("video", help="YouTube URL or video id")
+    parser.add_argument("--output-dir", "-d", type=Path, default=Path.cwd(), help="Directory for the SRT file")
+    return parser.parse_known_args()
+
+
+def sanitize(text: str) -> str:
+    """Make text safe for a filename: path separators and other reserved characters become '-', whitespace collapses."""
+    # [\\/:*?"<>|\x00-\x1f] = characters not allowed on common filesystems, plus control characters
+    text = re.sub(r'[\\/:*?"<>|\x00-\x1f]', "-", text)
+    return re.sub(r"\s+", " ", text).strip()
+
+
+def format_date(upload_date: str | None) -> str:
+    """Convert yt-dlp's YYYYMMDD upload date to yyyy-mm-dd ('unknown-date' if missing)."""
+    if not upload_date or len(upload_date) != 8:
+        return "unknown-date"
+    return f"{upload_date[:4]}-{upload_date[4:6]}-{upload_date[6:]}"
+
+
+def build_filename(info: dict) -> str:
+    """Build '<author> -- @<channel> -- <title> -- <yyyy-mm-dd>.srt' from yt-dlp metadata.
+
+    Author is the channel's display name, channel is its @handle (falls back to the channel id).
+    The title is shortened so the whole name stays under MAX_FILENAME_BYTES.
+    """
+    author = sanitize(info.get("uploader") or info.get("channel") or "unknown")
+    handle = (info.get("uploader_id") or info.get("channel_id") or "unknown").lstrip("@")
+    suffix = f" -- @{sanitize(handle)} -- "
+    date = f" -- {format_date(info.get('upload_date'))}.srt"
+    title = sanitize(info.get("title") or info["id"])
+    budget = MAX_FILENAME_BYTES - len(f"{author}{suffix}{date}".encode())
+    while len(title.encode()) > budget:
+        title = title[:-1]
+    return f"{author}{suffix}{title.strip()}{date}"
+
+
+def download_audio(url: str, directory: Path) -> tuple[Path, dict]:
+    """Download the best audio-only stream into `directory`. Returns (file path, yt-dlp metadata)."""
+    options = {"format": "bestaudio/best", "outtmpl": str(directory / "%(id)s.%(ext)s"), "noplaylist": True, "quiet": True}
+    with yt_dlp.YoutubeDL(options) as downloader:
+        info = downloader.extract_info(url, download=True)
+        return Path(downloader.prepare_filename(info)), info
+
+
+def main() -> None:
+    """Download the audio, transcribe it with video_to_srt.py and save the SRT under the metadata-based name."""
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s")
+    args, forwarded = parse_args()
+    url = args.video if "/" in args.video else f"https://www.youtube.com/watch?v={args.video}"
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+
+    with tempfile.TemporaryDirectory() as temp_directory:
+        audio, info = download_audio(url=url, directory=Path(temp_directory))
+        output = args.output_dir / build_filename(info=info)
+        logger.info("Downloaded %s, writing %s", audio.name, output)
+        subprocess.run([str(VIDEO_TO_SRT), str(audio), "--output", str(output), *forwarded], check=True)
+
+
+if __name__ == "__main__":
+    main()