diff --git a/anki.creds.json b/anki.creds.json new file mode 100644 index 0000000..1ae2081 --- /dev/null +++ b/anki.creds.json @@ -0,0 +1 @@ +{"username": "sssmmt@gmail.com", "password": "2126484"} \ No newline at end of file diff --git a/berlin_immigration.py b/berlin_immigration.py index 53afe37..84ecb0e 100644 --- a/berlin_immigration.py +++ b/berlin_immigration.py @@ -1,12 +1,12 @@ import argparse import asyncio import contextlib +import dataclasses import datetime import logging import random import re import subprocess -import time import urllib.parse from pathlib import Path @@ -39,7 +39,16 @@ dump_dir = Path('~/Desktop').expanduser() / 'immigration' dump_dir.mkdir(parents=True, exist_ok=True) -async def check_slots(page: Page, retry_count: int = 1) -> list[datetime.datetime]: +@dataclasses.dataclass +class ResidencyApplication: + citizenship_country: str + num_applicants: int + is_living_with_family: bool + residency_category: str + residency_purpose: str + + +async def check_slots(page: Page, values: ResidencyApplication, retry_count: int = 1) -> list[datetime.datetime]: await page.add_init_script('''Object.defineProperty(navigator, 'webdriver', { get: () => false })''') window_id = random.randint(1000, 9999) req_id = random.randint(0, 999) @@ -47,45 +56,37 @@ async def check_slots(page: Page, retry_count: int = 1) -> list[datetime.datetim f'https://otv.verwalt-berlin.de/ams/TerminBuchen/wizardng?dswid={window_id}&dsrid={req_id}', wait_until='networkidle', ) + + async with page.expect_navigation(): + await page.click('.langBlock [name="txtEn"] a') + await page.check('[name="gelesen"]') - async with page.expect_navigation(url=re.compile('st=2')): + + async with page.expect_navigation(url=re.compile('st=2'), wait_until='networkidle'): await page.click('[name="applicationForm:managedForm:proceed"]') - await page.get_by_role("combobox", name="Staatsangehörigkeit *").select_option(label="Türkei") + await page.get_by_role("combobox", name="Citizenship *").select_option(label=values.citizenship_country) await asyncio.sleep(0.3) - await page.get_by_role( - "combobox", - name="Anzahl der Personen, die einen Aufenthaltstitel beantragen (auch ausländische Ehepartner und Kinder) *", - ).select_option(label="eine Person") + # how many applicants + await page.select_option('[name="personenAnzahl_normal"]', value=str(values.num_applicants)) await asyncio.sleep(0.3) - await page.get_by_role( - "combobox", name="Leben Sie in Berlin zusammen mit einem Familienangehörigen (z.B. Ehepartner, Kind) *" - ).select_option(label="nein") + # living with family + await page.select_option('[name="lebnBrMitFmly"]', value='1' if values.is_living_with_family else '2') await asyncio.sleep(0.3) - await page.get_by_text("Aufenthaltstitel - beantragen").click() - await page.locator("label").filter(has_text="Erwerbstätigkeit").click() - await page.get_by_text("Blaue Karte EU (§ 18b Abs. 2)").click() + await page.get_by_text("Apply for a residence title").click() + await page.locator("label").filter(has_text=values.residency_category).click() + await page.get_by_text(values.residency_purpose).click() stop_at = datetime.datetime.now() + datetime.timedelta(minutes=28) - async def go_forward(): - async with page.expect_navigation(url=re.compile(r'st='), wait_until='networkidle', timeout=60_000): - await page.get_by_role("button", name="Weiter").click() - if 'st=2' in page.url: - error_message = await page.inner_text('.errorMessage') - if 'keine Termine frei' in error_message: - return False - raise Error(error_message) - return 'st=3' in page.url - while 'st=2' in page.url: retry_count -= 1 async with page.expect_navigation(url=re.compile(r'st='), wait_until='networkidle', timeout=60_000): - await page.get_by_role("button", name="Weiter").click() + await page.get_by_role("button", name="Next").click() if datetime.datetime.now() > stop_at: raise TimeoutError('could not find a slot') @@ -134,7 +135,14 @@ async def main(): args = parse_args() async with launch_browser(headless=args.headless) as page: try: - slots = await check_slots(page, retry_count=args.retry_count) + values = ResidencyApplication( + citizenship_country='India', + num_applicants=1, + is_living_with_family=False, + residency_category='Educational purposes', + residency_purpose='Residence permit for the purpose of studying (sect. 16b)', + ) + slots = await check_slots(page, values=values, retry_count=args.retry_count) if not slots: print('no slots yet') return @@ -146,4 +154,3 @@ async def main(): if __name__ == '__main__': asyncio.run(main()) logging.basicConfig(level=logging.INFO) - main() diff --git a/clean_vids.py b/clean_vids.py new file mode 100644 index 0000000..86117b2 --- /dev/null +++ b/clean_vids.py @@ -0,0 +1,782 @@ +import contextlib +import hashlib +import json +import logging +import re +import sqlite3 +import subprocess +import typing +from collections import Counter, defaultdict +from pathlib import Path + +import paramiko +import psycopg +import psycopg.rows + +import ffmpeg + +VIDS_DIR = Path(r"/Volumes/FIVER/_ingress/") +schema = """ +create table if not exists videos( + file_path text not null primary key, + size_bytes int not null, + hash_partial text not null, + ffprobe json not null +); + +create table if not exists scan_state( + file_path text not null primary key, + stderr text +); +""" + + +def find_vids(root: Path): + extensions = [".mkv", ".mp4"] + for ext in extensions: + for f in root.rglob(f"*{ext}"): + yield f + + +def hash_partial(f: Path) -> str: + sha1 = hashlib.sha1() + chunk_size = 1024 * 1024 * 10 # 10MB chunk size + + total_read = 0 + with f.open("rb") as file: + while chunk := file.read(chunk_size): + total_read += len(chunk) + sha1.update(chunk) + break # Only reads the first 10MB + + return f"sha1:{total_read}:{sha1.hexdigest()}" + + +@contextlib.contextmanager +def connect_db_sqlite() -> typing.Generator[psycopg.Connection, typing.Any, typing.Any]: + with sqlite3.connect("vids.sqlite3") as conn: + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = 1") + yield conn + + +@contextlib.contextmanager +def connect_db() -> typing.Generator[psycopg.Connection, typing.Any, typing.Any]: + with psycopg.connect("postgres://abdus:abdus@db.abdus.dev:5444/smut?sslmode=disable") as conn: + conn.row_factory = psycopg.rows.dict_row + yield conn + + +def video_exists(con: psycopg.Connection, file_path: str) -> bool: + stmt = con.execute( + "select exists(select 1 from videos where file_path = %s) as exists", + [file_path], + ) + row = stmt.fetchone() + return row["exists"] + + +def delete_video(con: psycopg.Connection, file_path: str) -> bool: + con.execute("delete from videos where file_path = %s", [file_path]) + + +def save_video( + con: psycopg.Connection, + category: str, + file_path: str, + ffprobe: dict, + hash_partial: str, +): + sql = """insert into videos(category, file_path, ffprobe, hash_partial) values (%s, %s, %s, %s);""" + con.execute( + sql, + [ + category, + file_path, + json.dumps(ffprobe), + hash_partial, + ], + ) + + +def save_error(con: psycopg.Connection, file_path: str, stderr: str): + sql = """insert into scan_state(file_path, stderr) values (%s, %s) on conflict do nothing""" + con.execute( + sql, + [ + file_path, + stderr, + ], + ) + + +def scan_videos(): + logging.basicConfig(format=logging.BASIC_FORMAT, level=logging.DEBUG) + with connect_db() as con: + # with con: + # con.executescript(schema) + + all_vids = list(find_vids(VIDS_DIR)) + for i, f in enumerate(all_vids): + if "/_picks/" in str(f): + continue + progress = f"[{i+1}/{len(all_vids)}]" + if f.is_dir(): + continue + + file_path = str(f.absolute()) + + if video_exists(con, file_path=file_path): + logging.debug(f"{progress} video already exists {f=}") + continue + + logging.debug(f"probing {f=}") + try: + probed = ffmpeg.ffprobe(f) + except subprocess.CalledProcessError as e: + save_error(con, file_path=file_path, stderr=e.stderr) + continue + + file_hash = hash_partial(f) + + save_video( + category="downloaded.fiver", + con=con, + file_path=file_path, + ffprobe=probed.__dict__, + hash_partial=file_hash, + ) + logging.info(f"{progress} saved {f=}") + con.commit() + + +def clean_records(): + all_vids = set(map(str, find_vids(VIDS_DIR))) + + with connect_db() as con: + for row in con.execute("select file_path from videos"): + file_path = row["file_path"] + if file_path not in all_vids: + delete_video(con, file_path=row["file_path"]) + + +def unwanted(dry_run: bool): + take = set( + r""" +aidra fox +rachel starr +abella danger +sata jones +octavia red +paige owens +lily lou +cj miles +arabelle raphael +chloe amour +erin everheart +kenzie reeves +alex grey +alicia trece +angel youngs +angela white +april olsen +blake blossom +brandy renee +chantal +demi sutra +eden ivy +ema karter +emma hix +gianna dior +hazel moore +jazmin luv +jessy jey +josephine jackson +katsaros +kira noir +kitana lure +kitana montana +kristy black +leal +lia lin +lily labeau +liya silver +maddy may +nata ocean +rebel rhyder +rein +savannah bond +scarlett alexis +scarlit scandal +shalina +simon kitty +slimthick vic +stratton +tatum +tommy king +vanna bardot +vic marie +violet starr +vittoria +emily willis +THUNDERCOCK +lucky bee +charlotte sins +mandy muse +skye blue +alexis tae +willow +TRUEANAL +ALLANAL +avery cristy +ANALONLY +SWALLOWED +kelly collins +nicole black +xxlayna marie +katrina colt +allie addison +DIRTYAUDITIONS +scarlet chase +#sloppiest kiss +liz ocean +medusa +nicole aria +stacy bloom +lara frost +jamie jett +brooklyn gray +kimmy granger +kelsey kane +kay lovely +ivy wolfe +stefany kyler +leo ahsoka +lydia black +renee rose +emily jade +anya olsen +alexis crystal +sybil +megan rain +charlotte sartre +madison ivy +maddy black +gina valentina +lacey jayne +hailey rose +graycee baybee +katie kush +lilu moon +juis wild +kylie rocket +dolly dyson +nichole saphir +Octokuro +reislin +isabella de laa +qween goddess +INTHECRACK +athenea rose +daniela garcia +kama oxi +mina +baby nicols +jade amor +daniela ortiz +NYMPHO +""".strip().splitlines(keepends=False) + ) + drop = set( + r""" +ADDICTED2GIRLS +ASIANSEEPLOITED +BANANAFEVER +BBCPOVD +BFFS +# BRAZZERSEXXTRA +BRIDE4K +CHLOELAMOUR +DRKINLA +ETERNALDESIRE +EVERYTHINGBUTT +FAKEHOSTEL +FAKEHUBORIGINALS +FAKETAXI +FAMILYSINNERS +FEETISHPOV +GIRLGIRLXXX +GOTFILLED +HOOKUPHOTSHOT +HOWWOMENORGASM +IMMORALLIVE +IWANTCLIPS +JACQUIEETMICHELTV +JERKAOKE +NICONICE +NURUMASSAGE +OHMYHOLES +OYELOCA +PASCALSSUBSLUTS +SCHOOLOFCOCK +SHOTHERFIRST +SLIPPERYMASSAGE +TABOOHEAT +TEAMSKEETX +WAXXXED +WOODMANCASTINGX +PISSVIDS +LEGALPORNO +ANALVIDS +PORNBOX +\btap\b +alice xo +alicia williams +mona azar +anna de ville +baby kxtten +bella rolland +betzz +charlie forde +charlie red +cheyla collins +claudia garcia +cory chase +dani blue +dellai +eimy zoren +emily pink +gia dibella +greta foss +danielle renae +harley king +ivy maddox +sasha rose +olive glass +katherin moore +kristina grace +kylie page +larissa leite +lily starfire +lina arian +luna legend +luna rishi +luna wolfs +lya cutie +margan lee +marie berger +marilyn johnson +mia cheers +monika fox +moona snake +nichole saphir +nicole murkovski +olivia sparkle +polly petrova +pornworld +publicagent +publicbang +stacy cruz +sussy sweet +triple anal +vanessa vega +venera maxima +victoria nyx +whitney wright +zirael rem + """.lower() + .strip() + .splitlines(keepends=False) + ) + + take_patterns = [re.compile(p.strip().replace(" ", "."), flags=re.I) for p in take if not p.strip().startswith("#")] + drop_patterns = [re.compile(p.strip().replace(" ", "."), flags=re.I) for p in drop if not p.strip().startswith("#")] + + to_delete = set() + with connect_db() as con: + for row in con.execute("select file_path from videos where category = 'box'"): + file_path = Path(row["file_path"]) + text = re.sub(r"[.]+", " ", file_path.stem.lower()) + + if any(p.search(text) for p in drop_patterns): + to_delete.add(file_path) + if any(p.search(text) for p in take_patterns): + to_delete.discard(file_path) + + for it in sorted(to_delete): + print(it) + + return + + with connect_db() as con: + for path in sorted(to_delete, key=str): + try: + print(path.stem) + con.execute( + "update videos set marked_for_deletion_at = now() where file_path = %s", + [str(path)], + ) + if not dry_run: + path.unlink() + delete_video(con, file_path=str(path)) + con.commit() + except: + pass + + print(len(to_delete)) + + +def large_unwanted(): + with connect_db() as con: + for row in con.execute("select file_path from videos order by size_bytes desc limit 500"): + file_path = Path(row["file_path"]) + print(file_path) + + +def list_studios(): + re_studio = re.compile(r"^(.+?)\.\d+", flags=re.IGNORECASE) + seen = defaultdict(int) + with connect_db() as con: + for row in con.execute("select file_path from videos"): + file_path = Path(row["file_path"]) + m = re_studio.search(file_path.stem.lower()) + if m: + seen[m.group(1)] += 1 + + for it, n in Counter(seen).most_common(20): + print(it, n) + + +def lower_resolutions(dry_run: bool): + with connect_db() as con: + for row in con.execute("""select file_path from videos where file_path like '%720p%';"""): + file_path = Path(row["file_path"]) + video_1080p_path = file_path.with_stem(file_path.stem.replace("720p", "1080p")) + if video_exists(con, file_path=str(video_1080p_path)) and video_1080p_path.is_file(): + print(file_path) + if not dry_run: + file_path.unlink() + + +def dupes(): + sql = """ +with durationed as ( + select v.*, json_extract(v.ffprobe, '$.duration_sec') as duration_sec from videos v +) +select v.file_path, dupe.file_path from durationed v +join durationed dupe on v.file_path != dupe.file_path and abs(dupe.duration_sec - v.duration_sec) < 3 +where abs(dupe.size_bytes - v.size_bytes) < 3000000 +""" + with connect_db() as con: + for row in con.execute(sql): + print(row[0], row[1]) + print() + + +def different_format(dry_run: bool): + sql = """ + select file_path from videos where file_path like '%.mkv' + """ + with connect_db() as con: + for row in con.execute(sql): + mkv_path = Path(row["file_path"]) + mp4_path = mkv_path.with_suffix(".mp4") + if mkv_path.is_file() and mp4_path.is_file(): + if abs(mp4_path.stat().st_size - mkv_path.stat().st_size) < 5_000_000: + print(mkv_path) + if not dry_run: + mkv_path.unlink() + + +def count_shared_words(a: str, b: str) -> int: + a_words = set(re.split(r"[\s\-.]+", a.lower())) + b_words = set(re.split(r"[\s\-.]+", b.lower())) + + return len(a_words.intersection(b_words)) + + +def reencoded_self(dry_run: bool): + sql = """ + with + durationed_v as ( + select v.*, json_extract(v.ffprobe, '$.duration_sec') as duration_sec from videos v + ), + durationed_dl as ( + select v.*, json_extract(v.ffprobe, '$.duration_sec') as duration_sec from downloaded_videos v + ) + select v.file_path, dlv.file_path from durationed_v v + join durationed_dl dlv on abs(v.duration_sec - dlv.duration_sec) < 0.1 + where dlv.file_path like '%/_reenc/%' +""" + + with connect_db() as con: + for row in con.execute(sql): + v_path = Path(row[0]) + dlv_path = Path(row[1]) + if count_shared_words(v_path.stem, dlv_path.stem) < 2: + continue + + print(v_path.name, dlv_path.name, sep="\n") + if not dry_run: + try: + v_path.unlink() + except: + pass + + +def local_duped(dry_run: bool): + sql = """ +with durationed_dl as ( + select v.*, json_extract(v.ffprobe, '$.duration_sec') as duration_sec from downloaded_videos v +) +select v.file_path, dupe.file_path +from durationed_dl v + join durationed_dl dupe on + v.id != dupe.id + and v.id > dupe.id + and abs(dupe.duration_sec - v.duration_sec) < 0.1 + and abs(dupe.size_bytes - v.size_bytes) < 3000000 + """ + + root = Path(r"/Volumes/FIVER/_ingress/") + with connect_db() as con: + for row in con.execute(sql): + a_path = Path(row[0]) + b_path = Path(row[1]) + if count_shared_words(a_path.stem, b_path.stem) < 6: + continue + + # print(a_path.name, b_path.name, sep='\n') + keep, remove = a_path, b_path + if len(b_path.stem) > len(a_path.stem): + keep, remove = b_path, a_path + print(" # ", "matching", keep.name) + print(" rm", f'"{remove}"') + print() + + +def delete_dupes(): + rows = [] + + with connect_db() as con: + for row in rows: + file_path = Path(row["file_path"]) + try: + file_path.unlink() + delete_video(con, file_path=row["file_path"]) + con.commit() + except: + pass + + +def save_suggested_names(): + rows = [] + + with connect_db() as con: + for row in rows: + bandaid_path = Path(row["file_path"]) + fiver_path = Path(row["fiver_path"]) + + if bandaid_path.stem == fiver_path.stem: + continue + + if count_shared_words(bandaid_path.stem, fiver_path.stem) < 3: + print("unsimilar", row) + continue + + if bandaid_path.stem.count("--") == 3: + better_stem = bandaid_path.stem + else: + better_stem = fiver_path.stem + + better_stem = re.sub(r"(\d{4})(\d{2})(\d{2})", r"\1-\2-\3", better_stem) + + if better_stem == fiver_path.stem: + continue + + print(fiver_path.name, better_stem, sep="\n") + + con.execute( + "update videos set suggested_filename = %s where file_path = %s", + [better_stem, fiver_path.name], + ) + con.commit() + + +def delete_remote_files(): + rows = [ + {"file_path": "/mnt/box/files/_raw/prt/5KPorn.24.10.02.Leya.Desantis.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/AllAnal.24.10.30.Aviana.Violet.And.Nicole.Aria.XXX.720p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/AllAnal.24.11.07.Khloe.Kingsley.And.Cherry.Kiss.XXX.720p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/AnalOnly.24.11.26.Aria.Sloane.And.Nicole.Doshi.XXX.720p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/AnalTherapyXXX.24.02.10.Willow.Ryder.Family.Competition.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/AngelsLove.23.07.29.Cherry.Candle.And.Stacy.Cruz.Hot.Juice.Is.Dripping.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/BBCPie.20.05.31.Hazel.Moore.Morning.Wood.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/BangBros18.24.06.16.Angel.Gostosa.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/BangPOV.24.02.20.Justine.Jakobs.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/Blacked.22.10.29.Kay.Lovely.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/BlacksOnBlondes.22.11.29.Hazel.Moore.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/BlacksOnBlondes.24.11.22.Kylie.Rocket.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/BrattySis.22.06.10.Angel.Gostosa.Everyone.Wants.To.Fuck.My.Stepsister.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/BrattySis.23.06.23.Angel.Gostosa.My.Stepsister.Takes.Charge.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/BrattySis.24.05.10.Jade.Maris.And.Vanessa.Marie.Sharing.Stepbros.Bed.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/BrattySis.24.05.31.Renee.Rose.Nobody.Wants.To.Fuck.A.Virgin.Stepbro.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/BrattySis.24.07.26.Jade.Maris.And.Sera.Ryder.I.Want.To.Bone.My.Stepsister.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/BrazzersExxtra.24.08.21.Destiny.Mira.Cyclist.Sweat.Leads.To.Shower.Sex.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/BrazzersExxtra.24.10.20.Baby.Gemini.Dripping.Wet.Hookup.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/BrazzersExxtra.24.11.04.Nichole.Saphir.Tattooed.Bombshells.Gaping.Anal.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/BrazzersExxtra.24.11.20.Katie.Kush.And.Kelly.Caprice.Free.Use.Maid.Service.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/BrazzersExxtra.24.11.20.Kira.Noir.Secret.Goth.Camgirl.Roomie.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/CherryPimps.Busted.24.08.14.Lacey.Jayne.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/CrazyCollegeGFs.24.02.20.Destiny.Mira.Bunk.Banging.The.Handyman.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/Creampie-Angels.22.07.03.Nata.Ocean.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/CumSwappingSis.24.03.16.Jade.Maris.And.Liz.Jordan.Giving.Stepbro.Something.To.Do.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/DFXSoleMates.24.06.15.Angel.Gostosa.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/DatingMyStepson.24.10.03.Shalina.Devine.A.Game.Of.Dare.Always.Spices.Things.Up.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/DirtyAuditions.24.08.17.Kazumi.XXX.720p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/ElegantAngel.24.01.01.Hazel.Moore.A.Touch.Of.Class.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/ElegantAngel.24.11.29.Scarlett.Alexis.Stretch.My.Hole.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/EvilAngel.24.06.01.Chloe.Amour.XXX.1080p.HEVC.x265.PRT.mkv"}, + {"file_path": "/mnt/box/files/_raw/prt/EvilAngel.24.11.01.Princess.Alice.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/FilthyTaboo.24.10.19.Kelly.Caprice.Unsatisfied.MILF.Gets.What.She.Craves.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/GirlGirlXXX.24.01.16.Kylie.Rocket.And.Melissa.Stratton.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/Hegre.24.10.22.Anna.L.Erotic.Goddess.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/HotwifeXXX.24.11.06.Willow.Ryder.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/InTheCrack.E1928.Lia.Lin.Provence.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/KarupsPC.20.04.01.Nata.Ocean.Horny.Housekeeper.XXX.1080p.HEVC.x265.PRT.mkv"}, + { + "file_path": "/mnt/box/files/_raw/prt/LegalPorno.23.11.10.Lady.Ana.Jureka.Del.Mar.Fisting.ATOGM.Double.Anal.Fist.Big.Gapes.Monster.ButtRose.Squirt.Creampie.Swallow.GIO2635.XXX.1080p.HEVC.x265.PRT.mkv" + }, + {"file_path": "/mnt/box/files/_raw/prt/LegalPorno.24.06.19.Kristy.Black.GIO2815.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/LegalPorno.24.10.02.Leila.Botwin.AH007.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/LegendaryX.24.01.11.Melissa.Stratton.And.Nicole.Doshi.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/LegendaryX.24.02.01.Chantal.Danielle.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/LilSis.24.05.17.Jade.Maris.Let.The.Games.Begin.Part.1.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/LittleCaprice-Dreams.24.11.09.Tiffany.Tatum.And.Nata.Ocean.Nasstyx.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/MilfBody.24.06.14.Kelly.Caprice.Kellys.New.Routine.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/MomIsHorny.23.12.29.Justine.Jakobs.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/MomXXX.24.11.05.Kiara.Lord.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/MomsTeachSex.23.11.24.Justine.Jakobs.Getting.Down.To.Business.XXX.1080p.HEVC.x265.PRT.mkv"}, + {"file_path": "/mnt/box/files/_raw/prt/MrLuckyPOV.24.01.05.Gianna.Dior.And.Kazumi.Squirts.An.All.Star.Threesome.XXX.1080p.HEVC.x265.PRT.mkv"}, + {"file_path": "/mnt/box/files/_raw/prt/MrLuckyPOV.24.01.05.Gianna.Dior.And.Kazumi.Squirts.An.All.Star.Threesome.XXX.1080p.HEVC.x265.PRT.mkv"}, + {"file_path": "/mnt/box/files/_raw/prt/MySistersHotFriend.24.04.09.Willow.Ryder.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/NewSensations.24.11.16.Ellie.Nova.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/NubileFilms.23.10.17.Liz.Jordan.My.Roommate.Has.A.Crush.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/NubileFilms.23.11.20.Hailey.Rose.And.Jade.Maris.Happy.Birthday.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/NubileFilms.24.02.01.Liz.Jordan.February.2024.Fantasy.Of.The.Month.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/NubileFilms.24.03.01.Jade.Maris.March.2024.Fantasy.Of.The.Month.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/NubileFilms.24.12.01.Chloe.Temple.December.2024.Fantasy.Of.The.Month.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/Nubiles-Casting.24.10.28.Chanel.Camryn.Cast.Addison.Vodka.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/PascalsSubSluts.21.06.25.Rebel.Rhyder.Every.Inch.Every.Hole.XXX.1080p.HEVC.x265.PRT.mkv"}, + {"file_path": "/mnt/box/files/_raw/prt/Passion-HD.24.04.03.Aria.Valencia.Good.Help.XXX.1080p.HEVC.x265.PRT.mp4"}, + { + "file_path": "/mnt/box/files/_raw/prt/PissVids.23.05.20.Juis.Wild.Black.Pee.1on1.BBC.ATM.Balls.Deep.No.Pussy.Rough.Sex.Gapes.Pee.Drink.Cum.In.Mouth.Swallow.GL836.XXX.1080p.HEVC.x265.PRT.mp4" + }, + {"file_path": "/mnt/box/files/_raw/prt/PornMegaLoad.24.07.18.Justine.Jakobs.Hardcore.40443.XXX.1080p.HEVC.x265.PRT.mkv"}, + {"file_path": "/mnt/box/files/_raw/prt/PornPlus.24.10.18.Hailey.Rose.Naturally.Gifted.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/Private.23.11.18.Dolly.Dyson.Enjoys.An.Anal.Threesome.With.Mary.Popiense.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/Private.24.11.18.Lia.Lin.Gets.Horny.By.The.Pool.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/RKPrime.24.10.28.Octokuro.Fucking.After.The.Club.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/RickysRoom.23.03.09.Destiny.Mira.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/SexSelector.23.03.17.Graycee.Baybee.XXX.1080p.HEVC.x265.PRT.mkv"}, + {"file_path": "/mnt/box/files/_raw/prt/StrapLez.23.08.17.Alexis.Crystal.And.Lilly.Bella.Everything.She.Wants.XXX.1080p.HEVC.x265.PRT.mkv"}, + {"file_path": "/mnt/box/files/_raw/prt/StrapLez.24.11.14.Cherry.Candle.And.Lilly.Mays.Pleasure.Control.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/StrapLez.24.11.28.Alexis.Crystal.And.Mia.Trejsi.Rhythm.Of.Desire.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/TeensLoveHugeCocks.23.04.28.Kylie.Rocket.Influencers.Delight.XXX.1080p.HEVC.x265.PRT.mkv"}, + {"file_path": "/mnt/box/files/_raw/prt/ThePOVGod.24.10.25.Willow.Ryder.The.Call.Of.That.Booty.XXX.1080p.HEVC.x265.PRT.mp4"}, + {"file_path": "/mnt/box/files/_raw/prt/Tiny4K.23.02.02.Liz.Jordan.Ice.Cream.Treat.XXX.1080p.HEVC.x265.PRT.mp4"}, + ] + with connect_server("klein") as ssh, connect_db() as con: + for row in rows: + file_path = row["file_path"] + try: + print("deleting", file_path) + ssh.exec_command(f'rm -rf "{file_path}"') + delete_video(con, file_path=file_path) + con.commit() + except: + pass + + +@contextlib.contextmanager +def connect_server(host: str) -> typing.Generator[paramiko.SSHClient, typing.Any, None]: + config = paramiko.SSHConfig.from_path(str(Path(r"~/.ssh/config").expanduser())) + key = paramiko.RSAKey.from_private_key_file(filename=str(Path(r"~/.ssh/id_rsa").expanduser())) + + target = config.lookup(host) + + ssh_client = paramiko.SSHClient() + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh_client.connect( + hostname=target["hostname"], + port=target.as_int("port"), + username=target["user"], + pkey=key, + ) + + yield ssh_client + + ssh_client.close() + + +def collect_all(): + root = Path("/Volumes/FATBLUE/_backup/_vidz/_all") + video_files = find_vids(root) + + # re_good = re.compile(r"(?P.+) - (?P.+) --? (?P<studio>\S+) (?P<date_year>\d{4})(?P<date_month>\d{2})(?P<date_day>\d{2})") + # re_good = re.compile(r"(?P<people>.+) - (?P<title>.+) --? (?P<studio>\S+)") + # re_good = re.compile(r"(?P<people>.+) - (?P<title>.+) --? (?P<date_year>\d{4})(?P<date_month>\d{2})(?P<date_day>\d{2})") + re_good = re.compile(r"(?P<people>.+) - (?P<title>[^-]+) (?P<date_year>\d{4})(?P<date_month>\d{2})(?P<date_day>\d{2})") + + for f in sorted(video_files): + m = re_good.search(f.stem) + if not m: + continue + params = m.groupdict() + title = params["title"] + if " " not in title: + print(f.stem, title, sep="\t\t") + continue + # better_name = f"{m['people']} -- @{m['studio']} -- {m['title']} -- {m['date_year']}-{m['date_month']}-{m['date_day']}" + better_name = f"{m['people']} -- {m['title']} -- {m['date_year']}-{m['date_month']}-{m['date_day']}" + # better_name = f"{m['people']} -- @{m['studio']} -- {m['title']}" + print(f.stem, better_name, sep="\n") + better_path = f.with_stem(better_name) + f.rename(better_path) + + +def clean_images(): + from PIL import Image + + for img in Path(r"/Users/abdus/Downloads/temp/Ploy Tigerstam -- OnlyFans Leak").glob("*.jpg"): + image = Image.open(img) + width, height = image.size + + if width < 2000 or height < 2000: + img.unlink() + + +def main(): + dry = True + try: + # clean_images() + # collect_all() + # save_suggested_names() + # delete_remote_files() + # transfer() + # different_format(dry_run=dry) + # local_duped(dry_run=dry) + # reencoded_self(dry_run=dry) + # dupes() + # scan_videos() + # list_studios() + # lower_resolutions(dry_run=dry) + # large_unwanted() + unwanted(dry_run=dry) + finally: + pass + # clean_records() + + +if __name__ == "__main__": + main() diff --git a/csv_to_html.py b/csv_to_html.py new file mode 100644 index 0000000..b0e9916 --- /dev/null +++ b/csv_to_html.py @@ -0,0 +1,87 @@ +import csv +import html +import sys + +page_css = """ +body { + font-family: consolas, monospace; +} +table { + border-collapse: collapse; + width: 100%; +} +th, td { + border: 1px solid black; + padding: 8px; + text-align: left; +} +th { + background-color: #f2f2f2; +} +tr:nth-child(even) { + background-color: #fdfdfd; +} +""" + + +def csv_to_html(csv_input: str) -> str: + """ + Converts TSV input from stdin to an HTML table. + + Args: + csv_input: The TSV input as a string. + + Returns: + A string containing the HTML table, or None if an error occurs. + """ + try: + delimiter = "\t" if "\t" in csv_input else "," + reader = csv.reader(csv_input.splitlines(), delimiter=delimiter) + header = next(reader) # Get the header row + + header_html = f'{"".join(f"<th>{html.escape(col)}</th>" for col in header)}' + rows = [f"<tr>{''.join(f'<td>{html.escape(cell)}</td>' for cell in row)}</tr>" for row in reader] + rows_html = "\n".join(rows) + + rendered = f""" + <!DOCTYPE html> + <html> + <head> + <title>Query Results + + + + + + {header_html} + + + {rows_html} + +
+ + + """ + + return rendered + + except csv.Error as e: + raise ValueError(f"Error parsing TSV: {e}") + except StopIteration: + return "

No data found in TSV input.

" + + +def main(): + try: + tsv_data = sys.stdin.read() + html_output = csv_to_html(tsv_data) + if html_output: + print(html_output) + except UnicodeDecodeError as e: + raise ValueError(f"Error decoding input: {e}. Ensure your TSV data is encoded correctly (e.g., UTF-8).") + except Exception as e: + raise Exception(f"An unexpected error occurred: {e}") + + +if __name__ == "__main__": + main() diff --git a/emp.py b/emp.py deleted file mode 100755 index 6b7a4ba..0000000 --- a/emp.py +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env python3.9 -from dataclasses import dataclass -import json -from pathlib import Path -import subprocess -import typing -from playwright.sync_api import Playwright, sync_playwright, Browser -from typer import Option, Typer - - -class Storage(typing.Protocol): - def get(self, key: str) -> typing.Optional[typing.Any]: - ... - - def set(self, key: str, value) -> None: - ... - - -@dataclass -class FileStorage: - path: Path - - def get(self, key: str): - try: - return json.loads(self.path.read_text()).get(key) - except FileNotFoundError: - return None - - def set(self, key: str, value) -> None: - try: - data = json.loads(self.path.read_text()) - except FileNotFoundError: - data = {} - data[key] = value - self.path.write_text(json.dumps(data)) - - -class Emp: - def __init__(self, browser: Browser, storage: Storage) -> None: - self.browser = browser - self.storage = storage - - def ensure_session(self, username: str, password: str) -> None: - if self.storage.get("cookies") is None: - self.login(username, password) - - def login(self, username: str, password: str) -> typing.Dict[str, str]: - """ - logins and returns session cookies - """ - context = self.browser.new_context() - - # Open new page - page = context.new_page() - - # Go to https://www.empornium.is/ - page.goto("https://www.empornium.is/login") - - page.locator('[placeholder="Username"]').fill(username) - page.locator('[placeholder="Password"]').fill(password) - # Click text=Stay logged in - page.locator("text=Stay logged in").click() - - # Click input:has-text("login") - page.locator('input:has-text("login")').click() - - cookies = {c["name"]: c["value"] for c in context.cookies()} - self.storage.set("cookies", {"sid": cookies["sid"]}) - - return { - "sid": cookies["sid"], - } - - def prepare_post( - self, - torrent_path: Path, - title: str, - tags: str, - description: str, - cover_image_url: str, - category: typing.Optional[str] = None, - ) -> None: - cookies = self.storage.get("cookies") - assert cookies, "You must login first" - - context = self.browser.new_context() - context.add_cookies( - [{"name": k, "value": v, "domain": "www.empornium.is", "path": "/"} for k, v in cookies.items()] - ) - - page = context.new_page() - - page.goto("https://www.empornium.is/upload.php") - - if torrent_path.is_file(): - page.locator('input[name="file_input"]').set_input_files(torrent_path.expanduser().resolve()) - page.locator('text="check for dupes"').click() - - # Select category - if category: - page.locator('select[name="category"]').select_option(label=category) - - page.locator('input[name="title"]').fill(title) - page.locator('textarea[name="taglist"]').fill(tags) - page.locator('input[name="image"]').fill(cover_image_url) - page.locator('textarea[name="desc"]').fill(description) - - # Click text=Preview - page.locator("text=Preview").click() - - -def submit_post(): - storage = FileStorage(Path("emp.json")) - - with sync_playwright() as playwright: - browser = playwright.chromium.launch(headless=False) - emp = Emp(browser=browser, storage=storage) - emp.ensure_session(username="zzzp", password="9arjs9za2o") - - emp.prepare_post( - torrent_path=Path("~/Downloads/v.torrent"), - title="A Title", - tags="tag.1 tag.2", - description="Some description", - cover_image_url="https://images.com/image.jpg", - category="Anal", - ) - input("Press Enter to continue...") - browser.close() - - -cli = Typer(name="emp") - - -torrent_cli = Typer(name="torrent") -cli.add_typer(torrent_cli) - - -@torrent_cli.callback("torrent") -def make_torrent(paths: typing.List[Path], announce_url: str = Option(..., envvar="ANNOUNCE_URL")): - if len(paths) == 1 and paths[0].is_dir(): - dir_path = paths[0] - - dir_path = dir_path.expanduser().resolve() - proc = subprocess.run( - [ - "torrentify", - f"-announce={announce_url}", - f"-comment=created by zzzp", - f"-created-by=zzzp", - f"-name={dir_path.name}", - str(dir_path), - ], - check=True, - capture_output=True, - ) - torrent_path = dir_path / f"{dir_path.name}.torrent" - torrent_path.write_bytes(proc.stdout) - - -@torrent_cli.command() -def clone(): - pass - - -if __name__ == "__main__": - cli() diff --git a/enc.py b/enc.py index bfcd495..2556117 100755 --- a/enc.py +++ b/enc.py @@ -175,7 +175,7 @@ def upload_file( size = file_path.stat().st_size while True: proc = subprocess.Popen( - ["rsync", "--bwlimit", "1100", "-rvPa", str(file_path), destination], + ["rsync", "--bwlimit", "4500", "-rvPa", str(file_path), destination], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, @@ -333,14 +333,14 @@ def parse_args(): arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) arger.add_argument("video_path", type=Path, help="path to video file") arger.add_argument("-e", "--encoder", choices=["handbrake", "ffmpeg"], default="handbrake", help="encoder engine") - arger.add_argument("-q", "--quality", type=float, default=25, help="x265 quality factor") + arger.add_argument("-q", "--quality", type=float, default=35, help="x265 quality factor") arger.add_argument("--output-dir", "-o", dest="output_dir", type=Path, help="Dir to save encoded files") arger.add_argument("--rsync", dest="upload_target", help="rsync encoded file to a host") arger.add_argument("--validate", action="store_true", default=False, help="perform validations before starting") arger.add_argument("--denoise", action="store_true", default=False, help="Enable denoise filter (Handbrake only)") arger.add_argument("--10bit", action="store_true", dest="is_10bit", help="Encode using 10-bit profile") - arger.add_argument("--8bit", action="store_false", dest="is_10bit", help="Encode using 8-bit profile") - arger.set_defaults(is_10bit=True) + arger.add_argument("--8bit", action="store_true", dest="is_10bit", help="Encode using 8-bit profile") + arger.set_defaults(is_8bit=True) def parse_time(val: str) -> int: parts = val.split(":") diff --git a/extract_archives.py b/extract_archives.py new file mode 100644 index 0000000..dd64e5e --- /dev/null +++ b/extract_archives.py @@ -0,0 +1,168 @@ +import argparse +import logging +import re +import subprocess +import sys +from pathlib import Path + +logger = logging.getLogger(__name__) + +KNOWN_PASSWORDS = """ +torgo +Yuzuki +oron.com +MonikGonPs +koth +drdoom-psuzy +psuzy +suzy +idle +dan4260 +""".strip().splitlines(keepends=False) + + +def find_archive_files(root_dir: Path) -> list[Path]: + extractables = [] + globs = ["*.rar", "*.zip", "*.7z", "*.zip.*", "*.7z.*"] + for glob in globs: + for f in root_dir.rglob(glob): + extractables.append(f) + + extractables = [f for f in extractables if is_extractable(f)] + + return sorted(extractables, key=str) + + +def is_extractable(archive: Path) -> bool: + if re.search(f"\.zip|\.7z", archive.name): + zip_part = re.search("(\d+)$", archive.suffix) + if zip_part and int(zip_part.group(1)) == 1: + return True + if not zip_part: + return True + + if archive.name.endswith(".rar"): + rar_part = re.search(r"\.part(\d+)", archive.name) + if not rar_part: + return True + if rar_part and int(rar_part.group(1)) == 1: + if archive.with_name(archive.name.replace("part1", "part2")).is_file(): + return True + + return False + + +def extract_archive_rar(archive: Path, output_dir: Path, password: str | None = None) -> None: + args = [ + "rar", + "e", + f"-p{password}" if password else "-p-", + f"{archive.name}", + f"{output_dir}", + ] + try: + subprocess.run(args, cwd=archive.parent, capture_output=True, text=True, check=True) + except subprocess.CalledProcessError as e: + output: str = e.stderr or e.stdout + if "corrupt file or wrong password" in output.lower(): + raise ValueError("corrupt or invalid password") + elif e.returncode == 3: + raise FileNotFoundError("missing parts") + elif e.returncode == 10: + raise ValueError("not a rar file") + elif e.returncode == 11: + raise ValueError("invalid password") + + raise + + +def extract_archive_7z(archive: Path, output_dir: Path, password: str | None = None): + args = [ + "7z", + "e", + f'-p{password or ""}', + f"-o{output_dir}", + "-y", + f"{archive}", + ] + + try: + subprocess.run(args, capture_output=True, text=True, check=True) + except subprocess.CalledProcessError as e: + output: str = e.stderr or e.stdout + if "missing volume" in output.lower(): + raise FileNotFoundError("missing parts") + if e.returncode == 2: + raise ValueError("invalid password") + + raise + + +def extract_archive(archive_path: Path, output_dir: Path, passwords: list[str]) -> None: + for p in ["", *passwords]: + try: + if ".rar" in archive_path.name: + extract_archive_rar(archive_path, output_dir, p) + else: + extract_archive_7z(archive_path, output_dir, p) + except Exception as e: + logger.error(f"unhandled error: {e}") + raise e + + +def delete_archive(archive: Path): + stem = archive.stem.rsplit(".", maxsplit=1)[0] + for f in archive.parent.rglob(f"*{archive.suffix}"): + if archive.stem.startswith(stem): + logger.info(f"deleting: {f.name}") + f.unlink() + + +def parse_args(): + arger = argparse.ArgumentParser() + arger.add_argument("archive_paths", type=Path, nargs="+", help="List of archive paths") + arger.add_argument("--cwd", type=Path, default=Path.cwd(), help="working directory") + arger.add_argument("--clean", type=bool, default=False, help="delete file(s) after successful extraction", action="store_true") + + if len(sys.argv) < 2: + arger.print_help() + exit(1) + + return arger.parse_args() + + +def main(): + args = parse_args() + cwd: Path = args.cwd + + archives = find_archives(cwd) + logger.info(f"found {len(archives)} archives") + for f in archives: + logger.info(f"\t{f.relative_to(cwd)}") + + typer.confirm("continue", default=True, abort=True) + + failed = [] + for i, f in enumerate(archives, start=1): + progress = f"{i:02}/{len(archives):02}" + logger.info(f"\nextracting {progress}: {f.relative_to(cwd)}") + try: + success = extract_archive(f, passwords=passwords) + if success: + if clean: + delete_archive(f) + logger.info(f"done") + else: + logger.error("failed") + except (ValueError, FileNotFoundError) as e: + logger.error(str(e)) + failed.append(f) + if failed: + logger.info(f"\n\nfailed to extract {len(failed)} archives") + for f in failed: + logger.info(f"{f.relative_to(cwd)}") + logger.info("\n\nfinished.") + + +if __name__ == "__main__": + typer.run(main) diff --git a/ffmpeg.py b/ffmpeg.py index b8b765d..2f9846a 100755 --- a/ffmpeg.py +++ b/ffmpeg.py @@ -4,12 +4,10 @@ import dataclasses import json import logging import subprocess -import typing from datetime import timedelta from math import ceil from pathlib import Path - logger = logging.getLogger(__name__) @@ -26,11 +24,28 @@ class FfprobeResult: container: str ar: float sample_ar: float - tags: typing.Dict[str, str] + tags: dict[str, str] + + @property + def resolution(self) -> str: + return f'{self.width}x{self.height}' + + @property + def has_metadata(self): + return any(k in self.tags for k in ['title', 'comment']) + + @property + def hd_mode(self) -> str: + if 700 < self.height < 800: + return '720p' + if 900 < self.height < 1200: + return '1080p' + if 2000 < self.height < 2200: + return '4K' + raise ValueError('unknown HD mode') -def ffprobe(video: Path) -> FfprobeResult: - logger.debug("executing ffprobe") +def ffprobe(video_path: Path) -> FfprobeResult: proc = subprocess.run( args=[ "ffprobe", @@ -40,7 +55,7 @@ def ffprobe(video: Path) -> FfprobeResult: "-show_format", "-print_format", "json", - str(video), + str(video_path), ], text=True, stdout=subprocess.PIPE, @@ -48,8 +63,8 @@ def ffprobe(video: Path) -> FfprobeResult: ) try: proc.check_returncode() - except subprocess.CalledProcessError: - logger.error("failed to run ffprobe", exc_info=True) + except subprocess.CalledProcessError as e: + logger.error(f"failed to run ffprobe. stderr={e.stderr}", exc_info=True) raise output: dict = json.loads(proc.stdout) video_stream: dict = [s for s in output.get("streams", []) if s.get("codec_type") == "video"][0] @@ -71,9 +86,6 @@ def ffprobe(video: Path) -> FfprobeResult: except: sample_ar = 1 - logger.debug(f"dimensions={width}x{height} duration={duration_time}") - # total_frames = video_stream.get("nb_frames") - return FfprobeResult( duration_sec=duration, duration_human=str(duration_time), @@ -83,19 +95,18 @@ def ffprobe(video: Path) -> FfprobeResult: width=width, bitrate=bitrate, height=height, - container=video.suffix.lstrip(".").lower(), + container=video_path.suffix.lstrip(".").lower(), ar=width / height, sample_ar=sample_ar, tags=tags, - # total_frames=total_frames, ) def strip_metadata(video_path: Path, save_path: Path) -> Path: media_info = ffprobe(video_path) # check if the first video stream has hevc codec - if media_info["streams"][0]["codec_name"] == "hevc": - logger.info(f"Video codec is hevc, adding hvc1 tag") + if media_info.codec == "hevc": + logging.info(f"Video codec is hevc, adding hvc1 tag") extra_args = ["-tag:v", "hvc1"] else: extra_args = [] @@ -106,7 +117,7 @@ def strip_metadata(video_path: Path, save_path: Path) -> Path: '-i', str(video_path), '-c:v', 'copy', - '-movflags', '+faststart', + # '-movflags', '+faststart', '-map_metadata', '-1', *extra_args, '-c:a', 'copy', @@ -114,32 +125,22 @@ def strip_metadata(video_path: Path, save_path: Path) -> Path: str(save_path), ] # fmt: on - logger.info(f"calling ffmpeg with {args=}") + logging.info(f"calling ffmpeg with {args=}") subprocess.run(args, check=True) return save_path def make_thumbnail_tile( video: Path, - image_path: Path = None, columns: int = 3, interval_seconds: int = 60, tile_width: int = 540, skip_first_sec: int = 10, - skip_if_exists: bool = False, -) -> Path: - if not image_path: - image_path = video.parent / f"{video.stem}.thumbnail.jpg" - if image_path.is_file() and skip_if_exists: - logger.debug("thumbnail already exists") - return image_path - +) -> bytes: info = ffprobe(video) min_frames = columns * 3 - sec_per_frame = min(interval_seconds, ceil(info.duration_sec / min_frames)) - rows = ceil(info.duration_sec // sec_per_frame / columns) tile = f"{columns}x{rows}" @@ -150,25 +151,32 @@ def make_thumbnail_tile( # timestamp_filter = rf"drawtext=r=1:timecode='00\:00\:00\:00':fontsize=16:fontcolor=white:x=10:y=10:box=1:boxcolor=black@0.5" # timestamp_filter = fr"drawtext=text='%{{(pts\\+{skip_first_sec})\:hms}}':fontsize=16:fontcolor=white:x=10:y=10:box=1:boxcolor=black@0.5" - proc = subprocess.run( + args = [ # fmt: off - args=[ - "ffmpeg", - "-v", "error", - "-skip_frame", "nokey", - "-ss", f"{skip_first_sec}", - "-i", str(video.absolute()), - # "-vf", f"fps=1/{sec_per_frame},scale={scale},{timestamp_filter},tile={tile}", - "-vf", f"fps=1/{sec_per_frame},scale={scale},tile={tile}", - "-frames", "1", - "-y", str(image_path.absolute()), - ], + "ffmpeg", + "-v", "error", + "-skip_frame", "nokey", + "-ss", f"{skip_first_sec}", + "-i", str(video.absolute()), + # "-vf", f"fps=1/{sec_per_frame},scale={scale},{timestamp_filter},tile={tile}", + "-vf", f"fps=1/{sec_per_frame},scale={scale},tile={tile}", + "-frames", "1", + "-c:v", + "jpeg2000", + "-q:v", "90", + "-f", "image2pipe", + "-", # fmt: on - stdout=subprocess.DEVNULL, + ] + logger.debug(f'calling ffmpeg with args={args}') + proc = subprocess.run( + args=args, + stdout=subprocess.PIPE, timeout=2000, + # preexec_fn=os.setpgrp, ) proc.check_returncode() - return image_path + return bytes(proc.stdout) if __name__ == "__main__": @@ -177,9 +185,11 @@ if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG) for it in sys.argv[1:]: video = Path(it) + image_path = video.parent / f"{video.stem}.thumbnail.jpg" if not video.exists(): continue logger.info("creating thumbnail for %s", video) - make_thumbnail_tile(video) + image_jpeg = make_thumbnail_tile(video) + image_path.write_bytes(image_jpeg) diff --git a/ffmpeg_gif.py b/ffmpeg_gif.py index 4fe23d2..94386ba 100755 --- a/ffmpeg_gif.py +++ b/ffmpeg_gif.py @@ -58,7 +58,7 @@ def optimize_gif(gif_path: Path) -> Path: subprocess.run( [ "gifsicle", - "--lossy=98", + "--lossy=200", "--optimize=3", "--batch", "-i", @@ -75,9 +75,9 @@ def parse_args(argv: typing.List[str] = None) -> argparse.Namespace: arger.add_argument( "--start", "-s", dest="from_time", type=str, help="Time to start from, e.g. 02:59", required=True ) - arger.add_argument("--duration", "-d", default=3, type=int, help="Duration of gif in seconds") - arger.add_argument("--fps", default=15, type=int, help="Frames per second") - arger.add_argument("--width", default=400, type=int, help="Image width") + arger.add_argument("--duration", "-d", default=3, type=float, help="Duration of gif in seconds") + arger.add_argument("--fps", default=20, type=int, help="Frames per second") + arger.add_argument("--width", default=376, type=int, help="Image width") arger.add_argument("--optimize", default=True, action="store_true", help="Reduce GIF filesize") if len(argv) == 0: @@ -105,7 +105,7 @@ def main(): save_path = None if str(args.video_path).startswith("http"): - save_path = Path("~/Downloads").expanduser() / f"{md5(str(video_path))}__{from_time.replace(':', '')}.gif" + save_path = Path("~/Pictures/Screenshots/").expanduser() / f"{md5(str(video_path))}__{from_time.replace(':', '')}.gif" gif_path = create_gif( video_path=video_path, diff --git a/ffmpeg_strip.py b/ffmpeg_strip.py index 67e2dba..2801e9d 100755 --- a/ffmpeg_strip.py +++ b/ffmpeg_strip.py @@ -41,7 +41,7 @@ def clean_video(video_path: Path, save_path: Path) -> Path: media_info = ffprobe(video_path) # check if the first video stream has hevc codec if media_info["streams"][0]["codec_name"] == "hevc": - logging.info(f"Video codec is hevc, adding hvc1 tag") + logging.debug(f"Video codec is hevc, adding hvc1 tag") extra_args = ["-tag:v", "hvc1"] else: extra_args = [] @@ -52,7 +52,7 @@ def clean_video(video_path: Path, save_path: Path) -> Path: '-i', str(video_path), '-c:v', 'copy', - # '-movflags', '+faststart', + # '-movflags', '+faststart', '-map_metadata', '-1', *extra_args, '-c:a', 'copy', diff --git a/file_renamer.py b/file_renamer.py index 64887a2..1ad494a 100755 --- a/file_renamer.py +++ b/file_renamer.py @@ -10,24 +10,25 @@ import typing from pathlib import Path script_path = Path(__file__) -db_filename = script_path.with_name(f'{script_path.stem}.known.txt') +db_filename = script_path.with_name(f"{script_path.stem}.known.txt") def save_actors(actors: typing.Iterable[str]): existing = load_actors() all_actors = sorted({*existing, *actors}) - with db_filename.open('w', encoding='utf-8', newline='\n') as f: + with db_filename.open("w", encoding="utf-8", newline="\n") as f: f.writelines(all_actors) def load_actors() -> set[str]: try: - with db_filename.open('w', encoding='utf-8', newline='\n') as f: + with db_filename.open(encoding="utf-8", newline="\n") as f: return {line.strip() for line in f} except FileNotFoundError: return set() +re_date_only = re.compile(r"\b(\d{4})\b") # 2024 re_date_us = re.compile(r"\b(\d{2})\D(\d{2})\D(\d{2})\b") # 12/31/21 re_date_iso = re.compile(r"\b(\d{4})\D?(\d{2})\D?(\d{2})\b") # 2020-12-31 re_date_iso_short = re.compile(r"(\d{2})\D?(\d{2})\D?(\d{2})") # 20-12-31"' @@ -53,10 +54,19 @@ class ParsedDate: def parse_date(filename: str) -> ParsedDate | None: + def from_date_only(): + match = re_date_only.search(filename) + y = int(match.group(1)) + return ParsedDate( + datetime.date(y, 1, 1), start=match.start(), end=match.end() + ) + def from_us_format(): match = re_date_us.search(filename) m, d, y = map(int, match.groups()) - return ParsedDate(datetime.date(y + 2000, m, d), start=match.start(), end=match.end()) + return ParsedDate( + datetime.date(y + 2000, m, d), start=match.start(), end=match.end() + ) def from_iso(): match = re_date_iso.search(filename) @@ -66,7 +76,9 @@ def parse_date(filename: str) -> ParsedDate | None: def from_iso_short(): match = re_date_iso_short.search(filename) y, m, d = map(int, match.groups()) - return ParsedDate(datetime.date(y + 2000, m, d), start=match.start(), end=match.end()) + return ParsedDate( + datetime.date(y + 2000, m, d), start=match.start(), end=match.end() + ) def from_iso_reversed(): match = re_date_iso_rev.search(filename) @@ -76,10 +88,13 @@ def parse_date(filename: str) -> ParsedDate | None: def from_iso_reversed_short(): match = re_date_iso_rev_short.search(filename) d, m, y = map(int, match.groups()) - return ParsedDate(datetime.date(y + 2000, m, d), start=match.start(), end=match.end()) + return ParsedDate( + datetime.date(y + 2000, m, d), start=match.start(), end=match.end() + ) candidates = [] for fn in [ + from_date_only, from_us_format, from_iso, from_iso_short, @@ -94,7 +109,9 @@ def parse_date(filename: str) -> ParsedDate | None: today = datetime.date.today() future_threshold = today + datetime.timedelta(days=60) past_threshold = datetime.date(2010, 1, 1) - candidates = [it for it in candidates if past_threshold < it.date < future_threshold] + candidates = [ + it for it in candidates if past_threshold < it.date < future_threshold + ] if not candidates: return @@ -112,31 +129,35 @@ class Release: def to_filename(self): parts = [] if self.actors: - parts.append(', '.join(self.actors)) + parts.append(", ".join(self.actors)) if self.studio: - parts.append(f'@{self.studio}') + parts.append(f"@{self.studio}") if self.title: parts.append(self.title) if self.released_at: parts.append(self.released_at.isoformat()) - return ' -- '.join(parts) + return " -- ".join(parts) def trash(path: Path): filename = f'"{path}"' - cmd = ['osascript', '-e', f'tell app "Finder" to move (POSIX file {filename}) to trash'] + cmd = [ + "osascript", + "-e", + f'tell app "Finder" to move (POSIX file {filename}) to trash', + ] subprocess.run(cmd).check_returncode() def filenames_to_actors(): sources = [ - Path(r'/Users/abdus/Downloads/temp/'), - Path(r'/Volumes/BANDAID/_temp/__reenc/'), - Path(r'/Volumes/BANDAID/_temp/'), + Path(r"/Users/abdus/Downloads/temp/"), + Path(r"/Volumes/BANDAID/_temp/__reenc/"), + Path(r"/Volumes/BANDAID/_temp/"), ] actors = set() for it in sources: - for f in it.glob('*.mp4'): + for f in it.glob("*.mp4"): if not f.is_file(): continue if r := parse_release(f.name): @@ -144,78 +165,83 @@ def filenames_to_actors(): return actors -def parse_release(filename: str, known_actors: set[str] | None = None): +def parse_release( + filename: str, known_actors: set[str] | None = None +) -> Release | None: if not known_actors: known_actors = set() - filename = re.sub(r'\.\w{3,4}$', '', filename) + if any(filename.lower().endswith(ext) for ext in [".mp4", ".mkv"]): + filename = re.sub(r"\.\w{3,4}$", "", filename) def from_own(): - remaining = re.sub(r'\s+\[[^]]+]$', '', filename) - remaining = re.sub(r'\s+\[([^]]+|\d+\w)(,\s*[^]]+)?]', '', remaining) - remaining = re.sub(r'\s*--\s*', ' -- ', remaining) + remaining = re.sub(r"\s+\[[^]]+]$", "", filename) + remaining = re.sub(r"\s+\[([^]]+|\d+\w)(,\s*[^]]+)?]", "", remaining) + remaining = re.sub(r"\s*--\s*", " -- ", remaining) - match remaining.split(' -- '): - case [actors, studio, title, date] if studio.startswith('@'): + match remaining.split(" -- "): + case [actors, studio, title, date] if studio.startswith("@"): return Release( - actors=sorted(actors.split(', ')), - studio=studio.removeprefix('@'), + actors=sorted(actors.split(", ")), + studio=studio.removeprefix("@"), title=title, released_at=datetime.date.fromisoformat(date), ) - case [actors, studio, date] if studio.startswith('@') and (parsed := parse_date(date)): + case [actors, studio, date] if studio.startswith("@") and ( + parsed := parse_date(date) + ): return Release( - actors=sorted(actors.split(', ')), - studio=studio.removeprefix('@'), + actors=sorted(actors.split(", ")), + studio=studio.removeprefix("@"), title=None, released_at=parsed.date, ) - case [actors, studio, title] if studio.startswith('@'): + case [actors, studio, title] if studio.startswith("@"): return Release( - actors=sorted(actors.split(', ')), - studio=studio.removeprefix('@'), + actors=sorted(actors.split(", ")), + studio=studio.removeprefix("@"), title=title, released_at=None, ) case [actors, title, date] if (parsed := parse_date(date)): return Release( - actors=sorted(actors.split(', ')), + actors=sorted(actors.split(", ")), studio=None, title=title, released_at=parsed.date, ) - case [actors, studio] if studio.startswith('@'): + case [actors, studio] if studio.startswith("@"): return Release( - actors=sorted(actors.split(', ')), + actors=sorted(actors.split(", ")), studio=studio, ) case [actors, title]: return Release( - actors=sorted(actors.split(', ')), + actors=sorted(actors.split(", ")), title=title, ) def from_prt(): - assert '.PRT' in filename - remaining = re.sub(r'\.(720p|1080p|HEVC|x265|PRT|XXX)', ' ', filename) - studio = remaining[:remaining.index('.')] + # assert '.PRT' in filename + remaining = re.sub(r"\.(720p|1080p|HEVC|x265|PRT|XXX)", " ", filename) + studio = remaining[: remaining.index(".")] parsed_date = parse_date(remaining) - remaining = remaining[parsed_date.end:] - remaining = re.sub(r'[. ]+', ' ', remaining).strip() + remaining = remaining[parsed_date.end :] + remaining = re.sub(r"[. ]+", " ", remaining).strip() actors = [] title = None - match remaining.split(' '): - case [a_first, a_last, 'And', b_first, b_last]: - actors = [f'{a_first} {a_last}', f'{b_first} {b_last}'] - case [a_first, a_last, 'And', b_first, b_last, *rest]: - actors = [f'{a_first} {a_last}', f'{b_first} {b_last}'] - title = ' '.join(rest) + match remaining.split(" "): + case [a_first, a_last, "And", b_first, b_last]: + actors = [f"{a_first} {a_last}", f"{b_first} {b_last}"] + case [a_first, a_last, "And", b_first, b_last, *rest]: + actors = [f"{a_first} {a_last}", f"{b_first} {b_last}"] + title = " ".join(rest) case [first, last]: - actors = [f'{first} {last}'] + actors = [f"{first} {last}"] case [first, last, *rest]: - actors = [f'{first} {last}'] - title = ' '.join(rest) + actors = [f"{first} {last}"] + title = " ".join(rest) case [first]: actors = [first] @@ -243,7 +269,9 @@ def parse_release(filename: str, known_actors: set[str] | None = None): def parse_args(): arger = argparse.ArgumentParser() - arger.add_argument('filenames', nargs='+', type=lambda v: Path(v).resolve(), help='Filenames') + arger.add_argument( + "filenames", nargs="+", type=lambda v: Path(v).resolve(), help="Filenames" + ) return arger.parse_args() @@ -254,17 +282,18 @@ class MoveOp: def to_dict(self) -> dict: return { - 'source': str(self.source), - 'target': str(self.target), + "source": str(self.source), + "target": str(self.target), } def write_undo_script(save_dir: Path, ops: list[MoveOp]): - now = datetime.datetime.now().isoformat().replace(':', '') - save_path = save_dir / f'undo_{now}.json' - with save_path.open('w', encoding='utf-8') as f: + now = datetime.datetime.now().isoformat().replace(":", "") + save_path = save_dir / f"undo_{now}.json" + with save_path.open("w", encoding="utf-8") as f: for it in ops: - f.write(json.dumps(it.to_dict()) + '\n') + f.write(json.dumps(it.to_dict()) + "\n") + def fix_filename(): pass @@ -279,18 +308,18 @@ def main(): jobs = [] for it in paths: if parsed := parse_release(it.name, known_actors=actors): - logging.debug(f'filename={it} release={parsed}') + logging.debug(f"filename={it} release={parsed}") suggested = parsed.to_filename() target = it.with_stem(suggested) if target.is_file(): - logging.error('target already exists') + logging.error("target already exists") continue jobs.append(MoveOp(source=it, target=target)) if not jobs: return - write_undo_script(paths[0].parent, jobs) + # write_undo_script(paths[0].parent, jobs) for it in jobs: it.target.hardlink_to(it.source) diff --git a/file_renamer_test.py b/file_renamer_test.py index f4acedb..4166a94 100644 --- a/file_renamer_test.py +++ b/file_renamer_test.py @@ -58,6 +58,10 @@ def test_parse_date(filename: str, date: datetime.date | None): @pytest.mark.parametrize(['filename', 'expected'], [ + [ + 'PenthouseGold.23.02.05.Tiffany.Tatum.XXX.720p.HEVC.x265.PRT', + Release(studio='DoctorAdventures', actors=['Jamie Michelle'], title='Nurse Jamie Knows Best', released_at=datetime.date(2021, 5, 26)), + ], [ 'Rebecca Volpetti -- @RealityKings -- Driving Him Crazy [1080p, x265] -- 2019-04-10', Release(actors=['Rebecca Volpetti'], studio='RealityKings', title='Driving Him Crazy', released_at=datetime.date(2019, 4, 10)), diff --git a/filejoker_downloader.py b/filejoker_downloader.py new file mode 100644 index 0000000..a916b05 --- /dev/null +++ b/filejoker_downloader.py @@ -0,0 +1,62 @@ +import argparse +import contextlib +import re + +from playwright.sync_api import sync_playwright, Page + + +def download_filejoker(url: str) -> dict: + with launch_browser() as page: + page.goto(url, referer='http://planetsuzy.org') + page.get_by_role("button", name="Slow Download").click() + page.get_by_role("button", name="Get Download Link").click() + with page.expect_response(re.compile('https://fs.+.filejoker.net/')) as res: + with page.expect_download(timeout=0) as dl: + page.get_by_role("button", name="Download File").click(timeout=0) + dl.value.cancel() + print(dl.value.url) + print(dl.value) + page.pause() + print() + + +def download_filefox(url: str) -> dict: + with launch_browser() as page: + page.goto(url, referer='http://planetsuzy.org') + page.get_by_role("button", name="Slow Download").click() + page.get_by_role("button", name="Slow Download").click(timeout=0) + with page.expect_response(re.compile('https://fs.+.filefox.net/')) as res: + with page.expect_download(timeout=0) as dl: + page.get_by_role("button", name="Download File").click(timeout=0) + dl.value.cancel() + print(dl.value.url) + print(dl.value) + page.pause() + print() + + +@contextlib.contextmanager +def launch_browser() -> Page: + with sync_playwright() as playwright: + with playwright.chromium.launch(headless=False) as browser: + with browser.new_context() as context: + with context.new_page() as page: + yield page + + +def parse_args(): + arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + arger.add_argument('url', help='URL to download', type=str) + return arger.parse_args() + + +def main(): + args = parse_args() + url = args.url + download_filejoker(url) + + +if __name__ == '__main__': + # download_filejoker('https://filejoker.net/ofg59z1pqjjb') + download_filefox('https://filefox.cc/qxd2yrxgu4ak') + main() diff --git a/gardener.py b/gardener.py deleted file mode 100644 index 7865c84..0000000 --- a/gardener.py +++ /dev/null @@ -1,207 +0,0 @@ -import dataclasses -import logging -import os -import webbrowser - - -def force_import(module: str): - import importlib - import subprocess - import sys - - try: - return importlib.import_module(module) - except ModuleNotFoundError: - subprocess.run([sys.executable, "-m", "pip", "install", module]) - importlib.invalidate_caches() - return importlib.import_module(module) - - -try: - import httpx -except ImportError: - torf = force_import("httpx") - -try: - import typer -except ImportError: - typer = force_import("typer") -try: - import rich -except ImportError: - rich = force_import("rich") -try: - import inquirer -except ImportError: - inquirer = force_import("inquirer") - -from rich import print -import rich.table - -JIRA_EMAIL = os.getenv('JIRA_EMAIL', 'abdussamet.kocak@akinon.com') -JIRA_TOKEN = os.getenv('JIRA_TOKEN', 'iAZhlJ1AdGwckbVllhlx48C4') -JIRA_URL = os.getenv('JIRA_URL', 'https://omniplatform.atlassian.net/') - -if not JIRA_TOKEN: - logging.info('') - webbrowser.open('https://id.atlassian.com/manage-profile/security/api-tokens') - - -@dataclasses.dataclass -class Issue: - id: str - key: str - summary: str - - -@dataclasses.dataclass -class PullRequest: - id: str - name: str - repository: str - branch: str - - -class Jira: - def __init__(self, email: str, token: str): - self.client = httpx.Client( - base_url=JIRA_URL, - auth=httpx.BasicAuth(email, token), - ) - - def list_issues_for_release(self) -> list[Issue]: - mergeable_status = 'In Review' - res = self.client.post( - '/rest/api/2/search', - json={ - 'jql': f'project = COM AND status = "{mergeable_status}" AND "Team[Dropdown]" = Backend ORDER BY created DESC' - }, - ) - - res.raise_for_status() - data = res.json() - development_type = ['Story', 'Bug', 'Task'] - return [ - Issue(id=it['id'], key=it['key'], summary=it['fields']['summary']) - for it in data['issues'] - if it['fields']['issuetype']['name'] in development_type - ] - - def get_approved_prs(self, issue_id: str): - res = self.client.post( - f'/jsw/graphql', - params={'operation': 'DevDetailsDialog'}, - json={ - "operationName": "DevDetailsDialog", - "query": """ - query DevDetailsDialog ($issueId: ID!) { - developmentInformation(issueId: $issueId){ - details { - instanceTypes { - repository { - name - branches { - name - pullRequests { - name - status - lastUpdate - } - reviews { - state - id - } - } - pullRequests { - id - name - branchName - status - reviewers{ - name - isApproved - } - } - } - danglingPullRequests { - id - name - branchName - status - reviewers{ - name - isApproved - } - } - } - } - } - } - """, - "variables": {"issueId": issue_id}, - }, - ) - res.raise_for_status() - data = res.json()['data'] - if not data['developmentInformation']['details']['instanceTypes']: - return [] - repo_name = data['developmentInformation']['details']['instanceTypes'][0]['repository'][0]['name'] - prs = data['developmentInformation']['details']['instanceTypes'][0]['danglingPullRequests'] - if not prs: - repo_name = data['developmentInformation']['details']['instanceTypes'][0]['repository'][0]['name'] - prs = data['developmentInformation']['details']['instanceTypes'][0]['repository'][0]['pullRequests'] - - return [ - PullRequest( - id=it['id'], - name=it['name'], - repository=repo_name, - branch=it['branchName'], - ) - for it in prs - if it['status'] != 'MERGED' - ] - - -console = rich.console.Console() - - -def render_issue(issue: Issue, prs: list[PullRequest]): - t = rich.table.Table( - title=f'{issue.key} -- {issue.summary}', - caption_justify='left', - ) - t.add_column("Repository", no_wrap=True) - t.add_column("Branch", no_wrap=True) - t.add_column("Pull Request") - - for it in prs: - t.add_row(it.repository, it.branch, it.name) - - console.print(t) - - -def main(): - logging.basicConfig(level=logging.INFO) - if not JIRA_EMAIL: - logging.error('JIRA_EMAIL is not set') - exit(1) - if not JIRA_TOKEN: - logging.error('JIRA_TOKEN is not set') - exit(1) - - j = Jira(email=JIRA_EMAIL, token=JIRA_TOKEN) - - logging.info('fetching issues in "ready to release" status') - issues = j.list_issues_for_release() - for it in issues: - logging.info(f'fetching pull requests for issue {it.key}') - prs = j.get_approved_prs(it.id) - if not prs: - logging.info(f'no non-merged pull requests found for issue {it.key}') - continue - render_issue(it, prs) - - -if __name__ == '__main__': - main() diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..8edb2f3 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/abdusco/playground + +go 1.23.0 + +require golang.org/x/sync v0.10.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..cf16d91 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= diff --git a/html_popup.py b/html_popup.py new file mode 100644 index 0000000..508b2b1 --- /dev/null +++ b/html_popup.py @@ -0,0 +1,133 @@ +import http.server +import random +import socketserver +import sys +import threading +import time +import typing +import webbrowser + +# language=html +inject_script = """ + +""" + + +def handler_for_html(html: str) -> typing.Type[http.server.SimpleHTTPRequestHandler]: + class Handler(http.server.SimpleHTTPRequestHandler): + def do_GET(self): + if self.path == "/": + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write(html.encode()) + else: + self.send_error(404) + + def do_POST(self): + if self.path == "/shutdown": + self.send_response(200) + self.end_headers() + self.server.last_heartbeat = 0 # Force shutdown + self.server.shutdown() + elif self.path == "/heartbeat": + self.send_response(200) + self.end_headers() + self.server.last_heartbeat = time.time() + else: + self.send_error(404) + + def log_message(self, format, *args): + # Suppress logging + pass + + return Handler + + +class ServerWithHeartbeat(socketserver.TCPServer): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.last_heartbeat = time.time() + + +def monitor_heartbeat(server: ServerWithHeartbeat, timeout: int = 2): + while True: + time.sleep(0.5) + if time.time() - server.last_heartbeat > timeout: + print("No heartbeat received, shutting down...") + server.shutdown() + break + + +def run_server(handler: http.server.SimpleHTTPRequestHandler, port: int = 8000): + with ServerWithHeartbeat(("", port), handler) as httpd: + print(f"Serving at port {port}") + + # Start heartbeat monitor in separate thread + monitor_thread = threading.Thread(target=monitor_heartbeat, args=(httpd,)) + monitor_thread.daemon = True + monitor_thread.start() + + httpd.serve_forever() + + +def main(): + html = sys.stdin.read() + if "" in html: + html_to_display = html.replace("", f"{inject_script}") + else: + html_to_display = f"{html}{inject_script}" + + handler = handler_for_html(html_to_display) + + # Start the server in a separate thread + port = random.randint(20000, 65535) + server_thread = threading.Thread(target=run_server, kwargs=dict(handler=handler, port=port)) + server_thread.daemon = True + server_thread.start() + + # Wait a moment for the server to start + time.sleep(0.5) + + # Open the web browser + webbrowser.open(f"http://localhost:{port}") + + # Wait for the server thread to finish + server_thread.join() + print("Server stopped") + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("\nReceived keyboard interrupt, exiting...") + raise SystemExit(0) diff --git a/htmlpopup.go b/htmlpopup.go new file mode 100755 index 0000000..8678552 --- /dev/null +++ b/htmlpopup.go @@ -0,0 +1,119 @@ +package main + +import ( + "context" + "fmt" + "io" + "log" + "net" + "net/http" + "os" + "os/exec" + "os/signal" + "runtime" + "strings" + "syscall" + "time" +) + +// language=html +const injectScript = ` + +` + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} + +func newHandler(html string, onShutdown func()) *http.ServeMux { + handler := http.NewServeMux() + + handler.HandleFunc("POST /shutdown", func(w http.ResponseWriter, r *http.Request) { + onShutdown() + }) + + handler.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.Write([]byte(html)) + }) + + return handler +} + +func run() error { + html, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("failed to read HTML from stdin: %w", err) + } + + htmlStr := string(html) + if strings.Contains(htmlStr, "") { + htmlStr = strings.ReplaceAll(string(html), "", injectScript+"") + } else { + htmlStr += injectScript + } + + listener, err := net.Listen("tcp", ":0") + if err != nil { + return fmt.Errorf("failed to create listener: %w", err) + } + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + port := listener.Addr().(*net.TCPAddr).Port + + server := &http.Server{Addr: listener.Addr().String()} + server.Handler = newHandler(htmlStr, func() { + cancel() + }) + + servedURL := fmt.Sprintf("http://localhost:%d", port) + + go func() { + log.Printf("serving at %s", servedURL) + if err := server.Serve(listener); err != nil && err != http.ErrServerClosed { + log.Fatalf("failed to serve: %v", err) + } + }() + + go func() { + // wait for server to start + time.Sleep(250 * time.Millisecond) + if err := openBrowser(servedURL); err != nil { + log.Fatalf("failed to open browser: %v", err) + } + }() + + <-ctx.Done() + + log.Printf("shutting down server") + + if err := server.Shutdown(ctx); err != nil { + return fmt.Errorf("failed to shutdown server: %w", err) + } + + return nil +} + +func openBrowser(url string) error { + var cmd *exec.Cmd + switch runtime.GOOS { + case "darwin": + cmd = exec.Command("open", url) + case "linux": + cmd = exec.Command("xdg-open", url) + case "windows": + cmd = exec.Command("cmd", "/c", "start", url) + default: + return fmt.Errorf("invalid URL: %s", url) + } + return cmd.Run() +} diff --git a/aws_codecommit.py b/legacy/aws_codecommit.py similarity index 100% rename from aws_codecommit.py rename to legacy/aws_codecommit.py diff --git a/aws_elasticache.py b/legacy/aws_elasticache.py similarity index 100% rename from aws_elasticache.py rename to legacy/aws_elasticache.py diff --git a/aws_es.py b/legacy/aws_es.py similarity index 100% rename from aws_es.py rename to legacy/aws_es.py diff --git a/aws_rds.py b/legacy/aws_rds.py similarity index 100% rename from aws_rds.py rename to legacy/aws_rds.py diff --git a/aws_redis_acl.py b/legacy/aws_redis_acl.py similarity index 100% rename from aws_redis_acl.py rename to legacy/aws_redis_acl.py diff --git a/aws_ses.py b/legacy/aws_ses.py similarity index 100% rename from aws_ses.py rename to legacy/aws_ses.py diff --git a/aws_ses_test.py b/legacy/aws_ses_test.py similarity index 100% rename from aws_ses_test.py rename to legacy/aws_ses_test.py diff --git a/multi_rename.py b/multi_rename.py new file mode 100644 index 0000000..33130b4 --- /dev/null +++ b/multi_rename.py @@ -0,0 +1,96 @@ +import argparse +import logging +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + + +def find_editor() -> str: + editor = os.getenv("EDITOR") + if editor: + return editor + + for candidate in ["micro", "nano"]: + if shutil.which(candidate): + return candidate + + return "nano" + + +def edit_text(text: str) -> str | None: + with tempfile.NamedTemporaryFile(suffix=".tmp", delete=False) as tf: + temp_file_path = Path(tf.name) + temp_file_path.write_text(text) + + old_modtime = temp_file_path.stat().st_mtime + subprocess.call([*find_editor().split(), str(temp_file_path)]) + new_modtime = temp_file_path.stat().st_mtime + if new_modtime == old_modtime: + logging.warning("editor exited without saving") + return None + + return temp_file_path.read_text() + + +def move_to_trash(file_path: Path): + if not file_path.is_file(): + raise FileNotFoundError + + subprocess.run( + [ + "osascript", + "-e", + f'tell app "Finder" to move POSIX file "{file_path}" to trash', + ], + check=True, + ) + + +def rename_files(files: list[Path], cwd: Path | None = None): + if not cwd: + cwd = files[0].parent + + originals = [f"{f.relative_to(cwd)}" for f in files] + edited_raw = edit_text("\n".join(originals)) + if not edited_raw: + return + edited = edited_raw.splitlines(keepends=False) + + deletables = [] + moveables = [] + for source, target in zip(originals, edited): + if target.startswith("#") or target == "": + deletables.append(Path(source)) + continue + if source != target: + moveables.append((cwd / source, cwd / target)) + + for source, target in moveables: + logging.info(f"moving {source} -> {target}") + source.rename(target) + + for f in deletables: + try: + logging.info(f"trashing {f}") + move_to_trash(f) + except: + logging.error(f"failed to trash {f}") + + +def parse_args(): + arger = argparse.ArgumentParser() + arger.add_argument("filenames", type=Path, nargs="+", help="File paths") + arger.add_argument("--cwd", type=Path, help="Current working directory") + return arger.parse_args() + + +def main(): + args = parse_args() + rename_files(args.filenames, cwd=args.cwd) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() diff --git a/openai_example_sentences.py b/openai_example_sentences.py new file mode 100644 index 0000000..d199807 --- /dev/null +++ b/openai_example_sentences.py @@ -0,0 +1,57 @@ +import argparse + +import httpx + +OPENAPI_TOKEN = "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" + + +http = httpx.Client( + base_url='https://api.openai.com/v1/', + headers={ + 'Authorization': f'Bearer {OPENAPI_TOKEN}', + }, + timeout=30, +) + + +def generate_examples(text: str) -> str: + payload = { + "messages": [ + { + "role": "system", + "content": """ +You are to provide colloquial and idiomatic German translations of given text. List 10 alternatives. + """, + }, + { + "role": "user", + "content": text, + }, + ], + "temperature": 0.7, + "max_tokens": 256, + "top_p": 1, + "frequency_penalty": 1.06, + "presence_penalty": 0.42, + "model": "gpt-3.5-turbo", + "stream": False, + } + res = http.post('/chat/completions', json=payload) + res.raise_for_status() + return res.json()['choices'][0]['message']['content'].strip() + + +def parse_args(): + arger = argparse.ArgumentParser() + arger.add_argument('prompt', help='Phrase to generate sentences for') + return arger.parse_args() + + +def main(): + args = parse_args() + response = generate_examples(args.prompt) + print(response) + + +if __name__ == '__main__': + main() diff --git a/realdebrid.py b/realdebrid.py index 71e5b90..7547b4c 100755 --- a/realdebrid.py +++ b/realdebrid.py @@ -36,6 +36,17 @@ def download_with_aria2(url: str, cwd: Path = Path.cwd(), extra_args: list[str] subprocess.run(args, text=True, cwd=str(cwd.resolve())) +def download_torrent(torrent_path: Path): + res = http.put('/torrents/addTorrent', content=torrent_path.read_bytes()) + res.raise_for_status() + data = res.json() + + res = http.get(f'/torrents/info/{data["id"]}') + res.raise_for_status() + data = res.json() + print() + + def parse_args(): arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) arger.add_argument('url', help='URL to unrestrict & download') @@ -66,4 +77,8 @@ def main(): if __name__ == "__main__": - main() + download_torrent( + Path( + '/Users/abdus/Downloads/[Empornium][PornWorld] All Her Holes Pounded Hard [Veronica Leal] [1080p] {Se7enSeas} [x265].torrent' + ) + ) diff --git a/rename_children.py b/rename_children.py new file mode 100755 index 0000000..a710768 --- /dev/null +++ b/rename_children.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3.10 + +import argparse +import dataclasses +import datetime +import json +import logging +from pathlib import Path +import re + + +def parse_args(): + arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + arger.add_argument('dirs', help='Path to the directories', type=Path, nargs='+') + return arger.parse_args() + + +def natural_sort_key(path: Path) -> tuple[str, ...]: + s = path.stem + return tuple(f'{int(p):010d}' if p.isnumeric() else p for p in re.findall(r'(\D+|\d+)', s)) + + +def rename_children(cwd: Path) -> None: + assert cwd.is_dir() + + files = [f for f in cwd.glob('*') if f.is_file()] + files = sorted(files, key=natural_sort_key) + + new_stem = cwd.name + + jobs = [] + for i, f in enumerate(files): + target = f.with_stem(f'{new_stem}__{i:04d}') + if f.suffix == '.jpeg': + target = target.with_suffix('.jpg') + if target.is_file(): + logging.error(f'target already exists: {target}') + continue + jobs.append(MoveOp(source=f, target=target)) + f.rename(target) + + if not jobs: + return + + write_undo_script(Path.cwd(), jobs) + + +@dataclasses.dataclass +class MoveOp: + source: Path + target: Path + + def to_dict(self) -> dict: + return { + 'source': str(self.source), + 'target': str(self.target), + } + + +def write_undo_script(save_dir: Path, ops: list[MoveOp]): + now = datetime.datetime.now().isoformat().replace(':', '') + save_path = save_dir / f'undo_{now}.json' + with save_path.open('w', encoding='utf-8') as f: + for it in ops: + f.write(json.dumps(it.to_dict()) + '\n') + + +def main(): + args = parse_args() + for d in args.dirs: + rename_children(d) + + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt index d62553e..cddb9ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,10 @@ beautifulsoup4==4.11.2 boto3==1.19.1 fastapi==0.86.0 httpx==0.23.3 -pytest==6.2.5 +pytest==7.2.1 typer==0.6.1 + +inquirer==3.1.2 +playwright==1.31.1 +torf==4.1.4 +bs4==0.0.1 \ No newline at end of file diff --git a/scan_vids.py b/scan_vids.py new file mode 100644 index 0000000..fd1c5f5 --- /dev/null +++ b/scan_vids.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "psycopg[binary]", +# ] +# /// +import argparse +import contextlib +import hashlib +import json +import logging +import subprocess +import typing +from pathlib import Path + +import psycopg +import psycopg.rows + +import ffmpeg + + +def hash_partial(f: Path) -> str: + sha1 = hashlib.sha1() + chunk_size = 1024 * 1024 * 10 # 10MB chunk size + + total_read = 0 + with f.open("rb") as file: + while chunk := file.read(chunk_size): + total_read += len(chunk) + sha1.update(chunk) + break # Only reads the first 10MB + + return f"sha1:{total_read}:{sha1.hexdigest()}" + + +@contextlib.contextmanager +def connect_db() -> typing.Generator[psycopg.Connection, typing.Any, typing.Any]: + with psycopg.connect("postgres://abdus:abdus@db.abdus.dev:5444/smut?sslmode=disable") as conn: + conn.row_factory = psycopg.rows.dict_row + yield conn + + +def video_exists(con: psycopg.Connection, file_path: str) -> bool: + stmt = con.execute( + "UPDATE videos SET last_seen_at = NOW() WHERE file_path = %s RETURNING true AS exists", + [file_path], + ) + row = stmt.fetchone() + return row["exists"] + + +def delete_video(con: psycopg.Connection, file_path: str) -> bool: + con.execute("DELETE FROM videos WHERE file_path = %s", [file_path]) + + +def save_video( + con: psycopg.Connection, + category: str, + file_path: str, + ffprobe: dict, + hash_partial: str, +): + sql = """INSERT INTO videos(category, file_path, ffprobe, hash_partial) VALUES (%s, %s, %s, %s) ON CONFLICT(file_path) SET last_seen_at = NOW();""" + con.execute( + sql, + [ + category, + file_path, + json.dumps(ffprobe), + hash_partial, + ], + ) + + +def save_error(con: psycopg.Connection, file_path: str, stderr: str): + sql = """INSERT INTO scan_state(file_path, stderr) VALUES (%s, %s) ON conflict do nothing""" + con.execute( + sql, + [ + file_path, + stderr, + ], + ) + + +def scan_videos(con: psycopg.Connection, video_paths: list[Path]) -> None: + for i, f in enumerate(video_paths): + if "/_picks/" in str(f): + continue + progress = f"[{i + 1}/{len(video_paths)}]" + if f.is_dir(): + continue + + file_path = str(f.absolute()) + + if video_exists(con, file_path=file_path): + logging.debug(f"{progress} video already exists {f=}") + continue + + logging.debug(f"probing {f=}") + try: + probed = ffmpeg.ffprobe(f) + except subprocess.CalledProcessError as e: + save_error(con, file_path=file_path, stderr=e.stderr) + continue + + file_hash = hash_partial(f) + + save_video( + category=category_for_file(f), + con=con, + file_path=file_path, + ffprobe=probed.__dict__, + hash_partial=file_hash, + ) + logging.info(f"{progress} saved {f=}") + con.commit() + + +def category_for_file(file_path: Path) -> str: + if str(file_path).startswith("/Volumes/"): + return f"downloaded.{file_path.parts[1].lower()}" + if str(file_path).startswith("/mnt/box"): + return "box" + raise ValueError(f"unknown category for {file_path}") + + +def find_copies(con: psycopg.Connection, video_path: Path) -> list[str]: + probed = ffmpeg.ffprobe(video_path) + + sql = """ +SELECT + file_path +FROM videos +WHERE + file_path != %(file_path)s + AND ( + hash_partial = %(hash_partial)s + OR (abs(duration_sec - %(duration_sec)s) < 0.1 AND abs(size_bytes - %(size_bytes)s) < 3000000) + ); + """ + + stmt: psycopg.Cursor = con.execute( + sql, + { + "file_path": str(video_path.absolute()), + "hash_partial": hash_partial(video_path), + "duration_sec": probed.duration_sec, + "size_bytes": probed.size_bytes, + }, + ) + rows = stmt.fetchall() + + return [it["file_path"] for it in rows] + + +def parse_args(): + arger = argparse.ArgumentParser() + arger.add_argument("video_paths", nargs="+", type=Path, help="Path to the videos") + arger.add_argument("--debug", action="store_true", help="Path to the videos") + + return arger.parse_args() + + +def main(): + logging.basicConfig(format=logging.BASIC_FORMAT, level=logging.INFO) + args = parse_args() + if args.debug: + logging.getLogger().setLevel(logging.DEBUG) + + video_paths: list[Path] = args.video_paths + if not video_paths: + logging.info("no files to scan") + return + + with connect_db() as con: + scan_videos(con=con, video_paths=video_paths) + + +if __name__ == "__main__": + main() diff --git a/tekton_api.py b/tekton_api.py deleted file mode 100644 index 97e1fa5..0000000 --- a/tekton_api.py +++ /dev/null @@ -1,190 +0,0 @@ -import functools -import hashlib -import json -import textwrap -import typing -import uuid -from pathlib import Path - -import kubernetes.config -import yaml - - -def make_client() -> kubernetes.client.CoreV1Api: - kubernetes.config.load_kube_config('/Users/abdus/Desktop/kubeconfig-kube.yml') - return kubernetes.client.CoreV1Api() - - -class Kubernetes: - def __init__(self, client: kubernetes.client.CoreV1Api, namespace='default'): - self.client = client - self.custom_objects = kubernetes.client.CustomObjectsApi(self.client.api_client) - self.namespace = namespace - - @classmethod - def from_kubeconfig(cls, kubeconfig_path: str, namespace: str) -> 'Kubernetes': - kubernetes.config.load_kube_config(kubeconfig_path) - return cls(kubernetes.client.CoreV1Api(), namespace=namespace) - - def read_logs(self, pod_name: str, container_name: typing.Optional[str] = None, timestamps: bool = False) -> str: - return self.client.read_namespaced_pod_log( - pod_name, - self.namespace, - container=container_name, - timestamps=timestamps, - pretty=True, - ) - - def get_tekton_object(self, plural: str, name: str = None): - getter = functools.partial( - self.custom_objects.get_namespaced_custom_object, - group='tekton.dev', - version='v1beta1', - namespace=self.namespace, - ) - return getter(plural=plural, name=name) - - def list_tekton_object(self, plural: str): - getter = functools.partial( - self.custom_objects.list_namespaced_custom_object, - group='tekton.dev', - version='v1beta1', - namespace=self.namespace, - ) - return getter(plural=plural)['items'] - - def create_from_yaml(self, yaml_text: str) -> list[dict]: - specs = list(yaml.safe_load_all(yaml_text)) - return kubernetes.utils.create_from_yaml( - kubernetes.client.CustomObjectsApi(self.client.api_client), - yaml_objects=specs, - namespace=self.namespace, - ) - - -def read_logs(): - task_hello_world = k8s.get_tekton_object(plural='tasks', name='echo-hello-world') - taskruns = k8s.list_tekton_object(plural='taskruns') - - pod_name = taskruns['items'][0]['status']['podName'] - container_name = taskruns['items'][0]['status']['containerName'] - logs = k8s.read_logs(pod_name, container_name) - print(logs) - - -k8s = Kubernetes.from_kubeconfig('/Users/abdus/Desktop/kubeconfig.yaml', namespace='api-server') - - -def get_pipelineru_status(): - result = k8s.get_tekton_object('pipelineruns', name='pr-2rr6t') - status = result['status']['conditions'][0]['status'] - reason = result['status']['conditions'][0]['reason'] - failed_taskruns = [ - name for name, it in result['status']['taskRuns'].items() if it['status']['conditions'][0]['status'] == 'False' - ] - - logs = [] - for it in result['status']['taskRuns'].values(): - task_name = it['pipelineTaskName'] - task_status = it['status']['conditions'][0]['reason'] - pod_name = it['status']['podName'] - - if task_status != 'Failed': - continue - - # fetch all logs for all steps to get a better picture of the error - # (regardless of whether it is successful or not) - for i, step in enumerate(it['status']['steps'], start=1): - container_name = step['container'] - header = f'# {task_name}, step {i}: {step["name"]}' - logs.append('#' * len(header)) - logs.append(header) - logs.append('#' * len(header)) - step_logs = k8s.read_logs(pod_name, container_name, timestamps=True) - logs.append(step_logs) - - log_text = '\n'.join(logs) - return { - 'pipelinerun_name': result['metadata']['name'], - 'namespace': result['metadata']['namespace'], - 'status': status, - 'reason': reason, - 'failed_taskruns': failed_taskruns, - 'logs': log_text, - } - - -def create_taskrun(): - k = kubernetes.client.CustomObjectsApi() - res = k.create_namespaced_custom_object( - group='tekton.dev', - version='v1beta1', - namespace='default', - plural='taskruns', - body={ - 'apiVersion': 'tekton.dev/v1beta1', - 'kind': 'TaskRun', - 'metadata': {'name': 'testing1234' + uuid.uuid4().hex}, - 'spec': { - 'taskRef': {'name': 'task-with-json'}, - 'params': [ - { - 'name': 'json_value', - 'value': '{"foo": "bar", "nested": {"a": "1"}}', - } - ], - }, - }, - ) - print(res) - - -def migrate_pipelines(): - k = kubernetes.client.CustomObjectsApi() - - pipeline_yaml = Path('sample_pipeline.yaml').read_text() - parsed_pipeline = yaml.safe_load(pipeline_yaml) - - params = dict(group='tekton.dev', version='v1beta1', namespace='default', plural='pipelines') - k.delete_namespaced_custom_object( - **params, - name=parsed_pipeline['metadata']['name'], - ) - res = k.create_namespaced_custom_object( - **params, - body=parsed_pipeline, - ) - return parsed_pipeline - - -def trigger_pipeline(): - k = kubernetes.client.CustomObjectsApi() - pipeline = migrate_pipelines() - pipeline_name = pipeline['metadata']['name'] - - payload = { - 'image': 'nginx', - 'json_value': {'nested': {'nested2': {'a': 1}}}, - } - - res = k.create_namespaced_custom_object( - group='tekton.dev', - version='v1beta1', - namespace='default', - plural='pipelineruns', - body={ - 'apiVersion': 'tekton.dev/v1beta1', - 'kind': 'PipelineRun', - 'metadata': {'name': 'deploy-app-' + uuid.uuid4().hex}, - 'spec': { - 'pipelineRef': {'name': pipeline_name}, - 'params': [ - {'name': k, 'value': json.dumps(v) if not isinstance(v, str) else v} for k, v in payload.items() - ], - }, - }, - ) - - -if __name__ == '__main__': - main() diff --git a/tormod.py b/tormod.py new file mode 100755 index 0000000..19544cb --- /dev/null +++ b/tormod.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3.11 +import argparse +import io +from pathlib import Path + +import torf + + +def parse_args() -> dict: + arger = argparse.ArgumentParser() + arger.add_argument('torrent_path', type=Path, help='Path to torrent file') + arger.add_argument('--name', type=Path, help='New filename', required=True) + return dict(arger.parse_args().__dict__) + + +def modify_torrent( + torrent_file: bytes, + name: str, + announce_url: str, + created_by: str = '', + comment: str = '', + private: bool = True, +) -> bytes: + t = torf.Torrent.read_stream(io.BytesIO(torrent_file)) + + if len(t.files) > 1: + raise ValueError('Torrent contains multiple files') + + t.name = name + t.private = private + t.comment = comment + t.created_by = created_by + t.trackers.clear() + t.trackers.insert(0, announce_url) + + return t.dump() + + +def main(): + args = parse_args() + torrent_path: Path = args.pop('torrent_path') + modded = modify_torrent( + **args, + torrent_file=torrent_path.read_bytes(), + announce_url='http://tracker.empornium.sx:2710/tegqucis10uanp672qh6nhn393xlkncs/announce', + created_by='zzzp', + comment='created by zzzp', + private=True, + ) + save_path = torrent_path.with_stem(f'{torrent_path.stem}.mod') + save_path.write_bytes(modded) + + +if __name__ == '__main__': + main() diff --git a/torrentify.py b/torrentify.py index 0bc8ef3..7c98081 100755 --- a/torrentify.py +++ b/torrentify.py @@ -4,6 +4,7 @@ import json import logging import os import re +import select import shutil import sys import time @@ -210,7 +211,10 @@ def upload_image(image_path: Path) -> str: return image_url -ParsedFilename = typing.TypedDict('ParsedFilename', {'actors': list[str], 'studio': str, 'date': str, 'title': str, 'tags': list[str]}) +ParsedFilename = typing.TypedDict( + 'ParsedFilename', {'actors': list[str], 'studio': str, 'date': str, 'title': str, 'tags': list[str]} +) + def parse_filename(filename: str) -> ParsedFilename: parsed = { @@ -244,6 +248,7 @@ def parse_filename(filename: str) -> ParsedFilename: return parsed + def generate_title(parsed: ParsedFilename) -> str: parts = [] if actors := parsed.get('actors'): @@ -473,7 +478,8 @@ def main(): thumbnail_path = video_path.with_suffix(".jpg") if old_thumb_path.is_file(): old_thumb_path.rename(thumbnail_path) - ffmpeg.make_thumbnail_tile(video_path, image_path=thumbnail_path, skip_if_exists=True) + image_jpeg = ffmpeg.make_thumbnail_tile(video_path) + thumbnail_path.write_bytes(image_jpeg) logging.info(f"Saved thumbnails at {thumbnail_path}") post_bbcode = generate_post_bbcode(video_path, thumbnail_path) diff --git a/torrentify_test.py b/torrentify_test.py deleted file mode 100644 index 72908b1..0000000 --- a/torrentify_test.py +++ /dev/null @@ -1,60 +0,0 @@ -from cmath import exp -from torrentify import generate_title, parse_filename, ParsedFilename -import pytest - - -@pytest.mark.parametrize( - ["filename", "expected"], - [ - [ - "To Ki -- @Studio -- 2022-10-04", - { - "actors": ["To Ki"], - "date": "2022-10-04", - "studio": "Studio", - "tags": ["to.ki", "studio.com", "2022", "2022.10"], - }, - ], - [ - "To Ki, Ki To -- @Studio -- Title -- 2022-10-04", - { - "actors": ["To Ki", "Ki To"], - "date": "2022-10-04", - "studio": "Studio", - "title": "Title", - "tags": ["to.ki", "ki.to", "studio.com", "2022", "2022.10"], - }, - ], - ], -) -def test_parse_filename(filename: str, expected: dict): - parsed = parse_filename(filename) - print(filename, parsed) - assert parsed == expected - - -@pytest.mark.parametrize( - ["parsed", "expected"], - [ - [ - { - "actors": ["To Ki"], - "date": "2022-10-04", - "studio": "Studio", - }, - 'To Ki -- @Studio -- 2022-10-04', - ], - [ - { - "actors": ["To Ki", "Ki To"], - "date": "2022-10-04", - "title": "Title", - "studio": "Studio", - }, - 'To Ki, Ki To -- @Studio -- Title -- 2022-10-04', - ], - ], -) -def test_generate_title(parsed: ParsedFilename, expected: str): - generated = generate_title(parsed) - assert generated == expected