abdus/playground
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

snapshot

Browse Source
This commit is contained in:
abdus
2024-12-24 20:49:41 +03:00
parent f8febafacb
commit 2b3521544b
35 changed files with 2043 additions and 767 deletions
Download Patch File Download Diff File Expand all files Collapse all files
anki.creds.json
+1
View File
@@ -0,0 +1 @@
{"username": "sssmmt@gmail.com", "password": "2126484"}
berlin_immigration.py
+34 -27
View File
@@ -1,12 +1,12 @@
import argparse import argparse
import asyncio import asyncio
import contextlib import contextlib
import dataclasses
import datetime import datetime
import logging import logging
import random import random
import re import re
import subprocess import subprocess
import time
import urllib.parse import urllib.parse
from pathlib import Path from pathlib import Path
@@ -39,7 +39,16 @@ dump_dir = Path('~/Desktop').expanduser() / 'immigration'
dump_dir.mkdir(parents=True, exist_ok=True) dump_dir.mkdir(parents=True, exist_ok=True)
async def check_slots(page: Page, retry_count: int = 1) -> list[datetime.datetime]: @dataclasses.dataclass
class ResidencyApplication:
citizenship_country: str
num_applicants: int
is_living_with_family: bool
residency_category: str
residency_purpose: str
async def check_slots(page: Page, values: ResidencyApplication, retry_count: int = 1) -> list[datetime.datetime]:
await page.add_init_script('''Object.defineProperty(navigator, 'webdriver', { get: () => false })''') await page.add_init_script('''Object.defineProperty(navigator, 'webdriver', { get: () => false })''')
window_id = random.randint(1000, 9999) window_id = random.randint(1000, 9999)
req_id = random.randint(0, 999) req_id = random.randint(0, 999)
@@ -47,45 +56,37 @@ async def check_slots(page: Page, retry_count: int = 1) -> list[datetime.datetim
f'https://otv.verwalt-berlin.de/ams/TerminBuchen/wizardng?dswid={window_id}&dsrid={req_id}', f'https://otv.verwalt-berlin.de/ams/TerminBuchen/wizardng?dswid={window_id}&dsrid={req_id}',
wait_until='networkidle', wait_until='networkidle',
) )
async with page.expect_navigation():
await page.click('.langBlock [name="txtEn"] a')
await page.check('[name="gelesen"]') await page.check('[name="gelesen"]')
async with page.expect_navigation(url=re.compile('st=2')):
async with page.expect_navigation(url=re.compile('st=2'), wait_until='networkidle'):
await page.click('[name="applicationForm:managedForm:proceed"]') await page.click('[name="applicationForm:managedForm:proceed"]')
await page.get_by_role("combobox", name="Staatsangehörigkeit *").select_option(label="Türkei") await page.get_by_role("combobox", name="Citizenship *").select_option(label=values.citizenship_country)
await asyncio.sleep(0.3) await asyncio.sleep(0.3)
await page.get_by_role( # how many applicants
"combobox", await page.select_option('[name="personenAnzahl_normal"]', value=str(values.num_applicants))
name="Anzahl der Personen, die einen Aufenthaltstitel beantragen (auch ausländische Ehepartner und Kinder) *",
).select_option(label="eine Person")
await asyncio.sleep(0.3) await asyncio.sleep(0.3)
await page.get_by_role( # living with family
"combobox", name="Leben Sie in Berlin zusammen mit einem Familienangehörigen (z.B. Ehepartner, Kind) *" await page.select_option('[name="lebnBrMitFmly"]', value='1' if values.is_living_with_family else '2')
).select_option(label="nein")
await asyncio.sleep(0.3) await asyncio.sleep(0.3)
await page.get_by_text("Aufenthaltstitel - beantragen").click() await page.get_by_text("Apply for a residence title").click()
await page.locator("label").filter(has_text="Erwerbstätigkeit").click() await page.locator("label").filter(has_text=values.residency_category).click()
await page.get_by_text("Blaue Karte EU (§ 18b Abs. 2)").click() await page.get_by_text(values.residency_purpose).click()
stop_at = datetime.datetime.now() + datetime.timedelta(minutes=28) stop_at = datetime.datetime.now() + datetime.timedelta(minutes=28)
async def go_forward():
async with page.expect_navigation(url=re.compile(r'st='), wait_until='networkidle', timeout=60_000):
await page.get_by_role("button", name="Weiter").click()
if 'st=2' in page.url:
error_message = await page.inner_text('.errorMessage')
if 'keine Termine frei' in error_message:
return False
raise Error(error_message)
return 'st=3' in page.url
while 'st=2' in page.url: while 'st=2' in page.url:
retry_count -= 1 retry_count -= 1
async with page.expect_navigation(url=re.compile(r'st='), wait_until='networkidle', timeout=60_000): async with page.expect_navigation(url=re.compile(r'st='), wait_until='networkidle', timeout=60_000):
await page.get_by_role("button", name="Weiter").click() await page.get_by_role("button", name="Next").click()
if datetime.datetime.now() > stop_at: if datetime.datetime.now() > stop_at:
raise TimeoutError('could not find a slot') raise TimeoutError('could not find a slot')
@@ -134,7 +135,14 @@ async def main():
args = parse_args() args = parse_args()
async with launch_browser(headless=args.headless) as page: async with launch_browser(headless=args.headless) as page:
try: try:
slots = await check_slots(page, retry_count=args.retry_count) values = ResidencyApplication(
citizenship_country='India',
num_applicants=1,
is_living_with_family=False,
residency_category='Educational purposes',
residency_purpose='Residence permit for the purpose of studying (sect. 16b)',
)
slots = await check_slots(page, values=values, retry_count=args.retry_count)
if not slots: if not slots:
print('no slots yet') print('no slots yet')
return return
@@ -146,4 +154,3 @@ async def main():
if __name__ == '__main__': if __name__ == '__main__':
asyncio.run(main()) asyncio.run(main())
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
main()
clean_vids.py
+782
View File
@@ -0,0 +1,782 @@
import contextlib
import hashlib
import json
import logging
import re
import sqlite3
import subprocess
import typing
from collections import Counter, defaultdict
from pathlib import Path
import paramiko
import psycopg
import psycopg.rows
import ffmpeg
VIDS_DIR = Path(r"/Volumes/FIVER/_ingress/")
schema = """
create table if not exists videos(
file_path text not null primary key,
size_bytes int not null,
hash_partial text not null,
ffprobe json not null
);
create table if not exists scan_state(
file_path text not null primary key,
stderr text
);
"""
def find_vids(root: Path):
extensions = [".mkv", ".mp4"]
for ext in extensions:
for f in root.rglob(f"*{ext}"):
yield f
def hash_partial(f: Path) -> str:
sha1 = hashlib.sha1()
chunk_size = 1024 * 1024 * 10 # 10MB chunk size
total_read = 0
with f.open("rb") as file:
while chunk := file.read(chunk_size):
total_read += len(chunk)
sha1.update(chunk)
break # Only reads the first 10MB
return f"sha1:{total_read}:{sha1.hexdigest()}"
@contextlib.contextmanager
def connect_db_sqlite() -> typing.Generator[psycopg.Connection, typing.Any, typing.Any]:
with sqlite3.connect("vids.sqlite3") as conn:
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA foreign_keys = 1")
yield conn
@contextlib.contextmanager
def connect_db() -> typing.Generator[psycopg.Connection, typing.Any, typing.Any]:
with psycopg.connect("postgres://abdus:abdus@db.abdus.dev:5444/smut?sslmode=disable") as conn:
conn.row_factory = psycopg.rows.dict_row
yield conn
def video_exists(con: psycopg.Connection, file_path: str) -> bool:
stmt = con.execute(
"select exists(select 1 from videos where file_path = %s) as exists",
[file_path],
)
row = stmt.fetchone()
return row["exists"]
def delete_video(con: psycopg.Connection, file_path: str) -> bool:
con.execute("delete from videos where file_path = %s", [file_path])
def save_video(
con: psycopg.Connection,
category: str,
file_path: str,
ffprobe: dict,
hash_partial: str,
):
sql = """insert into videos(category, file_path, ffprobe, hash_partial) values (%s, %s, %s, %s);"""
con.execute(
sql,
[
category,
file_path,
json.dumps(ffprobe),
hash_partial,
],
)
def save_error(con: psycopg.Connection, file_path: str, stderr: str):
sql = """insert into scan_state(file_path, stderr) values (%s, %s) on conflict do nothing"""
con.execute(
sql,
[
file_path,
stderr,
],
)
def scan_videos():
logging.basicConfig(format=logging.BASIC_FORMAT, level=logging.DEBUG)
with connect_db() as con:
# with con:
# con.executescript(schema)
all_vids = list(find_vids(VIDS_DIR))
for i, f in enumerate(all_vids):
if "/_picks/" in str(f):
continue
progress = f"[{i+1}/{len(all_vids)}]"
if f.is_dir():
continue
file_path = str(f.absolute())
if video_exists(con, file_path=file_path):
logging.debug(f"{progress} video already exists {f=}")
continue
logging.debug(f"probing {f=}")
try:
probed = ffmpeg.ffprobe(f)
except subprocess.CalledProcessError as e:
save_error(con, file_path=file_path, stderr=e.stderr)
continue
file_hash = hash_partial(f)
save_video(
category="downloaded.fiver",
con=con,
file_path=file_path,
ffprobe=probed.__dict__,
hash_partial=file_hash,
)
logging.info(f"{progress} saved {f=}")
con.commit()
def clean_records():
all_vids = set(map(str, find_vids(VIDS_DIR)))
with connect_db() as con:
for row in con.execute("select file_path from videos"):
file_path = row["file_path"]
if file_path not in all_vids:
delete_video(con, file_path=row["file_path"])
def unwanted(dry_run: bool):
take = set(
r"""
aidra fox
rachel starr
abella danger
sata jones
octavia red
paige owens
lily lou
cj miles
arabelle raphael
chloe amour
erin everheart
kenzie reeves
alex grey
alicia trece
angel youngs
angela white
april olsen
blake blossom
brandy renee
chantal
demi sutra
eden ivy
ema karter
emma hix
gianna dior
hazel moore
jazmin luv
jessy jey
josephine jackson
katsaros
kira noir
kitana lure
kitana montana
kristy black
leal
lia lin
lily labeau
liya silver
maddy may
nata ocean
rebel rhyder
rein
savannah bond
scarlett alexis
scarlit scandal
shalina
simon kitty
slimthick vic
stratton
tatum
tommy king
vanna bardot
vic marie
violet starr
vittoria
emily willis
THUNDERCOCK
lucky bee
charlotte sins
mandy muse
skye blue
alexis tae
willow
TRUEANAL
ALLANAL
avery cristy
ANALONLY
SWALLOWED
kelly collins
nicole black
xxlayna marie
katrina colt
allie addison
DIRTYAUDITIONS
scarlet chase
#sloppiest kiss
liz ocean
medusa
nicole aria
stacy bloom
lara frost
jamie jett
brooklyn gray
kimmy granger
kelsey kane
kay lovely
ivy wolfe
stefany kyler
leo ahsoka
lydia black
renee rose
emily jade
anya olsen
alexis crystal
sybil
megan rain
charlotte sartre
madison ivy
maddy black
gina valentina
lacey jayne
hailey rose
graycee baybee
katie kush
lilu moon
juis wild
kylie rocket
dolly dyson
nichole saphir
Octokuro
reislin
isabella de laa
qween goddess
INTHECRACK
athenea rose
daniela garcia
kama oxi
mina
baby nicols
jade amor
daniela ortiz
NYMPHO
""".strip().splitlines(keepends=False)
)
drop = set(
r"""
ADDICTED2GIRLS
ASIANSEEPLOITED
BANANAFEVER
BBCPOVD
BFFS
# BRAZZERSEXXTRA
BRIDE4K
CHLOELAMOUR
DRKINLA
ETERNALDESIRE
EVERYTHINGBUTT
FAKEHOSTEL
FAKEHUBORIGINALS
FAKETAXI
FAMILYSINNERS
FEETISHPOV
GIRLGIRLXXX
GOTFILLED
HOOKUPHOTSHOT
HOWWOMENORGASM
IMMORALLIVE
IWANTCLIPS
JACQUIEETMICHELTV
JERKAOKE
NICONICE
NURUMASSAGE
OHMYHOLES
OYELOCA
PASCALSSUBSLUTS
SCHOOLOFCOCK
SHOTHERFIRST
SLIPPERYMASSAGE
TABOOHEAT
TEAMSKEETX
WAXXXED
WOODMANCASTINGX
PISSVIDS
LEGALPORNO
ANALVIDS
PORNBOX
\btap\b
alice xo
alicia williams
mona azar
anna de ville
baby kxtten
bella rolland
betzz
charlie forde
charlie red
cheyla collins
claudia garcia
cory chase
dani blue
dellai
eimy zoren
emily pink
gia dibella
greta foss
danielle renae
harley king
ivy maddox
sasha rose
olive glass
katherin moore
kristina grace
kylie page
larissa leite
lily starfire
lina arian
luna legend
luna rishi
luna wolfs
lya cutie
margan lee
marie berger
marilyn johnson
mia cheers
monika fox
moona snake
nichole saphir
nicole murkovski
olivia sparkle
polly petrova
pornworld
publicagent
publicbang
stacy cruz
sussy sweet
triple anal
vanessa vega
venera maxima
victoria nyx
whitney wright
zirael rem
""".lower()
.strip()
.splitlines(keepends=False)
)
take_patterns = [re.compile(p.strip().replace(" ", "."), flags=re.I) for p in take if not p.strip().startswith("#")]
drop_patterns = [re.compile(p.strip().replace(" ", "."), flags=re.I) for p in drop if not p.strip().startswith("#")]
to_delete = set()
with connect_db() as con:
for row in con.execute("select file_path from videos where category = 'box'"):
file_path = Path(row["file_path"])
text = re.sub(r"[.]+", " ", file_path.stem.lower())
if any(p.search(text) for p in drop_patterns):
to_delete.add(file_path)
if any(p.search(text) for p in take_patterns):
to_delete.discard(file_path)
for it in sorted(to_delete):
print(it)
return
with connect_db() as con:
for path in sorted(to_delete, key=str):
try:
print(path.stem)
con.execute(
"update videos set marked_for_deletion_at = now() where file_path = %s",
[str(path)],
)
if not dry_run:
path.unlink()
delete_video(con, file_path=str(path))
con.commit()
except:
pass
print(len(to_delete))
def large_unwanted():
with connect_db() as con:
for row in con.execute("select file_path from videos order by size_bytes desc limit 500"):
file_path = Path(row["file_path"])
print(file_path)
def list_studios():
re_studio = re.compile(r"^(.+?)\.\d+", flags=re.IGNORECASE)
seen = defaultdict(int)
with connect_db() as con:
for row in con.execute("select file_path from videos"):
file_path = Path(row["file_path"])
m = re_studio.search(file_path.stem.lower())
if m:
seen[m.group(1)] += 1
for it, n in Counter(seen).most_common(20):
print(it, n)
def lower_resolutions(dry_run: bool):
with connect_db() as con:
for row in con.execute("""select file_path from videos where file_path like '%720p%';"""):
file_path = Path(row["file_path"])
video_1080p_path = file_path.with_stem(file_path.stem.replace("720p", "1080p"))
if video_exists(con, file_path=str(video_1080p_path)) and video_1080p_path.is_file():
print(file_path)
if not dry_run:
file_path.unlink()
def dupes():
sql = """
with durationed as (
select v.*, json_extract(v.ffprobe, '$.duration_sec') as duration_sec from videos v
)
select v.file_path, dupe.file_path from durationed v
join durationed dupe on v.file_path != dupe.file_path and abs(dupe.duration_sec - v.duration_sec) < 3
where abs(dupe.size_bytes - v.size_bytes) < 3000000
"""
with connect_db() as con:
for row in con.execute(sql):
print(row[0], row[1])
print()
def different_format(dry_run: bool):
sql = """
select file_path from videos where file_path like '%.mkv'
"""
with connect_db() as con:
for row in con.execute(sql):
mkv_path = Path(row["file_path"])
mp4_path = mkv_path.with_suffix(".mp4")
if mkv_path.is_file() and mp4_path.is_file():
if abs(mp4_path.stat().st_size - mkv_path.stat().st_size) < 5_000_000:
print(mkv_path)
if not dry_run:
mkv_path.unlink()
def count_shared_words(a: str, b: str) -> int:
a_words = set(re.split(r"[\s\-.]+", a.lower()))
b_words = set(re.split(r"[\s\-.]+", b.lower()))
return len(a_words.intersection(b_words))
def reencoded_self(dry_run: bool):
sql = """
with
durationed_v as (
select v.*, json_extract(v.ffprobe, '$.duration_sec') as duration_sec from videos v
),
durationed_dl as (
select v.*, json_extract(v.ffprobe, '$.duration_sec') as duration_sec from downloaded_videos v
)
select v.file_path, dlv.file_path from durationed_v v
join durationed_dl dlv on abs(v.duration_sec - dlv.duration_sec) < 0.1
where dlv.file_path like '%/_reenc/%'
"""
with connect_db() as con:
for row in con.execute(sql):
v_path = Path(row[0])
dlv_path = Path(row[1])
if count_shared_words(v_path.stem, dlv_path.stem) < 2:
continue
print(v_path.name, dlv_path.name, sep="\n")
if not dry_run:
try:
v_path.unlink()
except:
pass
def local_duped(dry_run: bool):
sql = """
with durationed_dl as (
select v.*, json_extract(v.ffprobe, '$.duration_sec') as duration_sec from downloaded_videos v
)
select v.file_path, dupe.file_path
from durationed_dl v
join durationed_dl dupe on
v.id != dupe.id
and v.id > dupe.id
and abs(dupe.duration_sec - v.duration_sec) < 0.1
and abs(dupe.size_bytes - v.size_bytes) < 3000000
"""
root = Path(r"/Volumes/FIVER/_ingress/")
with connect_db() as con:
for row in con.execute(sql):
a_path = Path(row[0])
b_path = Path(row[1])
if count_shared_words(a_path.stem, b_path.stem) < 6:
continue
# print(a_path.name, b_path.name, sep='\n')
keep, remove = a_path, b_path
if len(b_path.stem) > len(a_path.stem):
keep, remove = b_path, a_path
print(" # ", "matching", keep.name)
print(" rm", f'"{remove}"')
print()
def delete_dupes():
rows = []
with connect_db() as con:
for row in rows:
file_path = Path(row["file_path"])
try:
file_path.unlink()
delete_video(con, file_path=row["file_path"])
con.commit()
except:
pass
def save_suggested_names():
rows = []
with connect_db() as con:
for row in rows:
bandaid_path = Path(row["file_path"])
fiver_path = Path(row["fiver_path"])
if bandaid_path.stem == fiver_path.stem:
continue
if count_shared_words(bandaid_path.stem, fiver_path.stem) < 3:
print("unsimilar", row)
continue
if bandaid_path.stem.count("--") == 3:
better_stem = bandaid_path.stem
else:
better_stem = fiver_path.stem
better_stem = re.sub(r"(\d{4})(\d{2})(\d{2})", r"\1-\2-\3", better_stem)
if better_stem == fiver_path.stem:
continue
print(fiver_path.name, better_stem, sep="\n")
con.execute(
"update videos set suggested_filename = %s where file_path = %s",
[better_stem, fiver_path.name],
)
con.commit()
def delete_remote_files():
rows = [
{"file_path": "/mnt/box/files/_raw/prt/5KPorn.24.10.02.Leya.Desantis.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/AllAnal.24.10.30.Aviana.Violet.And.Nicole.Aria.XXX.720p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/AllAnal.24.11.07.Khloe.Kingsley.And.Cherry.Kiss.XXX.720p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/AnalOnly.24.11.26.Aria.Sloane.And.Nicole.Doshi.XXX.720p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/AnalTherapyXXX.24.02.10.Willow.Ryder.Family.Competition.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/AngelsLove.23.07.29.Cherry.Candle.And.Stacy.Cruz.Hot.Juice.Is.Dripping.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/BBCPie.20.05.31.Hazel.Moore.Morning.Wood.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/BangBros18.24.06.16.Angel.Gostosa.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/BangPOV.24.02.20.Justine.Jakobs.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/Blacked.22.10.29.Kay.Lovely.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/BlacksOnBlondes.22.11.29.Hazel.Moore.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/BlacksOnBlondes.24.11.22.Kylie.Rocket.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/BrattySis.22.06.10.Angel.Gostosa.Everyone.Wants.To.Fuck.My.Stepsister.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/BrattySis.23.06.23.Angel.Gostosa.My.Stepsister.Takes.Charge.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/BrattySis.24.05.10.Jade.Maris.And.Vanessa.Marie.Sharing.Stepbros.Bed.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/BrattySis.24.05.31.Renee.Rose.Nobody.Wants.To.Fuck.A.Virgin.Stepbro.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/BrattySis.24.07.26.Jade.Maris.And.Sera.Ryder.I.Want.To.Bone.My.Stepsister.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/BrazzersExxtra.24.08.21.Destiny.Mira.Cyclist.Sweat.Leads.To.Shower.Sex.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/BrazzersExxtra.24.10.20.Baby.Gemini.Dripping.Wet.Hookup.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/BrazzersExxtra.24.11.04.Nichole.Saphir.Tattooed.Bombshells.Gaping.Anal.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/BrazzersExxtra.24.11.20.Katie.Kush.And.Kelly.Caprice.Free.Use.Maid.Service.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/BrazzersExxtra.24.11.20.Kira.Noir.Secret.Goth.Camgirl.Roomie.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/CherryPimps.Busted.24.08.14.Lacey.Jayne.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/CrazyCollegeGFs.24.02.20.Destiny.Mira.Bunk.Banging.The.Handyman.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/Creampie-Angels.22.07.03.Nata.Ocean.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/CumSwappingSis.24.03.16.Jade.Maris.And.Liz.Jordan.Giving.Stepbro.Something.To.Do.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/DFXSoleMates.24.06.15.Angel.Gostosa.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/DatingMyStepson.24.10.03.Shalina.Devine.A.Game.Of.Dare.Always.Spices.Things.Up.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/DirtyAuditions.24.08.17.Kazumi.XXX.720p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/ElegantAngel.24.01.01.Hazel.Moore.A.Touch.Of.Class.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/ElegantAngel.24.11.29.Scarlett.Alexis.Stretch.My.Hole.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/EvilAngel.24.06.01.Chloe.Amour.XXX.1080p.HEVC.x265.PRT.mkv"},
{"file_path": "/mnt/box/files/_raw/prt/EvilAngel.24.11.01.Princess.Alice.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/FilthyTaboo.24.10.19.Kelly.Caprice.Unsatisfied.MILF.Gets.What.She.Craves.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/GirlGirlXXX.24.01.16.Kylie.Rocket.And.Melissa.Stratton.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/Hegre.24.10.22.Anna.L.Erotic.Goddess.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/HotwifeXXX.24.11.06.Willow.Ryder.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/InTheCrack.E1928.Lia.Lin.Provence.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/KarupsPC.20.04.01.Nata.Ocean.Horny.Housekeeper.XXX.1080p.HEVC.x265.PRT.mkv"},
{
"file_path": "/mnt/box/files/_raw/prt/LegalPorno.23.11.10.Lady.Ana.Jureka.Del.Mar.Fisting.ATOGM.Double.Anal.Fist.Big.Gapes.Monster.ButtRose.Squirt.Creampie.Swallow.GIO2635.XXX.1080p.HEVC.x265.PRT.mkv"
},
{"file_path": "/mnt/box/files/_raw/prt/LegalPorno.24.06.19.Kristy.Black.GIO2815.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/LegalPorno.24.10.02.Leila.Botwin.AH007.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/LegendaryX.24.01.11.Melissa.Stratton.And.Nicole.Doshi.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/LegendaryX.24.02.01.Chantal.Danielle.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/LilSis.24.05.17.Jade.Maris.Let.The.Games.Begin.Part.1.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/LittleCaprice-Dreams.24.11.09.Tiffany.Tatum.And.Nata.Ocean.Nasstyx.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/MilfBody.24.06.14.Kelly.Caprice.Kellys.New.Routine.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/MomIsHorny.23.12.29.Justine.Jakobs.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/MomXXX.24.11.05.Kiara.Lord.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/MomsTeachSex.23.11.24.Justine.Jakobs.Getting.Down.To.Business.XXX.1080p.HEVC.x265.PRT.mkv"},
{"file_path": "/mnt/box/files/_raw/prt/MrLuckyPOV.24.01.05.Gianna.Dior.And.Kazumi.Squirts.An.All.Star.Threesome.XXX.1080p.HEVC.x265.PRT.mkv"},
{"file_path": "/mnt/box/files/_raw/prt/MrLuckyPOV.24.01.05.Gianna.Dior.And.Kazumi.Squirts.An.All.Star.Threesome.XXX.1080p.HEVC.x265.PRT.mkv"},
{"file_path": "/mnt/box/files/_raw/prt/MySistersHotFriend.24.04.09.Willow.Ryder.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/NewSensations.24.11.16.Ellie.Nova.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/NubileFilms.23.10.17.Liz.Jordan.My.Roommate.Has.A.Crush.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/NubileFilms.23.11.20.Hailey.Rose.And.Jade.Maris.Happy.Birthday.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/NubileFilms.24.02.01.Liz.Jordan.February.2024.Fantasy.Of.The.Month.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/NubileFilms.24.03.01.Jade.Maris.March.2024.Fantasy.Of.The.Month.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/NubileFilms.24.12.01.Chloe.Temple.December.2024.Fantasy.Of.The.Month.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/Nubiles-Casting.24.10.28.Chanel.Camryn.Cast.Addison.Vodka.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/PascalsSubSluts.21.06.25.Rebel.Rhyder.Every.Inch.Every.Hole.XXX.1080p.HEVC.x265.PRT.mkv"},
{"file_path": "/mnt/box/files/_raw/prt/Passion-HD.24.04.03.Aria.Valencia.Good.Help.XXX.1080p.HEVC.x265.PRT.mp4"},
{
"file_path": "/mnt/box/files/_raw/prt/PissVids.23.05.20.Juis.Wild.Black.Pee.1on1.BBC.ATM.Balls.Deep.No.Pussy.Rough.Sex.Gapes.Pee.Drink.Cum.In.Mouth.Swallow.GL836.XXX.1080p.HEVC.x265.PRT.mp4"
},
{"file_path": "/mnt/box/files/_raw/prt/PornMegaLoad.24.07.18.Justine.Jakobs.Hardcore.40443.XXX.1080p.HEVC.x265.PRT.mkv"},
{"file_path": "/mnt/box/files/_raw/prt/PornPlus.24.10.18.Hailey.Rose.Naturally.Gifted.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/Private.23.11.18.Dolly.Dyson.Enjoys.An.Anal.Threesome.With.Mary.Popiense.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/Private.24.11.18.Lia.Lin.Gets.Horny.By.The.Pool.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/RKPrime.24.10.28.Octokuro.Fucking.After.The.Club.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/RickysRoom.23.03.09.Destiny.Mira.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/SexSelector.23.03.17.Graycee.Baybee.XXX.1080p.HEVC.x265.PRT.mkv"},
{"file_path": "/mnt/box/files/_raw/prt/StrapLez.23.08.17.Alexis.Crystal.And.Lilly.Bella.Everything.She.Wants.XXX.1080p.HEVC.x265.PRT.mkv"},
{"file_path": "/mnt/box/files/_raw/prt/StrapLez.24.11.14.Cherry.Candle.And.Lilly.Mays.Pleasure.Control.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/StrapLez.24.11.28.Alexis.Crystal.And.Mia.Trejsi.Rhythm.Of.Desire.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/TeensLoveHugeCocks.23.04.28.Kylie.Rocket.Influencers.Delight.XXX.1080p.HEVC.x265.PRT.mkv"},
{"file_path": "/mnt/box/files/_raw/prt/ThePOVGod.24.10.25.Willow.Ryder.The.Call.Of.That.Booty.XXX.1080p.HEVC.x265.PRT.mp4"},
{"file_path": "/mnt/box/files/_raw/prt/Tiny4K.23.02.02.Liz.Jordan.Ice.Cream.Treat.XXX.1080p.HEVC.x265.PRT.mp4"},
]
with connect_server("klein") as ssh, connect_db() as con:
for row in rows:
file_path = row["file_path"]
try:
print("deleting", file_path)
ssh.exec_command(f'rm -rf "{file_path}"')
delete_video(con, file_path=file_path)
con.commit()
except:
pass
@contextlib.contextmanager
def connect_server(host: str) -> typing.Generator[paramiko.SSHClient, typing.Any, None]:
config = paramiko.SSHConfig.from_path(str(Path(r"~/.ssh/config").expanduser()))
key = paramiko.RSAKey.from_private_key_file(filename=str(Path(r"~/.ssh/id_rsa").expanduser()))
target = config.lookup(host)
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(
hostname=target["hostname"],
port=target.as_int("port"),
username=target["user"],
pkey=key,
)
yield ssh_client
ssh_client.close()
def collect_all():
root = Path("/Volumes/FATBLUE/_backup/_vidz/_all")
video_files = find_vids(root)
# re_good = re.compile(r"(?P<people>.+) - (?P<title>.+) --? (?P<studio>\S+) (?P<date_year>\d{4})(?P<date_month>\d{2})(?P<date_day>\d{2})")
# re_good = re.compile(r"(?P<people>.+) - (?P<title>.+) --? (?P<studio>\S+)")
# re_good = re.compile(r"(?P<people>.+) - (?P<title>.+) --? (?P<date_year>\d{4})(?P<date_month>\d{2})(?P<date_day>\d{2})")
re_good = re.compile(r"(?P<people>.+) - (?P<title>[^-]+) (?P<date_year>\d{4})(?P<date_month>\d{2})(?P<date_day>\d{2})")
for f in sorted(video_files):
m = re_good.search(f.stem)
if not m:
continue
params = m.groupdict()
title = params["title"]
if " " not in title:
print(f.stem, title, sep="\t\t")
continue
# better_name = f"{m['people']} -- @{m['studio']} -- {m['title']} -- {m['date_year']}-{m['date_month']}-{m['date_day']}"
better_name = f"{m['people']} -- {m['title']} -- {m['date_year']}-{m['date_month']}-{m['date_day']}"
# better_name = f"{m['people']} -- @{m['studio']} -- {m['title']}"
print(f.stem, better_name, sep="\n")
better_path = f.with_stem(better_name)
f.rename(better_path)
def clean_images():
from PIL import Image
for img in Path(r"/Users/abdus/Downloads/temp/Ploy Tigerstam -- OnlyFans Leak").glob("*.jpg"):
image = Image.open(img)
width, height = image.size
if width < 2000 or height < 2000:
img.unlink()
def main():
dry = True
try:
# clean_images()
# collect_all()
# save_suggested_names()
# delete_remote_files()
# transfer()
# different_format(dry_run=dry)
# local_duped(dry_run=dry)
# reencoded_self(dry_run=dry)
# dupes()
# scan_videos()
# list_studios()
# lower_resolutions(dry_run=dry)
# large_unwanted()
unwanted(dry_run=dry)
finally:
pass
# clean_records()
if __name__ == "__main__":
main()
csv_to_html.py
+87
View File
@@ -0,0 +1,87 @@
import csv
import html
import sys
page_css = """
body {
font-family: consolas, monospace;
}
table {
border-collapse: collapse;
width: 100%;
}
th, td {
border: 1px solid black;
padding: 8px;
text-align: left;
}
th {
background-color: #f2f2f2;
}
tr:nth-child(even) {
background-color: #fdfdfd;
}
"""
def csv_to_html(csv_input: str) -> str:
"""
Converts TSV input from stdin to an HTML table.
Args:
csv_input: The TSV input as a string.
Returns:
A string containing the HTML table, or None if an error occurs.
"""
try:
delimiter = "\t" if "\t" in csv_input else ","
reader = csv.reader(csv_input.splitlines(), delimiter=delimiter)
header = next(reader) # Get the header row
header_html = f'{"".join(f"<th>{html.escape(col)}</th>" for col in header)}'
rows = [f"<tr>{''.join(f'<td>{html.escape(cell)}</td>' for cell in row)}</tr>" for row in reader]
rows_html = "\n".join(rows)
rendered = f"""
<!DOCTYPE html>
<html>
<head>
<title>Query Results</title>
<style>{page_css}</style>
</head>
<body>
<table>
<thead>
<tr>{header_html}</tr>
</thead>
<tbody>
{rows_html}
</tbody>
</table>
</body>
</html>
"""
return rendered
except csv.Error as e:
raise ValueError(f"Error parsing TSV: {e}")
except StopIteration:
return "<p>No data found in TSV input.</p>"
def main():
try:
tsv_data = sys.stdin.read()
html_output = csv_to_html(tsv_data)
if html_output:
print(html_output)
except UnicodeDecodeError as e:
raise ValueError(f"Error decoding input: {e}. Ensure your TSV data is encoded correctly (e.g., UTF-8).")
except Exception as e:
raise Exception(f"An unexpected error occurred: {e}")
if __name__ == "__main__":
main()
emp.py
-167
View File
@@ -1,167 +0,0 @@
#!/usr/bin/env python3.9
from dataclasses import dataclass
import json
from pathlib import Path
import subprocess
import typing
from playwright.sync_api import Playwright, sync_playwright, Browser
from typer import Option, Typer
class Storage(typing.Protocol):
def get(self, key: str) -> typing.Optional[typing.Any]:
...
def set(self, key: str, value) -> None:
...
@dataclass
class FileStorage:
path: Path
def get(self, key: str):
try:
return json.loads(self.path.read_text()).get(key)
except FileNotFoundError:
return None
def set(self, key: str, value) -> None:
try:
data = json.loads(self.path.read_text())
except FileNotFoundError:
data = {}
data[key] = value
self.path.write_text(json.dumps(data))
class Emp:
def __init__(self, browser: Browser, storage: Storage) -> None:
self.browser = browser
self.storage = storage
def ensure_session(self, username: str, password: str) -> None:
if self.storage.get("cookies") is None:
self.login(username, password)
def login(self, username: str, password: str) -> typing.Dict[str, str]:
"""
logins and returns session cookies
"""
context = self.browser.new_context()
# Open new page
page = context.new_page()
# Go to https://www.empornium.is/
page.goto("https://www.empornium.is/login")
page.locator('[placeholder="Username"]').fill(username)
page.locator('[placeholder="Password"]').fill(password)
# Click text=Stay logged in
page.locator("text=Stay logged in").click()
# Click input:has-text("login")
page.locator('input:has-text("login")').click()
cookies = {c["name"]: c["value"] for c in context.cookies()}
self.storage.set("cookies", {"sid": cookies["sid"]})
return {
"sid": cookies["sid"],
}
def prepare_post(
self,
torrent_path: Path,
title: str,
tags: str,
description: str,
cover_image_url: str,
category: typing.Optional[str] = None,
) -> None:
cookies = self.storage.get("cookies")
assert cookies, "You must login first"
context = self.browser.new_context()
context.add_cookies(
[{"name": k, "value": v, "domain": "www.empornium.is", "path": "/"} for k, v in cookies.items()]
)
page = context.new_page()
page.goto("https://www.empornium.is/upload.php")
if torrent_path.is_file():
page.locator('input[name="file_input"]').set_input_files(torrent_path.expanduser().resolve())
page.locator('text="check for dupes"').click()
# Select category
if category:
page.locator('select[name="category"]').select_option(label=category)
page.locator('input[name="title"]').fill(title)
page.locator('textarea[name="taglist"]').fill(tags)
page.locator('input[name="image"]').fill(cover_image_url)
page.locator('textarea[name="desc"]').fill(description)
# Click text=Preview
page.locator("text=Preview").click()
def submit_post():
storage = FileStorage(Path("emp.json"))
with sync_playwright() as playwright:
browser = playwright.chromium.launch(headless=False)
emp = Emp(browser=browser, storage=storage)
emp.ensure_session(username="zzzp", password="9arjs9za2o")
emp.prepare_post(
torrent_path=Path("~/Downloads/v.torrent"),
title="A Title",
tags="tag.1 tag.2",
description="Some description",
cover_image_url="https://images.com/image.jpg",
category="Anal",
)
input("Press Enter to continue...")
browser.close()
cli = Typer(name="emp")
torrent_cli = Typer(name="torrent")
cli.add_typer(torrent_cli)
@torrent_cli.callback("torrent")
def make_torrent(paths: typing.List[Path], announce_url: str = Option(..., envvar="ANNOUNCE_URL")):
if len(paths) == 1 and paths[0].is_dir():
dir_path = paths[0]
dir_path = dir_path.expanduser().resolve()
proc = subprocess.run(
[
"torrentify",
f"-announce={announce_url}",
f"-comment=created by zzzp",
f"-created-by=zzzp",
f"-name={dir_path.name}",
str(dir_path),
],
check=True,
capture_output=True,
)
torrent_path = dir_path / f"{dir_path.name}.torrent"
torrent_path.write_bytes(proc.stdout)
@torrent_cli.command()
def clone():
pass
if __name__ == "__main__":
cli()
enc.py
+4 -4
View File
@@ -175,7 +175,7 @@ def upload_file(
size = file_path.stat().st_size size = file_path.stat().st_size
while True: while True:
proc = subprocess.Popen( proc = subprocess.Popen(
["rsync", "--bwlimit", "1100", "-rvPa", str(file_path), destination], ["rsync", "--bwlimit", "4500", "-rvPa", str(file_path), destination],
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, text=True,
@@ -333,14 +333,14 @@ def parse_args():
arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
arger.add_argument("video_path", type=Path, help="path to video file") arger.add_argument("video_path", type=Path, help="path to video file")
arger.add_argument("-e", "--encoder", choices=["handbrake", "ffmpeg"], default="handbrake", help="encoder engine") arger.add_argument("-e", "--encoder", choices=["handbrake", "ffmpeg"], default="handbrake", help="encoder engine")
arger.add_argument("-q", "--quality", type=float, default=25, help="x265 quality factor") arger.add_argument("-q", "--quality", type=float, default=35, help="x265 quality factor")
arger.add_argument("--output-dir", "-o", dest="output_dir", type=Path, help="Dir to save encoded files") arger.add_argument("--output-dir", "-o", dest="output_dir", type=Path, help="Dir to save encoded files")
arger.add_argument("--rsync", dest="upload_target", help="rsync encoded file to a host") arger.add_argument("--rsync", dest="upload_target", help="rsync encoded file to a host")
arger.add_argument("--validate", action="store_true", default=False, help="perform validations before starting") arger.add_argument("--validate", action="store_true", default=False, help="perform validations before starting")
arger.add_argument("--denoise", action="store_true", default=False, help="Enable denoise filter (Handbrake only)") arger.add_argument("--denoise", action="store_true", default=False, help="Enable denoise filter (Handbrake only)")
arger.add_argument("--10bit", action="store_true", dest="is_10bit", help="Encode using 10-bit profile") arger.add_argument("--10bit", action="store_true", dest="is_10bit", help="Encode using 10-bit profile")
arger.add_argument("--8bit", action="store_false", dest="is_10bit", help="Encode using 8-bit profile") arger.add_argument("--8bit", action="store_true", dest="is_10bit", help="Encode using 8-bit profile")
arger.set_defaults(is_10bit=True) arger.set_defaults(is_8bit=True)
def parse_time(val: str) -> int: def parse_time(val: str) -> int:
parts = val.split(":") parts = val.split(":")
extract_archives.py
+168
View File
@@ -0,0 +1,168 @@
import argparse
import logging
import re
import subprocess
import sys
from pathlib import Path
logger = logging.getLogger(__name__)
KNOWN_PASSWORDS = """
torgo
Yuzuki
oron.com
MonikGonPs
koth
drdoom-psuzy
psuzy
suzy
idle
dan4260
""".strip().splitlines(keepends=False)
def find_archive_files(root_dir: Path) -> list[Path]:
extractables = []
globs = ["*.rar", "*.zip", "*.7z", "*.zip.*", "*.7z.*"]
for glob in globs:
for f in root_dir.rglob(glob):
extractables.append(f)
extractables = [f for f in extractables if is_extractable(f)]
return sorted(extractables, key=str)
def is_extractable(archive: Path) -> bool:
if re.search(f"\.zip|\.7z", archive.name):
zip_part = re.search("(\d+)$", archive.suffix)
if zip_part and int(zip_part.group(1)) == 1:
return True
if not zip_part:
return True
if archive.name.endswith(".rar"):
rar_part = re.search(r"\.part(\d+)", archive.name)
if not rar_part:
return True
if rar_part and int(rar_part.group(1)) == 1:
if archive.with_name(archive.name.replace("part1", "part2")).is_file():
return True
return False
def extract_archive_rar(archive: Path, output_dir: Path, password: str | None = None) -> None:
args = [
"rar",
"e",
f"-p{password}" if password else "-p-",
f"{archive.name}",
f"{output_dir}",
]
try:
subprocess.run(args, cwd=archive.parent, capture_output=True, text=True, check=True)
except subprocess.CalledProcessError as e:
output: str = e.stderr or e.stdout
if "corrupt file or wrong password" in output.lower():
raise ValueError("corrupt or invalid password")
elif e.returncode == 3:
raise FileNotFoundError("missing parts")
elif e.returncode == 10:
raise ValueError("not a rar file")
elif e.returncode == 11:
raise ValueError("invalid password")
raise
def extract_archive_7z(archive: Path, output_dir: Path, password: str | None = None):
args = [
"7z",
"e",
f'-p{password or ""}',
f"-o{output_dir}",
"-y",
f"{archive}",
]
try:
subprocess.run(args, capture_output=True, text=True, check=True)
except subprocess.CalledProcessError as e:
output: str = e.stderr or e.stdout
if "missing volume" in output.lower():
raise FileNotFoundError("missing parts")
if e.returncode == 2:
raise ValueError("invalid password")
raise
def extract_archive(archive_path: Path, output_dir: Path, passwords: list[str]) -> None:
for p in ["", *passwords]:
try:
if ".rar" in archive_path.name:
extract_archive_rar(archive_path, output_dir, p)
else:
extract_archive_7z(archive_path, output_dir, p)
except Exception as e:
logger.error(f"unhandled error: {e}")
raise e
def delete_archive(archive: Path):
stem = archive.stem.rsplit(".", maxsplit=1)[0]
for f in archive.parent.rglob(f"*{archive.suffix}"):
if archive.stem.startswith(stem):
logger.info(f"deleting: {f.name}")
f.unlink()
def parse_args():
arger = argparse.ArgumentParser()
arger.add_argument("archive_paths", type=Path, nargs="+", help="List of archive paths")
arger.add_argument("--cwd", type=Path, default=Path.cwd(), help="working directory")
arger.add_argument("--clean", type=bool, default=False, help="delete file(s) after successful extraction", action="store_true")
if len(sys.argv) < 2:
arger.print_help()
exit(1)
return arger.parse_args()
def main():
args = parse_args()
cwd: Path = args.cwd
archives = find_archives(cwd)
logger.info(f"found {len(archives)} archives")
for f in archives:
logger.info(f"\t{f.relative_to(cwd)}")
typer.confirm("continue", default=True, abort=True)
failed = []
for i, f in enumerate(archives, start=1):
progress = f"{i:02}/{len(archives):02}"
logger.info(f"\nextracting {progress}: {f.relative_to(cwd)}")
try:
success = extract_archive(f, passwords=passwords)
if success:
if clean:
delete_archive(f)
logger.info(f"done")
else:
logger.error("failed")
except (ValueError, FileNotFoundError) as e:
logger.error(str(e))
failed.append(f)
if failed:
logger.info(f"\n\nfailed to extract {len(failed)} archives")
for f in failed:
logger.info(f"{f.relative_to(cwd)}")
logger.info("\n\nfinished.")
if __name__ == "__main__":
typer.run(main)
ffmpeg.py
+53 -43
View File
@@ -4,12 +4,10 @@ import dataclasses
import json import json
import logging import logging
import subprocess import subprocess
import typing
from datetime import timedelta from datetime import timedelta
from math import ceil from math import ceil
from pathlib import Path from pathlib import Path
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -26,11 +24,28 @@ class FfprobeResult:
container: str container: str
ar: float ar: float
sample_ar: float sample_ar: float
tags: typing.Dict[str, str] tags: dict[str, str]
@property
def resolution(self) -> str:
return f'{self.width}x{self.height}'
@property
def has_metadata(self):
return any(k in self.tags for k in ['title', 'comment'])
@property
def hd_mode(self) -> str:
if 700 < self.height < 800:
return '720p'
if 900 < self.height < 1200:
return '1080p'
if 2000 < self.height < 2200:
return '4K'
raise ValueError('unknown HD mode')
def ffprobe(video: Path) -> FfprobeResult: def ffprobe(video_path: Path) -> FfprobeResult:
logger.debug("executing ffprobe")
proc = subprocess.run( proc = subprocess.run(
args=[ args=[
"ffprobe", "ffprobe",
@@ -40,7 +55,7 @@ def ffprobe(video: Path) -> FfprobeResult:
"-show_format", "-show_format",
"-print_format", "-print_format",
"json", "json",
str(video), str(video_path),
], ],
text=True, text=True,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
@@ -48,8 +63,8 @@ def ffprobe(video: Path) -> FfprobeResult:
) )
try: try:
proc.check_returncode() proc.check_returncode()
except subprocess.CalledProcessError: except subprocess.CalledProcessError as e:
logger.error("failed to run ffprobe", exc_info=True) logger.error(f"failed to run ffprobe. stderr={e.stderr}", exc_info=True)
raise raise
output: dict = json.loads(proc.stdout) output: dict = json.loads(proc.stdout)
video_stream: dict = [s for s in output.get("streams", []) if s.get("codec_type") == "video"][0] video_stream: dict = [s for s in output.get("streams", []) if s.get("codec_type") == "video"][0]
@@ -71,9 +86,6 @@ def ffprobe(video: Path) -> FfprobeResult:
except: except:
sample_ar = 1 sample_ar = 1
logger.debug(f"dimensions={width}x{height} duration={duration_time}")
# total_frames = video_stream.get("nb_frames")
return FfprobeResult( return FfprobeResult(
duration_sec=duration, duration_sec=duration,
duration_human=str(duration_time), duration_human=str(duration_time),
@@ -83,19 +95,18 @@ def ffprobe(video: Path) -> FfprobeResult:
width=width, width=width,
bitrate=bitrate, bitrate=bitrate,
height=height, height=height,
container=video.suffix.lstrip(".").lower(), container=video_path.suffix.lstrip(".").lower(),
ar=width / height, ar=width / height,
sample_ar=sample_ar, sample_ar=sample_ar,
tags=tags, tags=tags,
# total_frames=total_frames,
) )
def strip_metadata(video_path: Path, save_path: Path) -> Path: def strip_metadata(video_path: Path, save_path: Path) -> Path:
media_info = ffprobe(video_path) media_info = ffprobe(video_path)
# check if the first video stream has hevc codec # check if the first video stream has hevc codec
if media_info["streams"][0]["codec_name"] == "hevc": if media_info.codec == "hevc":
logger.info(f"Video codec is hevc, adding hvc1 tag") logging.info(f"Video codec is hevc, adding hvc1 tag")
extra_args = ["-tag:v", "hvc1"] extra_args = ["-tag:v", "hvc1"]
else: else:
extra_args = [] extra_args = []
@@ -106,7 +117,7 @@ def strip_metadata(video_path: Path, save_path: Path) -> Path:
'-i', '-i',
str(video_path), str(video_path),
'-c:v', 'copy', '-c:v', 'copy',
'-movflags', '+faststart', # '-movflags', '+faststart',
'-map_metadata', '-1', '-map_metadata', '-1',
*extra_args, *extra_args,
'-c:a', 'copy', '-c:a', 'copy',
@@ -114,32 +125,22 @@ def strip_metadata(video_path: Path, save_path: Path) -> Path:
str(save_path), str(save_path),
] ]
# fmt: on # fmt: on
logger.info(f"calling ffmpeg with {args=}") logging.info(f"calling ffmpeg with {args=}")
subprocess.run(args, check=True) subprocess.run(args, check=True)
return save_path return save_path
def make_thumbnail_tile( def make_thumbnail_tile(
video: Path, video: Path,
image_path: Path = None,
columns: int = 3, columns: int = 3,
interval_seconds: int = 60, interval_seconds: int = 60,
tile_width: int = 540, tile_width: int = 540,
skip_first_sec: int = 10, skip_first_sec: int = 10,
skip_if_exists: bool = False, ) -> bytes:
) -> Path:
if not image_path:
image_path = video.parent / f"{video.stem}.thumbnail.jpg"
if image_path.is_file() and skip_if_exists:
logger.debug("thumbnail already exists")
return image_path
info = ffprobe(video) info = ffprobe(video)
min_frames = columns * 3 min_frames = columns * 3
sec_per_frame = min(interval_seconds, ceil(info.duration_sec / min_frames)) sec_per_frame = min(interval_seconds, ceil(info.duration_sec / min_frames))
rows = ceil(info.duration_sec // sec_per_frame / columns) rows = ceil(info.duration_sec // sec_per_frame / columns)
tile = f"{columns}x{rows}" tile = f"{columns}x{rows}"
@@ -150,25 +151,32 @@ def make_thumbnail_tile(
# timestamp_filter = rf"drawtext=r=1:timecode='00\:00\:00\:00':fontsize=16:fontcolor=white:x=10:y=10:box=1:boxcolor=black@0.5" # timestamp_filter = rf"drawtext=r=1:timecode='00\:00\:00\:00':fontsize=16:fontcolor=white:x=10:y=10:box=1:boxcolor=black@0.5"
# timestamp_filter = fr"drawtext=text='%{{(pts\\+{skip_first_sec})\:hms}}':fontsize=16:fontcolor=white:x=10:y=10:box=1:boxcolor=black@0.5" # timestamp_filter = fr"drawtext=text='%{{(pts\\+{skip_first_sec})\:hms}}':fontsize=16:fontcolor=white:x=10:y=10:box=1:boxcolor=black@0.5"
proc = subprocess.run( args = [
# fmt: off # fmt: off
args=[ "ffmpeg",
"ffmpeg", "-v", "error",
"-v", "error", "-skip_frame", "nokey",
"-skip_frame", "nokey", "-ss", f"{skip_first_sec}",
"-ss", f"{skip_first_sec}", "-i", str(video.absolute()),
"-i", str(video.absolute()), # "-vf", f"fps=1/{sec_per_frame},scale={scale},{timestamp_filter},tile={tile}",
# "-vf", f"fps=1/{sec_per_frame},scale={scale},{timestamp_filter},tile={tile}", "-vf", f"fps=1/{sec_per_frame},scale={scale},tile={tile}",
"-vf", f"fps=1/{sec_per_frame},scale={scale},tile={tile}", "-frames", "1",
"-frames", "1", "-c:v",
"-y", str(image_path.absolute()), "jpeg2000",
], "-q:v", "90",
"-f", "image2pipe",
"-",
# fmt: on # fmt: on
stdout=subprocess.DEVNULL, ]
logger.debug(f'calling ffmpeg with args={args}')
proc = subprocess.run(
args=args,
stdout=subprocess.PIPE,
timeout=2000, timeout=2000,
# preexec_fn=os.setpgrp,
) )
proc.check_returncode() proc.check_returncode()
return image_path return bytes(proc.stdout)
if __name__ == "__main__": if __name__ == "__main__":
@@ -177,9 +185,11 @@ if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)
for it in sys.argv[1:]: for it in sys.argv[1:]:
video = Path(it) video = Path(it)
image_path = video.parent / f"{video.stem}.thumbnail.jpg"
if not video.exists(): if not video.exists():
continue continue
logger.info("creating thumbnail for %s", video) logger.info("creating thumbnail for %s", video)
make_thumbnail_tile(video) image_jpeg = make_thumbnail_tile(video)
image_path.write_bytes(image_jpeg)
ffmpeg_gif.py
+5 -5
View File
@@ -58,7 +58,7 @@ def optimize_gif(gif_path: Path) -> Path:
subprocess.run( subprocess.run(
[ [
"gifsicle", "gifsicle",
"--lossy=98", "--lossy=200",
"--optimize=3", "--optimize=3",
"--batch", "--batch",
"-i", "-i",
@@ -75,9 +75,9 @@ def parse_args(argv: typing.List[str] = None) -> argparse.Namespace:
arger.add_argument( arger.add_argument(
"--start", "-s", dest="from_time", type=str, help="Time to start from, e.g. 02:59", required=True "--start", "-s", dest="from_time", type=str, help="Time to start from, e.g. 02:59", required=True
) )
arger.add_argument("--duration", "-d", default=3, type=int, help="Duration of gif in seconds") arger.add_argument("--duration", "-d", default=3, type=float, help="Duration of gif in seconds")
arger.add_argument("--fps", default=15, type=int, help="Frames per second") arger.add_argument("--fps", default=20, type=int, help="Frames per second")
arger.add_argument("--width", default=400, type=int, help="Image width") arger.add_argument("--width", default=376, type=int, help="Image width")
arger.add_argument("--optimize", default=True, action="store_true", help="Reduce GIF filesize") arger.add_argument("--optimize", default=True, action="store_true", help="Reduce GIF filesize")
if len(argv) == 0: if len(argv) == 0:
@@ -105,7 +105,7 @@ def main():
save_path = None save_path = None
if str(args.video_path).startswith("http"): if str(args.video_path).startswith("http"):
save_path = Path("~/Downloads").expanduser() / f"{md5(str(video_path))}__{from_time.replace(':', '')}.gif" save_path = Path("~/Pictures/Screenshots/").expanduser() / f"{md5(str(video_path))}__{from_time.replace(':', '')}.gif"
gif_path = create_gif( gif_path = create_gif(
video_path=video_path, video_path=video_path,
ffmpeg_strip.py
+1 -1
View File
@@ -41,7 +41,7 @@ def clean_video(video_path: Path, save_path: Path) -> Path:
media_info = ffprobe(video_path) media_info = ffprobe(video_path)
# check if the first video stream has hevc codec # check if the first video stream has hevc codec
if media_info["streams"][0]["codec_name"] == "hevc": if media_info["streams"][0]["codec_name"] == "hevc":
logging.info(f"Video codec is hevc, adding hvc1 tag") logging.debug(f"Video codec is hevc, adding hvc1 tag")
extra_args = ["-tag:v", "hvc1"] extra_args = ["-tag:v", "hvc1"]
else: else:
extra_args = [] extra_args = []
file_renamer.py
+87 -58
View File
@@ -10,24 +10,25 @@ import typing
from pathlib import Path from pathlib import Path
script_path = Path(__file__) script_path = Path(__file__)
db_filename = script_path.with_name(f'{script_path.stem}.known.txt') db_filename = script_path.with_name(f"{script_path.stem}.known.txt")
def save_actors(actors: typing.Iterable[str]): def save_actors(actors: typing.Iterable[str]):
existing = load_actors() existing = load_actors()
all_actors = sorted({*existing, *actors}) all_actors = sorted({*existing, *actors})
with db_filename.open('w', encoding='utf-8', newline='\n') as f: with db_filename.open("w", encoding="utf-8", newline="\n") as f:
f.writelines(all_actors) f.writelines(all_actors)
def load_actors() -> set[str]: def load_actors() -> set[str]:
try: try:
with db_filename.open('w', encoding='utf-8', newline='\n') as f: with db_filename.open(encoding="utf-8", newline="\n") as f:
return {line.strip() for line in f} return {line.strip() for line in f}
except FileNotFoundError: except FileNotFoundError:
return set() return set()
re_date_only = re.compile(r"\b(\d{4})\b") # 2024
re_date_us = re.compile(r"\b(\d{2})\D(\d{2})\D(\d{2})\b") # 12/31/21 re_date_us = re.compile(r"\b(\d{2})\D(\d{2})\D(\d{2})\b") # 12/31/21
re_date_iso = re.compile(r"\b(\d{4})\D?(\d{2})\D?(\d{2})\b") # 2020-12-31 re_date_iso = re.compile(r"\b(\d{4})\D?(\d{2})\D?(\d{2})\b") # 2020-12-31
re_date_iso_short = re.compile(r"(\d{2})\D?(\d{2})\D?(\d{2})") # 20-12-31"' re_date_iso_short = re.compile(r"(\d{2})\D?(\d{2})\D?(\d{2})") # 20-12-31"'
@@ -53,10 +54,19 @@ class ParsedDate:
def parse_date(filename: str) -> ParsedDate | None: def parse_date(filename: str) -> ParsedDate | None:
def from_date_only():
match = re_date_only.search(filename)
y = int(match.group(1))
return ParsedDate(
datetime.date(y, 1, 1), start=match.start(), end=match.end()
)
def from_us_format(): def from_us_format():
match = re_date_us.search(filename) match = re_date_us.search(filename)
m, d, y = map(int, match.groups()) m, d, y = map(int, match.groups())
return ParsedDate(datetime.date(y + 2000, m, d), start=match.start(), end=match.end()) return ParsedDate(
datetime.date(y + 2000, m, d), start=match.start(), end=match.end()
)
def from_iso(): def from_iso():
match = re_date_iso.search(filename) match = re_date_iso.search(filename)
@@ -66,7 +76,9 @@ def parse_date(filename: str) -> ParsedDate | None:
def from_iso_short(): def from_iso_short():
match = re_date_iso_short.search(filename) match = re_date_iso_short.search(filename)
y, m, d = map(int, match.groups()) y, m, d = map(int, match.groups())
return ParsedDate(datetime.date(y + 2000, m, d), start=match.start(), end=match.end()) return ParsedDate(
datetime.date(y + 2000, m, d), start=match.start(), end=match.end()
)
def from_iso_reversed(): def from_iso_reversed():
match = re_date_iso_rev.search(filename) match = re_date_iso_rev.search(filename)
@@ -76,10 +88,13 @@ def parse_date(filename: str) -> ParsedDate | None:
def from_iso_reversed_short(): def from_iso_reversed_short():
match = re_date_iso_rev_short.search(filename) match = re_date_iso_rev_short.search(filename)
d, m, y = map(int, match.groups()) d, m, y = map(int, match.groups())
return ParsedDate(datetime.date(y + 2000, m, d), start=match.start(), end=match.end()) return ParsedDate(
datetime.date(y + 2000, m, d), start=match.start(), end=match.end()
)
candidates = [] candidates = []
for fn in [ for fn in [
from_date_only,
from_us_format, from_us_format,
from_iso, from_iso,
from_iso_short, from_iso_short,
@@ -94,7 +109,9 @@ def parse_date(filename: str) -> ParsedDate | None:
today = datetime.date.today() today = datetime.date.today()
future_threshold = today + datetime.timedelta(days=60) future_threshold = today + datetime.timedelta(days=60)
past_threshold = datetime.date(2010, 1, 1) past_threshold = datetime.date(2010, 1, 1)
candidates = [it for it in candidates if past_threshold < it.date < future_threshold] candidates = [
it for it in candidates if past_threshold < it.date < future_threshold
]
if not candidates: if not candidates:
return return
@@ -112,31 +129,35 @@ class Release:
def to_filename(self): def to_filename(self):
parts = [] parts = []
if self.actors: if self.actors:
parts.append(', '.join(self.actors)) parts.append(", ".join(self.actors))
if self.studio: if self.studio:
parts.append(f'@{self.studio}') parts.append(f"@{self.studio}")
if self.title: if self.title:
parts.append(self.title) parts.append(self.title)
if self.released_at: if self.released_at:
parts.append(self.released_at.isoformat()) parts.append(self.released_at.isoformat())
return ' -- '.join(parts) return " -- ".join(parts)
def trash(path: Path): def trash(path: Path):
filename = f'"{path}"' filename = f'"{path}"'
cmd = ['osascript', '-e', f'tell app "Finder" to move (POSIX file {filename}) to trash'] cmd = [
"osascript",
"-e",
f'tell app "Finder" to move (POSIX file {filename}) to trash',
]
subprocess.run(cmd).check_returncode() subprocess.run(cmd).check_returncode()
def filenames_to_actors(): def filenames_to_actors():
sources = [ sources = [
Path(r'/Users/abdus/Downloads/temp/'), Path(r"/Users/abdus/Downloads/temp/"),
Path(r'/Volumes/BANDAID/_temp/__reenc/'), Path(r"/Volumes/BANDAID/_temp/__reenc/"),
Path(r'/Volumes/BANDAID/_temp/'), Path(r"/Volumes/BANDAID/_temp/"),
] ]
actors = set() actors = set()
for it in sources: for it in sources:
for f in it.glob('*.mp4'): for f in it.glob("*.mp4"):
if not f.is_file(): if not f.is_file():
continue continue
if r := parse_release(f.name): if r := parse_release(f.name):
@@ -144,78 +165,83 @@ def filenames_to_actors():
return actors return actors
def parse_release(filename: str, known_actors: set[str] | None = None): def parse_release(
filename: str, known_actors: set[str] | None = None
) -> Release | None:
if not known_actors: if not known_actors:
known_actors = set() known_actors = set()
filename = re.sub(r'\.\w{3,4}$', '', filename) if any(filename.lower().endswith(ext) for ext in [".mp4", ".mkv"]):
filename = re.sub(r"\.\w{3,4}$", "", filename)
def from_own(): def from_own():
remaining = re.sub(r'\s+\[[^]]+]$', '', filename) remaining = re.sub(r"\s+\[[^]]+]$", "", filename)
remaining = re.sub(r'\s+\[([^]]+|\d+\w)(,\s*[^]]+)?]', '', remaining) remaining = re.sub(r"\s+\[([^]]+|\d+\w)(,\s*[^]]+)?]", "", remaining)
remaining = re.sub(r'\s*--\s*', ' -- ', remaining) remaining = re.sub(r"\s*--\s*", " -- ", remaining)
match remaining.split(' -- '): match remaining.split(" -- "):
case [actors, studio, title, date] if studio.startswith('@'): case [actors, studio, title, date] if studio.startswith("@"):
return Release( return Release(
actors=sorted(actors.split(', ')), actors=sorted(actors.split(", ")),
studio=studio.removeprefix('@'), studio=studio.removeprefix("@"),
title=title, title=title,
released_at=datetime.date.fromisoformat(date), released_at=datetime.date.fromisoformat(date),
) )
case [actors, studio, date] if studio.startswith('@') and (parsed := parse_date(date)): case [actors, studio, date] if studio.startswith("@") and (
parsed := parse_date(date)
):
return Release( return Release(
actors=sorted(actors.split(', ')), actors=sorted(actors.split(", ")),
studio=studio.removeprefix('@'), studio=studio.removeprefix("@"),
title=None, title=None,
released_at=parsed.date, released_at=parsed.date,
) )
case [actors, studio, title] if studio.startswith('@'): case [actors, studio, title] if studio.startswith("@"):
return Release( return Release(
actors=sorted(actors.split(', ')), actors=sorted(actors.split(", ")),
studio=studio.removeprefix('@'), studio=studio.removeprefix("@"),
title=title, title=title,
released_at=None, released_at=None,
) )
case [actors, title, date] if (parsed := parse_date(date)): case [actors, title, date] if (parsed := parse_date(date)):
return Release( return Release(
actors=sorted(actors.split(', ')), actors=sorted(actors.split(", ")),
studio=None, studio=None,
title=title, title=title,
released_at=parsed.date, released_at=parsed.date,
) )
case [actors, studio] if studio.startswith('@'): case [actors, studio] if studio.startswith("@"):
return Release( return Release(
actors=sorted(actors.split(', ')), actors=sorted(actors.split(", ")),
studio=studio, studio=studio,
) )
case [actors, title]: case [actors, title]:
return Release( return Release(
actors=sorted(actors.split(', ')), actors=sorted(actors.split(", ")),
title=title, title=title,
) )
def from_prt(): def from_prt():
assert '.PRT' in filename # assert '.PRT' in filename
remaining = re.sub(r'\.(720p|1080p|HEVC|x265|PRT|XXX)', ' ', filename) remaining = re.sub(r"\.(720p|1080p|HEVC|x265|PRT|XXX)", " ", filename)
studio = remaining[:remaining.index('.')] studio = remaining[: remaining.index(".")]
parsed_date = parse_date(remaining) parsed_date = parse_date(remaining)
remaining = remaining[parsed_date.end:] remaining = remaining[parsed_date.end :]
remaining = re.sub(r'[. ]+', ' ', remaining).strip() remaining = re.sub(r"[. ]+", " ", remaining).strip()
actors = [] actors = []
title = None title = None
match remaining.split(' '): match remaining.split(" "):
case [a_first, a_last, 'And', b_first, b_last]: case [a_first, a_last, "And", b_first, b_last]:
actors = [f'{a_first} {a_last}', f'{b_first} {b_last}'] actors = [f"{a_first} {a_last}", f"{b_first} {b_last}"]
case [a_first, a_last, 'And', b_first, b_last, *rest]: case [a_first, a_last, "And", b_first, b_last, *rest]:
actors = [f'{a_first} {a_last}', f'{b_first} {b_last}'] actors = [f"{a_first} {a_last}", f"{b_first} {b_last}"]
title = ' '.join(rest) title = " ".join(rest)
case [first, last]: case [first, last]:
actors = [f'{first} {last}'] actors = [f"{first} {last}"]
case [first, last, *rest]: case [first, last, *rest]:
actors = [f'{first} {last}'] actors = [f"{first} {last}"]
title = ' '.join(rest) title = " ".join(rest)
case [first]: case [first]:
actors = [first] actors = [first]
@@ -243,7 +269,9 @@ def parse_release(filename: str, known_actors: set[str] | None = None):
def parse_args(): def parse_args():
arger = argparse.ArgumentParser() arger = argparse.ArgumentParser()
arger.add_argument('filenames', nargs='+', type=lambda v: Path(v).resolve(), help='Filenames') arger.add_argument(
"filenames", nargs="+", type=lambda v: Path(v).resolve(), help="Filenames"
)
return arger.parse_args() return arger.parse_args()
@@ -254,17 +282,18 @@ class MoveOp:
def to_dict(self) -> dict: def to_dict(self) -> dict:
return { return {
'source': str(self.source), "source": str(self.source),
'target': str(self.target), "target": str(self.target),
} }
def write_undo_script(save_dir: Path, ops: list[MoveOp]): def write_undo_script(save_dir: Path, ops: list[MoveOp]):
now = datetime.datetime.now().isoformat().replace(':', '') now = datetime.datetime.now().isoformat().replace(":", "")
save_path = save_dir / f'undo_{now}.json' save_path = save_dir / f"undo_{now}.json"
with save_path.open('w', encoding='utf-8') as f: with save_path.open("w", encoding="utf-8") as f:
for it in ops: for it in ops:
f.write(json.dumps(it.to_dict()) + '\n') f.write(json.dumps(it.to_dict()) + "\n")
def fix_filename(): def fix_filename():
pass pass
@@ -279,18 +308,18 @@ def main():
jobs = [] jobs = []
for it in paths: for it in paths:
if parsed := parse_release(it.name, known_actors=actors): if parsed := parse_release(it.name, known_actors=actors):
logging.debug(f'filename={it} release={parsed}') logging.debug(f"filename={it} release={parsed}")
suggested = parsed.to_filename() suggested = parsed.to_filename()
target = it.with_stem(suggested) target = it.with_stem(suggested)
if target.is_file(): if target.is_file():
logging.error('target already exists') logging.error("target already exists")
continue continue
jobs.append(MoveOp(source=it, target=target)) jobs.append(MoveOp(source=it, target=target))
if not jobs: if not jobs:
return return
write_undo_script(paths[0].parent, jobs) # write_undo_script(paths[0].parent, jobs)
for it in jobs: for it in jobs:
it.target.hardlink_to(it.source) it.target.hardlink_to(it.source)
file_renamer_test.py
+4
View File
@@ -58,6 +58,10 @@ def test_parse_date(filename: str, date: datetime.date | None):
@pytest.mark.parametrize(['filename', 'expected'], [ @pytest.mark.parametrize(['filename', 'expected'], [
[
'PenthouseGold.23.02.05.Tiffany.Tatum.XXX.720p.HEVC.x265.PRT',
Release(studio='DoctorAdventures', actors=['Jamie Michelle'], title='Nurse Jamie Knows Best', released_at=datetime.date(2021, 5, 26)),
],
[ [
'Rebecca Volpetti -- @RealityKings -- Driving Him Crazy [1080p, x265] -- 2019-04-10', 'Rebecca Volpetti -- @RealityKings -- Driving Him Crazy [1080p, x265] -- 2019-04-10',
Release(actors=['Rebecca Volpetti'], studio='RealityKings', title='Driving Him Crazy', released_at=datetime.date(2019, 4, 10)), Release(actors=['Rebecca Volpetti'], studio='RealityKings', title='Driving Him Crazy', released_at=datetime.date(2019, 4, 10)),
filejoker_downloader.py
+62
View File
@@ -0,0 +1,62 @@
import argparse
import contextlib
import re
from playwright.sync_api import sync_playwright, Page
def download_filejoker(url: str) -> dict:
with launch_browser() as page:
page.goto(url, referer='http://planetsuzy.org')
page.get_by_role("button", name="Slow Download").click()
page.get_by_role("button", name="Get Download Link").click()
with page.expect_response(re.compile('https://fs.+.filejoker.net/')) as res:
with page.expect_download(timeout=0) as dl:
page.get_by_role("button", name="Download File").click(timeout=0)
dl.value.cancel()
print(dl.value.url)
print(dl.value)
page.pause()
print()
def download_filefox(url: str) -> dict:
with launch_browser() as page:
page.goto(url, referer='http://planetsuzy.org')
page.get_by_role("button", name="Slow Download").click()
page.get_by_role("button", name="Slow Download").click(timeout=0)
with page.expect_response(re.compile('https://fs.+.filefox.net/')) as res:
with page.expect_download(timeout=0) as dl:
page.get_by_role("button", name="Download File").click(timeout=0)
dl.value.cancel()
print(dl.value.url)
print(dl.value)
page.pause()
print()
@contextlib.contextmanager
def launch_browser() -> Page:
with sync_playwright() as playwright:
with playwright.chromium.launch(headless=False) as browser:
with browser.new_context() as context:
with context.new_page() as page:
yield page
def parse_args():
arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
arger.add_argument('url', help='URL to download', type=str)
return arger.parse_args()
def main():
args = parse_args()
url = args.url
download_filejoker(url)
if __name__ == '__main__':
# download_filejoker('https://filejoker.net/ofg59z1pqjjb')
download_filefox('https://filefox.cc/qxd2yrxgu4ak')
main()
gardener.py
-207
View File
@@ -1,207 +0,0 @@
import dataclasses
import logging
import os
import webbrowser
def force_import(module: str):
import importlib
import subprocess
import sys
try:
return importlib.import_module(module)
except ModuleNotFoundError:
subprocess.run([sys.executable, "-m", "pip", "install", module])
importlib.invalidate_caches()
return importlib.import_module(module)
try:
import httpx
except ImportError:
torf = force_import("httpx")
try:
import typer
except ImportError:
typer = force_import("typer")
try:
import rich
except ImportError:
rich = force_import("rich")
try:
import inquirer
except ImportError:
inquirer = force_import("inquirer")
from rich import print
import rich.table
JIRA_EMAIL = os.getenv('JIRA_EMAIL', 'abdussamet.kocak@akinon.com')
JIRA_TOKEN = os.getenv('JIRA_TOKEN', 'iAZhlJ1AdGwckbVllhlx48C4')
JIRA_URL = os.getenv('JIRA_URL', 'https://omniplatform.atlassian.net/')
if not JIRA_TOKEN:
logging.info('')
webbrowser.open('https://id.atlassian.com/manage-profile/security/api-tokens')
@dataclasses.dataclass
class Issue:
id: str
key: str
summary: str
@dataclasses.dataclass
class PullRequest:
id: str
name: str
repository: str
branch: str
class Jira:
def __init__(self, email: str, token: str):
self.client = httpx.Client(
base_url=JIRA_URL,
auth=httpx.BasicAuth(email, token),
)
def list_issues_for_release(self) -> list[Issue]:
mergeable_status = 'In Review'
res = self.client.post(
'/rest/api/2/search',
json={
'jql': f'project = COM AND status = "{mergeable_status}" AND "Team[Dropdown]" = Backend ORDER BY created DESC'
},
)
res.raise_for_status()
data = res.json()
development_type = ['Story', 'Bug', 'Task']
return [
Issue(id=it['id'], key=it['key'], summary=it['fields']['summary'])
for it in data['issues']
if it['fields']['issuetype']['name'] in development_type
]
def get_approved_prs(self, issue_id: str):
res = self.client.post(
f'/jsw/graphql',
params={'operation': 'DevDetailsDialog'},
json={
"operationName": "DevDetailsDialog",
"query": """
query DevDetailsDialog ($issueId: ID!) {
developmentInformation(issueId: $issueId){
details {
instanceTypes {
repository {
name
branches {
name
pullRequests {
name
status
lastUpdate
}
reviews {
state
id
}
}
pullRequests {
id
name
branchName
status
reviewers{
name
isApproved
}
}
}
danglingPullRequests {
id
name
branchName
status
reviewers{
name
isApproved
}
}
}
}
}
}
""",
"variables": {"issueId": issue_id},
},
)
res.raise_for_status()
data = res.json()['data']
if not data['developmentInformation']['details']['instanceTypes']:
return []
repo_name = data['developmentInformation']['details']['instanceTypes'][0]['repository'][0]['name']
prs = data['developmentInformation']['details']['instanceTypes'][0]['danglingPullRequests']
if not prs:
repo_name = data['developmentInformation']['details']['instanceTypes'][0]['repository'][0]['name']
prs = data['developmentInformation']['details']['instanceTypes'][0]['repository'][0]['pullRequests']
return [
PullRequest(
id=it['id'],
name=it['name'],
repository=repo_name,
branch=it['branchName'],
)
for it in prs
if it['status'] != 'MERGED'
]
console = rich.console.Console()
def render_issue(issue: Issue, prs: list[PullRequest]):
t = rich.table.Table(
title=f'{issue.key} -- {issue.summary}',
caption_justify='left',
)
t.add_column("Repository", no_wrap=True)
t.add_column("Branch", no_wrap=True)
t.add_column("Pull Request")
for it in prs:
t.add_row(it.repository, it.branch, it.name)
console.print(t)
def main():
logging.basicConfig(level=logging.INFO)
if not JIRA_EMAIL:
logging.error('JIRA_EMAIL is not set')
exit(1)
if not JIRA_TOKEN:
logging.error('JIRA_TOKEN is not set')
exit(1)
j = Jira(email=JIRA_EMAIL, token=JIRA_TOKEN)
logging.info('fetching issues in "ready to release" status')
issues = j.list_issues_for_release()
for it in issues:
logging.info(f'fetching pull requests for issue {it.key}')
prs = j.get_approved_prs(it.id)
if not prs:
logging.info(f'no non-merged pull requests found for issue {it.key}')
continue
render_issue(it, prs)
if __name__ == '__main__':
main()
go.mod
+5
View File
@@ -0,0 +1,5 @@
module github.com/abdusco/playground
go 1.23.0
require golang.org/x/sync v0.10.0
go.sum
+2
View File
@@ -0,0 +1,2 @@
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
html_popup.py
+133
View File
@@ -0,0 +1,133 @@
import http.server
import random
import socketserver
import sys
import threading
import time
import typing
import webbrowser
# language=html
inject_script = """
<script>
// Send periodic heartbeat
let heartbeatInterval = setInterval(() => {
fetch('/heartbeat', {method: 'POST'})
.catch(() => {}); // Ignore errors
}, 1000);
// Multiple event handlers for different close scenarios
function sendShutdown() {
clearInterval(heartbeatInterval);
// Synchronous request to ensure it gets sent
const xhr = new XMLHttpRequest();
xhr.open('POST', '/shutdown', false); // false makes it synchronous
try {
xhr.send();
} catch (e) {}
}
window.addEventListener('beforeunload', sendShutdown);
window.addEventListener('unload', sendShutdown);
// setTimeout(sendShutdown, 5000);
// Handle visibility change (tab hidden/visible)
document.addEventListener('visibilitychange', () => {
if (document.visibilityState === 'hidden') {
sendShutdown();
}
});
</script>
"""
def handler_for_html(html: str) -> typing.Type[http.server.SimpleHTTPRequestHandler]:
class Handler(http.server.SimpleHTTPRequestHandler):
def do_GET(self):
if self.path == "/":
self.send_response(200)
self.send_header("Content-type", "text/html")
self.end_headers()
self.wfile.write(html.encode())
else:
self.send_error(404)
def do_POST(self):
if self.path == "/shutdown":
self.send_response(200)
self.end_headers()
self.server.last_heartbeat = 0 # Force shutdown
self.server.shutdown()
elif self.path == "/heartbeat":
self.send_response(200)
self.end_headers()
self.server.last_heartbeat = time.time()
else:
self.send_error(404)
def log_message(self, format, *args):
# Suppress logging
pass
return Handler
class ServerWithHeartbeat(socketserver.TCPServer):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.last_heartbeat = time.time()
def monitor_heartbeat(server: ServerWithHeartbeat, timeout: int = 2):
while True:
time.sleep(0.5)
if time.time() - server.last_heartbeat > timeout:
print("No heartbeat received, shutting down...")
server.shutdown()
break
def run_server(handler: http.server.SimpleHTTPRequestHandler, port: int = 8000):
with ServerWithHeartbeat(("", port), handler) as httpd:
print(f"Serving at port {port}")
# Start heartbeat monitor in separate thread
monitor_thread = threading.Thread(target=monitor_heartbeat, args=(httpd,))
monitor_thread.daemon = True
monitor_thread.start()
httpd.serve_forever()
def main():
html = sys.stdin.read()
if "<body>" in html:
html_to_display = html.replace("</body>", f"{inject_script}</body>")
else:
html_to_display = f"<html><body>{html}{inject_script}</body></html>"
handler = handler_for_html(html_to_display)
# Start the server in a separate thread
port = random.randint(20000, 65535)
server_thread = threading.Thread(target=run_server, kwargs=dict(handler=handler, port=port))
server_thread.daemon = True
server_thread.start()
# Wait a moment for the server to start
time.sleep(0.5)
# Open the web browser
webbrowser.open(f"http://localhost:{port}")
# Wait for the server thread to finish
server_thread.join()
print("Server stopped")
if __name__ == "__main__":
try:
main()
except KeyboardInterrupt:
print("\nReceived keyboard interrupt, exiting...")
raise SystemExit(0)
htmlpopup.go
Executable
+119
View File
@@ -0,0 +1,119 @@
package main
import (
"context"
"fmt"
"io"
"log"
"net"
"net/http"
"os"
"os/exec"
"os/signal"
"runtime"
"strings"
"syscall"
"time"
)
// language=html
const injectScript = `
<script>
const sendShutdown = () => navigator.sendBeacon('/shutdown');
window.addEventListener('beforeunload', sendShutdown);
window.addEventListener('unload', sendShutdown);
</script>
`
func main() {
if err := run(); err != nil {
log.Fatal(err)
}
}
func newHandler(html string, onShutdown func()) *http.ServeMux {
handler := http.NewServeMux()
handler.HandleFunc("POST /shutdown", func(w http.ResponseWriter, r *http.Request) {
onShutdown()
})
handler.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
w.Write([]byte(html))
})
return handler
}
func run() error {
html, err := io.ReadAll(os.Stdin)
if err != nil {
return fmt.Errorf("failed to read HTML from stdin: %w", err)
}
htmlStr := string(html)
if strings.Contains(htmlStr, "</body>") {
htmlStr = strings.ReplaceAll(string(html), "</body>", injectScript+"</body>")
} else {
htmlStr += injectScript
}
listener, err := net.Listen("tcp", ":0")
if err != nil {
return fmt.Errorf("failed to create listener: %w", err)
}
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer cancel()
port := listener.Addr().(*net.TCPAddr).Port
server := &http.Server{Addr: listener.Addr().String()}
server.Handler = newHandler(htmlStr, func() {
cancel()
})
servedURL := fmt.Sprintf("http://localhost:%d", port)
go func() {
log.Printf("serving at %s", servedURL)
if err := server.Serve(listener); err != nil && err != http.ErrServerClosed {
log.Fatalf("failed to serve: %v", err)
}
}()
go func() {
// wait for server to start
time.Sleep(250 * time.Millisecond)
if err := openBrowser(servedURL); err != nil {
log.Fatalf("failed to open browser: %v", err)
}
}()
<-ctx.Done()
log.Printf("shutting down server")
if err := server.Shutdown(ctx); err != nil {
return fmt.Errorf("failed to shutdown server: %w", err)
}
return nil
}
func openBrowser(url string) error {
var cmd *exec.Cmd
switch runtime.GOOS {
case "darwin":
cmd = exec.Command("open", url)
case "linux":
cmd = exec.Command("xdg-open", url)
case "windows":
cmd = exec.Command("cmd", "/c", "start", url)
default:
return fmt.Errorf("invalid URL: %s", url)
}
return cmd.Run()
}
aws_codecommit.py → legacy/aws_codecommit.py
View File
aws_elasticache.py → legacy/aws_elasticache.py
View File
aws_es.py → legacy/aws_es.py
View File
aws_rds.py → legacy/aws_rds.py
View File
aws_redis_acl.py → legacy/aws_redis_acl.py
View File
aws_ses.py → legacy/aws_ses.py
View File
aws_ses_test.py → legacy/aws_ses_test.py
View File
multi_rename.py
+96
View File
@@ -0,0 +1,96 @@
import argparse
import logging
import os
import shutil
import subprocess
import tempfile
from pathlib import Path
def find_editor() -> str:
editor = os.getenv("EDITOR")
if editor:
return editor
for candidate in ["micro", "nano"]:
if shutil.which(candidate):
return candidate
return "nano"
def edit_text(text: str) -> str | None:
with tempfile.NamedTemporaryFile(suffix=".tmp", delete=False) as tf:
temp_file_path = Path(tf.name)
temp_file_path.write_text(text)
old_modtime = temp_file_path.stat().st_mtime
subprocess.call([*find_editor().split(), str(temp_file_path)])
new_modtime = temp_file_path.stat().st_mtime
if new_modtime == old_modtime:
logging.warning("editor exited without saving")
return None
return temp_file_path.read_text()
def move_to_trash(file_path: Path):
if not file_path.is_file():
raise FileNotFoundError
subprocess.run(
[
"osascript",
"-e",
f'tell app "Finder" to move POSIX file "{file_path}" to trash',
],
check=True,
)
def rename_files(files: list[Path], cwd: Path | None = None):
if not cwd:
cwd = files[0].parent
originals = [f"{f.relative_to(cwd)}" for f in files]
edited_raw = edit_text("\n".join(originals))
if not edited_raw:
return
edited = edited_raw.splitlines(keepends=False)
deletables = []
moveables = []
for source, target in zip(originals, edited):
if target.startswith("#") or target == "":
deletables.append(Path(source))
continue
if source != target:
moveables.append((cwd / source, cwd / target))
for source, target in moveables:
logging.info(f"moving {source} -> {target}")
source.rename(target)
for f in deletables:
try:
logging.info(f"trashing {f}")
move_to_trash(f)
except:
logging.error(f"failed to trash {f}")
def parse_args():
arger = argparse.ArgumentParser()
arger.add_argument("filenames", type=Path, nargs="+", help="File paths")
arger.add_argument("--cwd", type=Path, help="Current working directory")
return arger.parse_args()
def main():
args = parse_args()
rename_files(args.filenames, cwd=args.cwd)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
main()
openai_example_sentences.py
+57
View File
@@ -0,0 +1,57 @@
import argparse
import httpx
OPENAPI_TOKEN = "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"
http = httpx.Client(
base_url='https://api.openai.com/v1/',
headers={
'Authorization': f'Bearer {OPENAPI_TOKEN}',
},
timeout=30,
)
def generate_examples(text: str) -> str:
payload = {
"messages": [
{
"role": "system",
"content": """
You are to provide colloquial and idiomatic German translations of given text. List 10 alternatives.
""",
},
{
"role": "user",
"content": text,
},
],
"temperature": 0.7,
"max_tokens": 256,
"top_p": 1,
"frequency_penalty": 1.06,
"presence_penalty": 0.42,
"model": "gpt-3.5-turbo",
"stream": False,
}
res = http.post('/chat/completions', json=payload)
res.raise_for_status()
return res.json()['choices'][0]['message']['content'].strip()
def parse_args():
arger = argparse.ArgumentParser()
arger.add_argument('prompt', help='Phrase to generate sentences for')
return arger.parse_args()
def main():
args = parse_args()
response = generate_examples(args.prompt)
print(response)
if __name__ == '__main__':
main()
realdebrid.py
+16 -1
View File
@@ -36,6 +36,17 @@ def download_with_aria2(url: str, cwd: Path = Path.cwd(), extra_args: list[str]
subprocess.run(args, text=True, cwd=str(cwd.resolve())) subprocess.run(args, text=True, cwd=str(cwd.resolve()))
def download_torrent(torrent_path: Path):
res = http.put('/torrents/addTorrent', content=torrent_path.read_bytes())
res.raise_for_status()
data = res.json()
res = http.get(f'/torrents/info/{data["id"]}')
res.raise_for_status()
data = res.json()
print()
def parse_args(): def parse_args():
arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
arger.add_argument('url', help='URL to unrestrict & download') arger.add_argument('url', help='URL to unrestrict & download')
@@ -66,4 +77,8 @@ def main():
if __name__ == "__main__": if __name__ == "__main__":
main() download_torrent(
Path(
'/Users/abdus/Downloads/[Empornium][PornWorld] All Her Holes Pounded Hard [Veronica Leal] [1080p] {Se7enSeas} [x265].torrent'
)
)
rename_children.py
Executable
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env python3.10
import argparse
import dataclasses
import datetime
import json
import logging
from pathlib import Path
import re
def parse_args():
arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
arger.add_argument('dirs', help='Path to the directories', type=Path, nargs='+')
return arger.parse_args()
def natural_sort_key(path: Path) -> tuple[str, ...]:
s = path.stem
return tuple(f'{int(p):010d}' if p.isnumeric() else p for p in re.findall(r'(\D+|\d+)', s))
def rename_children(cwd: Path) -> None:
assert cwd.is_dir()
files = [f for f in cwd.glob('*') if f.is_file()]
files = sorted(files, key=natural_sort_key)
new_stem = cwd.name
jobs = []
for i, f in enumerate(files):
target = f.with_stem(f'{new_stem}__{i:04d}')
if f.suffix == '.jpeg':
target = target.with_suffix('.jpg')
if target.is_file():
logging.error(f'target already exists: {target}')
continue
jobs.append(MoveOp(source=f, target=target))
f.rename(target)
if not jobs:
return
write_undo_script(Path.cwd(), jobs)
@dataclasses.dataclass
class MoveOp:
source: Path
target: Path
def to_dict(self) -> dict:
return {
'source': str(self.source),
'target': str(self.target),
}
def write_undo_script(save_dir: Path, ops: list[MoveOp]):
now = datetime.datetime.now().isoformat().replace(':', '')
save_path = save_dir / f'undo_{now}.json'
with save_path.open('w', encoding='utf-8') as f:
for it in ops:
f.write(json.dumps(it.to_dict()) + '\n')
def main():
args = parse_args()
for d in args.dirs:
rename_children(d)
if __name__ == '__main__':
main()
requirements.txt
+6 -1
View File
@@ -2,5 +2,10 @@ beautifulsoup4==4.11.2
boto3==1.19.1 boto3==1.19.1
fastapi==0.86.0 fastapi==0.86.0
httpx==0.23.3 httpx==0.23.3
pytest==6.2.5 pytest==7.2.1
typer==0.6.1 typer==0.6.1
inquirer==3.1.2
playwright==1.31.1
torf==4.1.4
bs4==0.0.1
scan_vids.py
+182
View File
@@ -0,0 +1,182 @@
#!/usr/bin/env python3
# /// script
# requires-python = ">=3.11"
# dependencies = [
# "psycopg[binary]",
# ]
# ///
import argparse
import contextlib
import hashlib
import json
import logging
import subprocess
import typing
from pathlib import Path
import psycopg
import psycopg.rows
import ffmpeg
def hash_partial(f: Path) -> str:
sha1 = hashlib.sha1()
chunk_size = 1024 * 1024 * 10 # 10MB chunk size
total_read = 0
with f.open("rb") as file:
while chunk := file.read(chunk_size):
total_read += len(chunk)
sha1.update(chunk)
break # Only reads the first 10MB
return f"sha1:{total_read}:{sha1.hexdigest()}"
@contextlib.contextmanager
def connect_db() -> typing.Generator[psycopg.Connection, typing.Any, typing.Any]:
with psycopg.connect("postgres://abdus:abdus@db.abdus.dev:5444/smut?sslmode=disable") as conn:
conn.row_factory = psycopg.rows.dict_row
yield conn
def video_exists(con: psycopg.Connection, file_path: str) -> bool:
stmt = con.execute(
"UPDATE videos SET last_seen_at = NOW() WHERE file_path = %s RETURNING true AS exists",
[file_path],
)
row = stmt.fetchone()
return row["exists"]
def delete_video(con: psycopg.Connection, file_path: str) -> bool:
con.execute("DELETE FROM videos WHERE file_path = %s", [file_path])
def save_video(
con: psycopg.Connection,
category: str,
file_path: str,
ffprobe: dict,
hash_partial: str,
):
sql = """INSERT INTO videos(category, file_path, ffprobe, hash_partial) VALUES (%s, %s, %s, %s) ON CONFLICT(file_path) SET last_seen_at = NOW();"""
con.execute(
sql,
[
category,
file_path,
json.dumps(ffprobe),
hash_partial,
],
)
def save_error(con: psycopg.Connection, file_path: str, stderr: str):
sql = """INSERT INTO scan_state(file_path, stderr) VALUES (%s, %s) ON conflict do nothing"""
con.execute(
sql,
[
file_path,
stderr,
],
)
def scan_videos(con: psycopg.Connection, video_paths: list[Path]) -> None:
for i, f in enumerate(video_paths):
if "/_picks/" in str(f):
continue
progress = f"[{i + 1}/{len(video_paths)}]"
if f.is_dir():
continue
file_path = str(f.absolute())
if video_exists(con, file_path=file_path):
logging.debug(f"{progress} video already exists {f=}")
continue
logging.debug(f"probing {f=}")
try:
probed = ffmpeg.ffprobe(f)
except subprocess.CalledProcessError as e:
save_error(con, file_path=file_path, stderr=e.stderr)
continue
file_hash = hash_partial(f)
save_video(
category=category_for_file(f),
con=con,
file_path=file_path,
ffprobe=probed.__dict__,
hash_partial=file_hash,
)
logging.info(f"{progress} saved {f=}")
con.commit()
def category_for_file(file_path: Path) -> str:
if str(file_path).startswith("/Volumes/"):
return f"downloaded.{file_path.parts[1].lower()}"
if str(file_path).startswith("/mnt/box"):
return "box"
raise ValueError(f"unknown category for {file_path}")
def find_copies(con: psycopg.Connection, video_path: Path) -> list[str]:
probed = ffmpeg.ffprobe(video_path)
sql = """
SELECT
file_path
FROM videos
WHERE
file_path != %(file_path)s
AND (
hash_partial = %(hash_partial)s
OR (abs(duration_sec - %(duration_sec)s) < 0.1 AND abs(size_bytes - %(size_bytes)s) < 3000000)
);
"""
stmt: psycopg.Cursor = con.execute(
sql,
{
"file_path": str(video_path.absolute()),
"hash_partial": hash_partial(video_path),
"duration_sec": probed.duration_sec,
"size_bytes": probed.size_bytes,
},
)
rows = stmt.fetchall()
return [it["file_path"] for it in rows]
def parse_args():
arger = argparse.ArgumentParser()
arger.add_argument("video_paths", nargs="+", type=Path, help="Path to the videos")
arger.add_argument("--debug", action="store_true", help="Path to the videos")
return arger.parse_args()
def main():
logging.basicConfig(format=logging.BASIC_FORMAT, level=logging.INFO)
args = parse_args()
if args.debug:
logging.getLogger().setLevel(logging.DEBUG)
video_paths: list[Path] = args.video_paths
if not video_paths:
logging.info("no files to scan")
return
with connect_db() as con:
scan_videos(con=con, video_paths=video_paths)
if __name__ == "__main__":
main()
tekton_api.py
-190
View File
@@ -1,190 +0,0 @@
import functools
import hashlib
import json
import textwrap
import typing
import uuid
from pathlib import Path
import kubernetes.config
import yaml
def make_client() -> kubernetes.client.CoreV1Api:
kubernetes.config.load_kube_config('/Users/abdus/Desktop/kubeconfig-kube.yml')
return kubernetes.client.CoreV1Api()
class Kubernetes:
def __init__(self, client: kubernetes.client.CoreV1Api, namespace='default'):
self.client = client
self.custom_objects = kubernetes.client.CustomObjectsApi(self.client.api_client)
self.namespace = namespace
@classmethod
def from_kubeconfig(cls, kubeconfig_path: str, namespace: str) -> 'Kubernetes':
kubernetes.config.load_kube_config(kubeconfig_path)
return cls(kubernetes.client.CoreV1Api(), namespace=namespace)
def read_logs(self, pod_name: str, container_name: typing.Optional[str] = None, timestamps: bool = False) -> str:
return self.client.read_namespaced_pod_log(
pod_name,
self.namespace,
container=container_name,
timestamps=timestamps,
pretty=True,
)
def get_tekton_object(self, plural: str, name: str = None):
getter = functools.partial(
self.custom_objects.get_namespaced_custom_object,
group='tekton.dev',
version='v1beta1',
namespace=self.namespace,
)
return getter(plural=plural, name=name)
def list_tekton_object(self, plural: str):
getter = functools.partial(
self.custom_objects.list_namespaced_custom_object,
group='tekton.dev',
version='v1beta1',
namespace=self.namespace,
)
return getter(plural=plural)['items']
def create_from_yaml(self, yaml_text: str) -> list[dict]:
specs = list(yaml.safe_load_all(yaml_text))
return kubernetes.utils.create_from_yaml(
kubernetes.client.CustomObjectsApi(self.client.api_client),
yaml_objects=specs,
namespace=self.namespace,
)
def read_logs():
task_hello_world = k8s.get_tekton_object(plural='tasks', name='echo-hello-world')
taskruns = k8s.list_tekton_object(plural='taskruns')
pod_name = taskruns['items'][0]['status']['podName']
container_name = taskruns['items'][0]['status']['containerName']
logs = k8s.read_logs(pod_name, container_name)
print(logs)
k8s = Kubernetes.from_kubeconfig('/Users/abdus/Desktop/kubeconfig.yaml', namespace='api-server')
def get_pipelineru_status():
result = k8s.get_tekton_object('pipelineruns', name='pr-2rr6t')
status = result['status']['conditions'][0]['status']
reason = result['status']['conditions'][0]['reason']
failed_taskruns = [
name for name, it in result['status']['taskRuns'].items() if it['status']['conditions'][0]['status'] == 'False'
]
logs = []
for it in result['status']['taskRuns'].values():
task_name = it['pipelineTaskName']
task_status = it['status']['conditions'][0]['reason']
pod_name = it['status']['podName']
if task_status != 'Failed':
continue
# fetch all logs for all steps to get a better picture of the error
# (regardless of whether it is successful or not)
for i, step in enumerate(it['status']['steps'], start=1):
container_name = step['container']
header = f'# {task_name}, step {i}: {step["name"]}'
logs.append('#' * len(header))
logs.append(header)
logs.append('#' * len(header))
step_logs = k8s.read_logs(pod_name, container_name, timestamps=True)
logs.append(step_logs)
log_text = '\n'.join(logs)
return {
'pipelinerun_name': result['metadata']['name'],
'namespace': result['metadata']['namespace'],
'status': status,
'reason': reason,
'failed_taskruns': failed_taskruns,
'logs': log_text,
}
def create_taskrun():
k = kubernetes.client.CustomObjectsApi()
res = k.create_namespaced_custom_object(
group='tekton.dev',
version='v1beta1',
namespace='default',
plural='taskruns',
body={
'apiVersion': 'tekton.dev/v1beta1',
'kind': 'TaskRun',
'metadata': {'name': 'testing1234' + uuid.uuid4().hex},
'spec': {
'taskRef': {'name': 'task-with-json'},
'params': [
{
'name': 'json_value',
'value': '{"foo": "bar", "nested": {"a": "1"}}',
}
],
},
},
)
print(res)
def migrate_pipelines():
k = kubernetes.client.CustomObjectsApi()
pipeline_yaml = Path('sample_pipeline.yaml').read_text()
parsed_pipeline = yaml.safe_load(pipeline_yaml)
params = dict(group='tekton.dev', version='v1beta1', namespace='default', plural='pipelines')
k.delete_namespaced_custom_object(
**params,
name=parsed_pipeline['metadata']['name'],
)
res = k.create_namespaced_custom_object(
**params,
body=parsed_pipeline,
)
return parsed_pipeline
def trigger_pipeline():
k = kubernetes.client.CustomObjectsApi()
pipeline = migrate_pipelines()
pipeline_name = pipeline['metadata']['name']
payload = {
'image': 'nginx',
'json_value': {'nested': {'nested2': {'a': 1}}},
}
res = k.create_namespaced_custom_object(
group='tekton.dev',
version='v1beta1',
namespace='default',
plural='pipelineruns',
body={
'apiVersion': 'tekton.dev/v1beta1',
'kind': 'PipelineRun',
'metadata': {'name': 'deploy-app-' + uuid.uuid4().hex},
'spec': {
'pipelineRef': {'name': pipeline_name},
'params': [
{'name': k, 'value': json.dumps(v) if not isinstance(v, str) else v} for k, v in payload.items()
],
},
},
)
if __name__ == '__main__':
main()
tormod.py
Executable
+55
View File
@@ -0,0 +1,55 @@
#!/usr/bin/env python3.11
import argparse
import io
from pathlib import Path
import torf
def parse_args() -> dict:
arger = argparse.ArgumentParser()
arger.add_argument('torrent_path', type=Path, help='Path to torrent file')
arger.add_argument('--name', type=Path, help='New filename', required=True)
return dict(arger.parse_args().__dict__)
def modify_torrent(
torrent_file: bytes,
name: str,
announce_url: str,
created_by: str = '',
comment: str = '',
private: bool = True,
) -> bytes:
t = torf.Torrent.read_stream(io.BytesIO(torrent_file))
if len(t.files) > 1:
raise ValueError('Torrent contains multiple files')
t.name = name
t.private = private
t.comment = comment
t.created_by = created_by
t.trackers.clear()
t.trackers.insert(0, announce_url)
return t.dump()
def main():
args = parse_args()
torrent_path: Path = args.pop('torrent_path')
modded = modify_torrent(
**args,
torrent_file=torrent_path.read_bytes(),
announce_url='http://tracker.empornium.sx:2710/tegqucis10uanp672qh6nhn393xlkncs/announce',
created_by='zzzp',
comment='created by zzzp',
private=True,
)
save_path = torrent_path.with_stem(f'{torrent_path.stem}.mod')
save_path.write_bytes(modded)
if __name__ == '__main__':
main()
torrentify.py
+8 -2
View File
@@ -4,6 +4,7 @@ import json
import logging import logging
import os import os
import re import re
import select
import shutil import shutil
import sys import sys
import time import time
@@ -210,7 +211,10 @@ def upload_image(image_path: Path) -> str:
return image_url return image_url
ParsedFilename = typing.TypedDict('ParsedFilename', {'actors': list[str], 'studio': str, 'date': str, 'title': str, 'tags': list[str]}) ParsedFilename = typing.TypedDict(
'ParsedFilename', {'actors': list[str], 'studio': str, 'date': str, 'title': str, 'tags': list[str]}
)
def parse_filename(filename: str) -> ParsedFilename: def parse_filename(filename: str) -> ParsedFilename:
parsed = { parsed = {
@@ -244,6 +248,7 @@ def parse_filename(filename: str) -> ParsedFilename:
return parsed return parsed
def generate_title(parsed: ParsedFilename) -> str: def generate_title(parsed: ParsedFilename) -> str:
parts = [] parts = []
if actors := parsed.get('actors'): if actors := parsed.get('actors'):
@@ -473,7 +478,8 @@ def main():
thumbnail_path = video_path.with_suffix(".jpg") thumbnail_path = video_path.with_suffix(".jpg")
if old_thumb_path.is_file(): if old_thumb_path.is_file():
old_thumb_path.rename(thumbnail_path) old_thumb_path.rename(thumbnail_path)
ffmpeg.make_thumbnail_tile(video_path, image_path=thumbnail_path, skip_if_exists=True) image_jpeg = ffmpeg.make_thumbnail_tile(video_path)
thumbnail_path.write_bytes(image_jpeg)
logging.info(f"Saved thumbnails at {thumbnail_path}") logging.info(f"Saved thumbnails at {thumbnail_path}")
post_bbcode = generate_post_bbcode(video_path, thumbnail_path) post_bbcode = generate_post_bbcode(video_path, thumbnail_path)
torrentify_test.py
-60
View File
@@ -1,60 +0,0 @@
from cmath import exp
from torrentify import generate_title, parse_filename, ParsedFilename
import pytest
@pytest.mark.parametrize(
["filename", "expected"],
[
[
"To Ki -- @Studio -- 2022-10-04",
{
"actors": ["To Ki"],
"date": "2022-10-04",
"studio": "Studio",
"tags": ["to.ki", "studio.com", "2022", "2022.10"],
},
],
[
"To Ki, Ki To -- @Studio -- Title -- 2022-10-04",
{
"actors": ["To Ki", "Ki To"],
"date": "2022-10-04",
"studio": "Studio",
"title": "Title",
"tags": ["to.ki", "ki.to", "studio.com", "2022", "2022.10"],
},
],
],
)
def test_parse_filename(filename: str, expected: dict):
parsed = parse_filename(filename)
print(filename, parsed)
assert parsed == expected
@pytest.mark.parametrize(
["parsed", "expected"],
[
[
{
"actors": ["To Ki"],
"date": "2022-10-04",
"studio": "Studio",
},
'To Ki -- @Studio -- 2022-10-04',
],
[
{
"actors": ["To Ki", "Ki To"],
"date": "2022-10-04",
"title": "Title",
"studio": "Studio",
},
'To Ki, Ki To -- @Studio -- Title -- 2022-10-04',
],
],
)
def test_generate_title(parsed: ParsedFilename, expected: str):
generated = generate_title(parsed)
assert generated == expected