snapshot
This commit is contained in:
@@ -0,0 +1 @@
|
||||
{"username": "sssmmt@gmail.com", "password": "2126484"}
|
||||
+34
-27
@@ -1,12 +1,12 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import contextlib
|
||||
import dataclasses
|
||||
import datetime
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
@@ -39,7 +39,16 @@ dump_dir = Path('~/Desktop').expanduser() / 'immigration'
|
||||
dump_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
async def check_slots(page: Page, retry_count: int = 1) -> list[datetime.datetime]:
|
||||
@dataclasses.dataclass
|
||||
class ResidencyApplication:
|
||||
citizenship_country: str
|
||||
num_applicants: int
|
||||
is_living_with_family: bool
|
||||
residency_category: str
|
||||
residency_purpose: str
|
||||
|
||||
|
||||
async def check_slots(page: Page, values: ResidencyApplication, retry_count: int = 1) -> list[datetime.datetime]:
|
||||
await page.add_init_script('''Object.defineProperty(navigator, 'webdriver', { get: () => false })''')
|
||||
window_id = random.randint(1000, 9999)
|
||||
req_id = random.randint(0, 999)
|
||||
@@ -47,45 +56,37 @@ async def check_slots(page: Page, retry_count: int = 1) -> list[datetime.datetim
|
||||
f'https://otv.verwalt-berlin.de/ams/TerminBuchen/wizardng?dswid={window_id}&dsrid={req_id}',
|
||||
wait_until='networkidle',
|
||||
)
|
||||
|
||||
async with page.expect_navigation():
|
||||
await page.click('.langBlock [name="txtEn"] a')
|
||||
|
||||
await page.check('[name="gelesen"]')
|
||||
async with page.expect_navigation(url=re.compile('st=2')):
|
||||
|
||||
async with page.expect_navigation(url=re.compile('st=2'), wait_until='networkidle'):
|
||||
await page.click('[name="applicationForm:managedForm:proceed"]')
|
||||
|
||||
await page.get_by_role("combobox", name="Staatsangehörigkeit *").select_option(label="Türkei")
|
||||
await page.get_by_role("combobox", name="Citizenship *").select_option(label=values.citizenship_country)
|
||||
await asyncio.sleep(0.3)
|
||||
|
||||
await page.get_by_role(
|
||||
"combobox",
|
||||
name="Anzahl der Personen, die einen Aufenthaltstitel beantragen (auch ausländische Ehepartner und Kinder) *",
|
||||
).select_option(label="eine Person")
|
||||
# how many applicants
|
||||
await page.select_option('[name="personenAnzahl_normal"]', value=str(values.num_applicants))
|
||||
await asyncio.sleep(0.3)
|
||||
|
||||
await page.get_by_role(
|
||||
"combobox", name="Leben Sie in Berlin zusammen mit einem Familienangehörigen (z.B. Ehepartner, Kind) *"
|
||||
).select_option(label="nein")
|
||||
# living with family
|
||||
await page.select_option('[name="lebnBrMitFmly"]', value='1' if values.is_living_with_family else '2')
|
||||
await asyncio.sleep(0.3)
|
||||
|
||||
await page.get_by_text("Aufenthaltstitel - beantragen").click()
|
||||
await page.locator("label").filter(has_text="Erwerbstätigkeit").click()
|
||||
await page.get_by_text("Blaue Karte EU (§ 18b Abs. 2)").click()
|
||||
await page.get_by_text("Apply for a residence title").click()
|
||||
await page.locator("label").filter(has_text=values.residency_category).click()
|
||||
await page.get_by_text(values.residency_purpose).click()
|
||||
|
||||
stop_at = datetime.datetime.now() + datetime.timedelta(minutes=28)
|
||||
|
||||
async def go_forward():
|
||||
async with page.expect_navigation(url=re.compile(r'st='), wait_until='networkidle', timeout=60_000):
|
||||
await page.get_by_role("button", name="Weiter").click()
|
||||
if 'st=2' in page.url:
|
||||
error_message = await page.inner_text('.errorMessage')
|
||||
if 'keine Termine frei' in error_message:
|
||||
return False
|
||||
raise Error(error_message)
|
||||
return 'st=3' in page.url
|
||||
|
||||
while 'st=2' in page.url:
|
||||
retry_count -= 1
|
||||
|
||||
async with page.expect_navigation(url=re.compile(r'st='), wait_until='networkidle', timeout=60_000):
|
||||
await page.get_by_role("button", name="Weiter").click()
|
||||
await page.get_by_role("button", name="Next").click()
|
||||
|
||||
if datetime.datetime.now() > stop_at:
|
||||
raise TimeoutError('could not find a slot')
|
||||
@@ -134,7 +135,14 @@ async def main():
|
||||
args = parse_args()
|
||||
async with launch_browser(headless=args.headless) as page:
|
||||
try:
|
||||
slots = await check_slots(page, retry_count=args.retry_count)
|
||||
values = ResidencyApplication(
|
||||
citizenship_country='India',
|
||||
num_applicants=1,
|
||||
is_living_with_family=False,
|
||||
residency_category='Educational purposes',
|
||||
residency_purpose='Residence permit for the purpose of studying (sect. 16b)',
|
||||
)
|
||||
slots = await check_slots(page, values=values, retry_count=args.retry_count)
|
||||
if not slots:
|
||||
print('no slots yet')
|
||||
return
|
||||
@@ -146,4 +154,3 @@ async def main():
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
main()
|
||||
|
||||
+782
@@ -0,0 +1,782 @@
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import typing
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
import paramiko
|
||||
import psycopg
|
||||
import psycopg.rows
|
||||
|
||||
import ffmpeg
|
||||
|
||||
VIDS_DIR = Path(r"/Volumes/FIVER/_ingress/")
|
||||
schema = """
|
||||
create table if not exists videos(
|
||||
file_path text not null primary key,
|
||||
size_bytes int not null,
|
||||
hash_partial text not null,
|
||||
ffprobe json not null
|
||||
);
|
||||
|
||||
create table if not exists scan_state(
|
||||
file_path text not null primary key,
|
||||
stderr text
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def find_vids(root: Path):
|
||||
extensions = [".mkv", ".mp4"]
|
||||
for ext in extensions:
|
||||
for f in root.rglob(f"*{ext}"):
|
||||
yield f
|
||||
|
||||
|
||||
def hash_partial(f: Path) -> str:
|
||||
sha1 = hashlib.sha1()
|
||||
chunk_size = 1024 * 1024 * 10 # 10MB chunk size
|
||||
|
||||
total_read = 0
|
||||
with f.open("rb") as file:
|
||||
while chunk := file.read(chunk_size):
|
||||
total_read += len(chunk)
|
||||
sha1.update(chunk)
|
||||
break # Only reads the first 10MB
|
||||
|
||||
return f"sha1:{total_read}:{sha1.hexdigest()}"
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def connect_db_sqlite() -> typing.Generator[psycopg.Connection, typing.Any, typing.Any]:
|
||||
with sqlite3.connect("vids.sqlite3") as conn:
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA foreign_keys = 1")
|
||||
yield conn
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def connect_db() -> typing.Generator[psycopg.Connection, typing.Any, typing.Any]:
|
||||
with psycopg.connect("postgres://abdus:abdus@db.abdus.dev:5444/smut?sslmode=disable") as conn:
|
||||
conn.row_factory = psycopg.rows.dict_row
|
||||
yield conn
|
||||
|
||||
|
||||
def video_exists(con: psycopg.Connection, file_path: str) -> bool:
|
||||
stmt = con.execute(
|
||||
"select exists(select 1 from videos where file_path = %s) as exists",
|
||||
[file_path],
|
||||
)
|
||||
row = stmt.fetchone()
|
||||
return row["exists"]
|
||||
|
||||
|
||||
def delete_video(con: psycopg.Connection, file_path: str) -> bool:
|
||||
con.execute("delete from videos where file_path = %s", [file_path])
|
||||
|
||||
|
||||
def save_video(
|
||||
con: psycopg.Connection,
|
||||
category: str,
|
||||
file_path: str,
|
||||
ffprobe: dict,
|
||||
hash_partial: str,
|
||||
):
|
||||
sql = """insert into videos(category, file_path, ffprobe, hash_partial) values (%s, %s, %s, %s);"""
|
||||
con.execute(
|
||||
sql,
|
||||
[
|
||||
category,
|
||||
file_path,
|
||||
json.dumps(ffprobe),
|
||||
hash_partial,
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def save_error(con: psycopg.Connection, file_path: str, stderr: str):
|
||||
sql = """insert into scan_state(file_path, stderr) values (%s, %s) on conflict do nothing"""
|
||||
con.execute(
|
||||
sql,
|
||||
[
|
||||
file_path,
|
||||
stderr,
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def scan_videos():
|
||||
logging.basicConfig(format=logging.BASIC_FORMAT, level=logging.DEBUG)
|
||||
with connect_db() as con:
|
||||
# with con:
|
||||
# con.executescript(schema)
|
||||
|
||||
all_vids = list(find_vids(VIDS_DIR))
|
||||
for i, f in enumerate(all_vids):
|
||||
if "/_picks/" in str(f):
|
||||
continue
|
||||
progress = f"[{i+1}/{len(all_vids)}]"
|
||||
if f.is_dir():
|
||||
continue
|
||||
|
||||
file_path = str(f.absolute())
|
||||
|
||||
if video_exists(con, file_path=file_path):
|
||||
logging.debug(f"{progress} video already exists {f=}")
|
||||
continue
|
||||
|
||||
logging.debug(f"probing {f=}")
|
||||
try:
|
||||
probed = ffmpeg.ffprobe(f)
|
||||
except subprocess.CalledProcessError as e:
|
||||
save_error(con, file_path=file_path, stderr=e.stderr)
|
||||
continue
|
||||
|
||||
file_hash = hash_partial(f)
|
||||
|
||||
save_video(
|
||||
category="downloaded.fiver",
|
||||
con=con,
|
||||
file_path=file_path,
|
||||
ffprobe=probed.__dict__,
|
||||
hash_partial=file_hash,
|
||||
)
|
||||
logging.info(f"{progress} saved {f=}")
|
||||
con.commit()
|
||||
|
||||
|
||||
def clean_records():
|
||||
all_vids = set(map(str, find_vids(VIDS_DIR)))
|
||||
|
||||
with connect_db() as con:
|
||||
for row in con.execute("select file_path from videos"):
|
||||
file_path = row["file_path"]
|
||||
if file_path not in all_vids:
|
||||
delete_video(con, file_path=row["file_path"])
|
||||
|
||||
|
||||
def unwanted(dry_run: bool):
|
||||
take = set(
|
||||
r"""
|
||||
aidra fox
|
||||
rachel starr
|
||||
abella danger
|
||||
sata jones
|
||||
octavia red
|
||||
paige owens
|
||||
lily lou
|
||||
cj miles
|
||||
arabelle raphael
|
||||
chloe amour
|
||||
erin everheart
|
||||
kenzie reeves
|
||||
alex grey
|
||||
alicia trece
|
||||
angel youngs
|
||||
angela white
|
||||
april olsen
|
||||
blake blossom
|
||||
brandy renee
|
||||
chantal
|
||||
demi sutra
|
||||
eden ivy
|
||||
ema karter
|
||||
emma hix
|
||||
gianna dior
|
||||
hazel moore
|
||||
jazmin luv
|
||||
jessy jey
|
||||
josephine jackson
|
||||
katsaros
|
||||
kira noir
|
||||
kitana lure
|
||||
kitana montana
|
||||
kristy black
|
||||
leal
|
||||
lia lin
|
||||
lily labeau
|
||||
liya silver
|
||||
maddy may
|
||||
nata ocean
|
||||
rebel rhyder
|
||||
rein
|
||||
savannah bond
|
||||
scarlett alexis
|
||||
scarlit scandal
|
||||
shalina
|
||||
simon kitty
|
||||
slimthick vic
|
||||
stratton
|
||||
tatum
|
||||
tommy king
|
||||
vanna bardot
|
||||
vic marie
|
||||
violet starr
|
||||
vittoria
|
||||
emily willis
|
||||
THUNDERCOCK
|
||||
lucky bee
|
||||
charlotte sins
|
||||
mandy muse
|
||||
skye blue
|
||||
alexis tae
|
||||
willow
|
||||
TRUEANAL
|
||||
ALLANAL
|
||||
avery cristy
|
||||
ANALONLY
|
||||
SWALLOWED
|
||||
kelly collins
|
||||
nicole black
|
||||
xxlayna marie
|
||||
katrina colt
|
||||
allie addison
|
||||
DIRTYAUDITIONS
|
||||
scarlet chase
|
||||
#sloppiest kiss
|
||||
liz ocean
|
||||
medusa
|
||||
nicole aria
|
||||
stacy bloom
|
||||
lara frost
|
||||
jamie jett
|
||||
brooklyn gray
|
||||
kimmy granger
|
||||
kelsey kane
|
||||
kay lovely
|
||||
ivy wolfe
|
||||
stefany kyler
|
||||
leo ahsoka
|
||||
lydia black
|
||||
renee rose
|
||||
emily jade
|
||||
anya olsen
|
||||
alexis crystal
|
||||
sybil
|
||||
megan rain
|
||||
charlotte sartre
|
||||
madison ivy
|
||||
maddy black
|
||||
gina valentina
|
||||
lacey jayne
|
||||
hailey rose
|
||||
graycee baybee
|
||||
katie kush
|
||||
lilu moon
|
||||
juis wild
|
||||
kylie rocket
|
||||
dolly dyson
|
||||
nichole saphir
|
||||
Octokuro
|
||||
reislin
|
||||
isabella de laa
|
||||
qween goddess
|
||||
INTHECRACK
|
||||
athenea rose
|
||||
daniela garcia
|
||||
kama oxi
|
||||
mina
|
||||
baby nicols
|
||||
jade amor
|
||||
daniela ortiz
|
||||
NYMPHO
|
||||
""".strip().splitlines(keepends=False)
|
||||
)
|
||||
drop = set(
|
||||
r"""
|
||||
ADDICTED2GIRLS
|
||||
ASIANSEEPLOITED
|
||||
BANANAFEVER
|
||||
BBCPOVD
|
||||
BFFS
|
||||
# BRAZZERSEXXTRA
|
||||
BRIDE4K
|
||||
CHLOELAMOUR
|
||||
DRKINLA
|
||||
ETERNALDESIRE
|
||||
EVERYTHINGBUTT
|
||||
FAKEHOSTEL
|
||||
FAKEHUBORIGINALS
|
||||
FAKETAXI
|
||||
FAMILYSINNERS
|
||||
FEETISHPOV
|
||||
GIRLGIRLXXX
|
||||
GOTFILLED
|
||||
HOOKUPHOTSHOT
|
||||
HOWWOMENORGASM
|
||||
IMMORALLIVE
|
||||
IWANTCLIPS
|
||||
JACQUIEETMICHELTV
|
||||
JERKAOKE
|
||||
NICONICE
|
||||
NURUMASSAGE
|
||||
OHMYHOLES
|
||||
OYELOCA
|
||||
PASCALSSUBSLUTS
|
||||
SCHOOLOFCOCK
|
||||
SHOTHERFIRST
|
||||
SLIPPERYMASSAGE
|
||||
TABOOHEAT
|
||||
TEAMSKEETX
|
||||
WAXXXED
|
||||
WOODMANCASTINGX
|
||||
PISSVIDS
|
||||
LEGALPORNO
|
||||
ANALVIDS
|
||||
PORNBOX
|
||||
\btap\b
|
||||
alice xo
|
||||
alicia williams
|
||||
mona azar
|
||||
anna de ville
|
||||
baby kxtten
|
||||
bella rolland
|
||||
betzz
|
||||
charlie forde
|
||||
charlie red
|
||||
cheyla collins
|
||||
claudia garcia
|
||||
cory chase
|
||||
dani blue
|
||||
dellai
|
||||
eimy zoren
|
||||
emily pink
|
||||
gia dibella
|
||||
greta foss
|
||||
danielle renae
|
||||
harley king
|
||||
ivy maddox
|
||||
sasha rose
|
||||
olive glass
|
||||
katherin moore
|
||||
kristina grace
|
||||
kylie page
|
||||
larissa leite
|
||||
lily starfire
|
||||
lina arian
|
||||
luna legend
|
||||
luna rishi
|
||||
luna wolfs
|
||||
lya cutie
|
||||
margan lee
|
||||
marie berger
|
||||
marilyn johnson
|
||||
mia cheers
|
||||
monika fox
|
||||
moona snake
|
||||
nichole saphir
|
||||
nicole murkovski
|
||||
olivia sparkle
|
||||
polly petrova
|
||||
pornworld
|
||||
publicagent
|
||||
publicbang
|
||||
stacy cruz
|
||||
sussy sweet
|
||||
triple anal
|
||||
vanessa vega
|
||||
venera maxima
|
||||
victoria nyx
|
||||
whitney wright
|
||||
zirael rem
|
||||
""".lower()
|
||||
.strip()
|
||||
.splitlines(keepends=False)
|
||||
)
|
||||
|
||||
take_patterns = [re.compile(p.strip().replace(" ", "."), flags=re.I) for p in take if not p.strip().startswith("#")]
|
||||
drop_patterns = [re.compile(p.strip().replace(" ", "."), flags=re.I) for p in drop if not p.strip().startswith("#")]
|
||||
|
||||
to_delete = set()
|
||||
with connect_db() as con:
|
||||
for row in con.execute("select file_path from videos where category = 'box'"):
|
||||
file_path = Path(row["file_path"])
|
||||
text = re.sub(r"[.]+", " ", file_path.stem.lower())
|
||||
|
||||
if any(p.search(text) for p in drop_patterns):
|
||||
to_delete.add(file_path)
|
||||
if any(p.search(text) for p in take_patterns):
|
||||
to_delete.discard(file_path)
|
||||
|
||||
for it in sorted(to_delete):
|
||||
print(it)
|
||||
|
||||
return
|
||||
|
||||
with connect_db() as con:
|
||||
for path in sorted(to_delete, key=str):
|
||||
try:
|
||||
print(path.stem)
|
||||
con.execute(
|
||||
"update videos set marked_for_deletion_at = now() where file_path = %s",
|
||||
[str(path)],
|
||||
)
|
||||
if not dry_run:
|
||||
path.unlink()
|
||||
delete_video(con, file_path=str(path))
|
||||
con.commit()
|
||||
except:
|
||||
pass
|
||||
|
||||
print(len(to_delete))
|
||||
|
||||
|
||||
def large_unwanted():
|
||||
with connect_db() as con:
|
||||
for row in con.execute("select file_path from videos order by size_bytes desc limit 500"):
|
||||
file_path = Path(row["file_path"])
|
||||
print(file_path)
|
||||
|
||||
|
||||
def list_studios():
|
||||
re_studio = re.compile(r"^(.+?)\.\d+", flags=re.IGNORECASE)
|
||||
seen = defaultdict(int)
|
||||
with connect_db() as con:
|
||||
for row in con.execute("select file_path from videos"):
|
||||
file_path = Path(row["file_path"])
|
||||
m = re_studio.search(file_path.stem.lower())
|
||||
if m:
|
||||
seen[m.group(1)] += 1
|
||||
|
||||
for it, n in Counter(seen).most_common(20):
|
||||
print(it, n)
|
||||
|
||||
|
||||
def lower_resolutions(dry_run: bool):
|
||||
with connect_db() as con:
|
||||
for row in con.execute("""select file_path from videos where file_path like '%720p%';"""):
|
||||
file_path = Path(row["file_path"])
|
||||
video_1080p_path = file_path.with_stem(file_path.stem.replace("720p", "1080p"))
|
||||
if video_exists(con, file_path=str(video_1080p_path)) and video_1080p_path.is_file():
|
||||
print(file_path)
|
||||
if not dry_run:
|
||||
file_path.unlink()
|
||||
|
||||
|
||||
def dupes():
|
||||
sql = """
|
||||
with durationed as (
|
||||
select v.*, json_extract(v.ffprobe, '$.duration_sec') as duration_sec from videos v
|
||||
)
|
||||
select v.file_path, dupe.file_path from durationed v
|
||||
join durationed dupe on v.file_path != dupe.file_path and abs(dupe.duration_sec - v.duration_sec) < 3
|
||||
where abs(dupe.size_bytes - v.size_bytes) < 3000000
|
||||
"""
|
||||
with connect_db() as con:
|
||||
for row in con.execute(sql):
|
||||
print(row[0], row[1])
|
||||
print()
|
||||
|
||||
|
||||
def different_format(dry_run: bool):
|
||||
sql = """
|
||||
select file_path from videos where file_path like '%.mkv'
|
||||
"""
|
||||
with connect_db() as con:
|
||||
for row in con.execute(sql):
|
||||
mkv_path = Path(row["file_path"])
|
||||
mp4_path = mkv_path.with_suffix(".mp4")
|
||||
if mkv_path.is_file() and mp4_path.is_file():
|
||||
if abs(mp4_path.stat().st_size - mkv_path.stat().st_size) < 5_000_000:
|
||||
print(mkv_path)
|
||||
if not dry_run:
|
||||
mkv_path.unlink()
|
||||
|
||||
|
||||
def count_shared_words(a: str, b: str) -> int:
|
||||
a_words = set(re.split(r"[\s\-.]+", a.lower()))
|
||||
b_words = set(re.split(r"[\s\-.]+", b.lower()))
|
||||
|
||||
return len(a_words.intersection(b_words))
|
||||
|
||||
|
||||
def reencoded_self(dry_run: bool):
|
||||
sql = """
|
||||
with
|
||||
durationed_v as (
|
||||
select v.*, json_extract(v.ffprobe, '$.duration_sec') as duration_sec from videos v
|
||||
),
|
||||
durationed_dl as (
|
||||
select v.*, json_extract(v.ffprobe, '$.duration_sec') as duration_sec from downloaded_videos v
|
||||
)
|
||||
select v.file_path, dlv.file_path from durationed_v v
|
||||
join durationed_dl dlv on abs(v.duration_sec - dlv.duration_sec) < 0.1
|
||||
where dlv.file_path like '%/_reenc/%'
|
||||
"""
|
||||
|
||||
with connect_db() as con:
|
||||
for row in con.execute(sql):
|
||||
v_path = Path(row[0])
|
||||
dlv_path = Path(row[1])
|
||||
if count_shared_words(v_path.stem, dlv_path.stem) < 2:
|
||||
continue
|
||||
|
||||
print(v_path.name, dlv_path.name, sep="\n")
|
||||
if not dry_run:
|
||||
try:
|
||||
v_path.unlink()
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def local_duped(dry_run: bool):
|
||||
sql = """
|
||||
with durationed_dl as (
|
||||
select v.*, json_extract(v.ffprobe, '$.duration_sec') as duration_sec from downloaded_videos v
|
||||
)
|
||||
select v.file_path, dupe.file_path
|
||||
from durationed_dl v
|
||||
join durationed_dl dupe on
|
||||
v.id != dupe.id
|
||||
and v.id > dupe.id
|
||||
and abs(dupe.duration_sec - v.duration_sec) < 0.1
|
||||
and abs(dupe.size_bytes - v.size_bytes) < 3000000
|
||||
"""
|
||||
|
||||
root = Path(r"/Volumes/FIVER/_ingress/")
|
||||
with connect_db() as con:
|
||||
for row in con.execute(sql):
|
||||
a_path = Path(row[0])
|
||||
b_path = Path(row[1])
|
||||
if count_shared_words(a_path.stem, b_path.stem) < 6:
|
||||
continue
|
||||
|
||||
# print(a_path.name, b_path.name, sep='\n')
|
||||
keep, remove = a_path, b_path
|
||||
if len(b_path.stem) > len(a_path.stem):
|
||||
keep, remove = b_path, a_path
|
||||
print(" # ", "matching", keep.name)
|
||||
print(" rm", f'"{remove}"')
|
||||
print()
|
||||
|
||||
|
||||
def delete_dupes():
|
||||
rows = []
|
||||
|
||||
with connect_db() as con:
|
||||
for row in rows:
|
||||
file_path = Path(row["file_path"])
|
||||
try:
|
||||
file_path.unlink()
|
||||
delete_video(con, file_path=row["file_path"])
|
||||
con.commit()
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def save_suggested_names():
|
||||
rows = []
|
||||
|
||||
with connect_db() as con:
|
||||
for row in rows:
|
||||
bandaid_path = Path(row["file_path"])
|
||||
fiver_path = Path(row["fiver_path"])
|
||||
|
||||
if bandaid_path.stem == fiver_path.stem:
|
||||
continue
|
||||
|
||||
if count_shared_words(bandaid_path.stem, fiver_path.stem) < 3:
|
||||
print("unsimilar", row)
|
||||
continue
|
||||
|
||||
if bandaid_path.stem.count("--") == 3:
|
||||
better_stem = bandaid_path.stem
|
||||
else:
|
||||
better_stem = fiver_path.stem
|
||||
|
||||
better_stem = re.sub(r"(\d{4})(\d{2})(\d{2})", r"\1-\2-\3", better_stem)
|
||||
|
||||
if better_stem == fiver_path.stem:
|
||||
continue
|
||||
|
||||
print(fiver_path.name, better_stem, sep="\n")
|
||||
|
||||
con.execute(
|
||||
"update videos set suggested_filename = %s where file_path = %s",
|
||||
[better_stem, fiver_path.name],
|
||||
)
|
||||
con.commit()
|
||||
|
||||
|
||||
def delete_remote_files():
|
||||
rows = [
|
||||
{"file_path": "/mnt/box/files/_raw/prt/5KPorn.24.10.02.Leya.Desantis.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/AllAnal.24.10.30.Aviana.Violet.And.Nicole.Aria.XXX.720p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/AllAnal.24.11.07.Khloe.Kingsley.And.Cherry.Kiss.XXX.720p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/AnalOnly.24.11.26.Aria.Sloane.And.Nicole.Doshi.XXX.720p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/AnalTherapyXXX.24.02.10.Willow.Ryder.Family.Competition.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/AngelsLove.23.07.29.Cherry.Candle.And.Stacy.Cruz.Hot.Juice.Is.Dripping.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/BBCPie.20.05.31.Hazel.Moore.Morning.Wood.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/BangBros18.24.06.16.Angel.Gostosa.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/BangPOV.24.02.20.Justine.Jakobs.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/Blacked.22.10.29.Kay.Lovely.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/BlacksOnBlondes.22.11.29.Hazel.Moore.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/BlacksOnBlondes.24.11.22.Kylie.Rocket.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/BrattySis.22.06.10.Angel.Gostosa.Everyone.Wants.To.Fuck.My.Stepsister.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/BrattySis.23.06.23.Angel.Gostosa.My.Stepsister.Takes.Charge.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/BrattySis.24.05.10.Jade.Maris.And.Vanessa.Marie.Sharing.Stepbros.Bed.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/BrattySis.24.05.31.Renee.Rose.Nobody.Wants.To.Fuck.A.Virgin.Stepbro.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/BrattySis.24.07.26.Jade.Maris.And.Sera.Ryder.I.Want.To.Bone.My.Stepsister.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/BrazzersExxtra.24.08.21.Destiny.Mira.Cyclist.Sweat.Leads.To.Shower.Sex.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/BrazzersExxtra.24.10.20.Baby.Gemini.Dripping.Wet.Hookup.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/BrazzersExxtra.24.11.04.Nichole.Saphir.Tattooed.Bombshells.Gaping.Anal.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/BrazzersExxtra.24.11.20.Katie.Kush.And.Kelly.Caprice.Free.Use.Maid.Service.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/BrazzersExxtra.24.11.20.Kira.Noir.Secret.Goth.Camgirl.Roomie.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/CherryPimps.Busted.24.08.14.Lacey.Jayne.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/CrazyCollegeGFs.24.02.20.Destiny.Mira.Bunk.Banging.The.Handyman.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/Creampie-Angels.22.07.03.Nata.Ocean.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/CumSwappingSis.24.03.16.Jade.Maris.And.Liz.Jordan.Giving.Stepbro.Something.To.Do.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/DFXSoleMates.24.06.15.Angel.Gostosa.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/DatingMyStepson.24.10.03.Shalina.Devine.A.Game.Of.Dare.Always.Spices.Things.Up.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/DirtyAuditions.24.08.17.Kazumi.XXX.720p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/ElegantAngel.24.01.01.Hazel.Moore.A.Touch.Of.Class.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/ElegantAngel.24.11.29.Scarlett.Alexis.Stretch.My.Hole.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/EvilAngel.24.06.01.Chloe.Amour.XXX.1080p.HEVC.x265.PRT.mkv"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/EvilAngel.24.11.01.Princess.Alice.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/FilthyTaboo.24.10.19.Kelly.Caprice.Unsatisfied.MILF.Gets.What.She.Craves.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/GirlGirlXXX.24.01.16.Kylie.Rocket.And.Melissa.Stratton.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/Hegre.24.10.22.Anna.L.Erotic.Goddess.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/HotwifeXXX.24.11.06.Willow.Ryder.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/InTheCrack.E1928.Lia.Lin.Provence.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/KarupsPC.20.04.01.Nata.Ocean.Horny.Housekeeper.XXX.1080p.HEVC.x265.PRT.mkv"},
|
||||
{
|
||||
"file_path": "/mnt/box/files/_raw/prt/LegalPorno.23.11.10.Lady.Ana.Jureka.Del.Mar.Fisting.ATOGM.Double.Anal.Fist.Big.Gapes.Monster.ButtRose.Squirt.Creampie.Swallow.GIO2635.XXX.1080p.HEVC.x265.PRT.mkv"
|
||||
},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/LegalPorno.24.06.19.Kristy.Black.GIO2815.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/LegalPorno.24.10.02.Leila.Botwin.AH007.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/LegendaryX.24.01.11.Melissa.Stratton.And.Nicole.Doshi.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/LegendaryX.24.02.01.Chantal.Danielle.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/LilSis.24.05.17.Jade.Maris.Let.The.Games.Begin.Part.1.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/LittleCaprice-Dreams.24.11.09.Tiffany.Tatum.And.Nata.Ocean.Nasstyx.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/MilfBody.24.06.14.Kelly.Caprice.Kellys.New.Routine.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/MomIsHorny.23.12.29.Justine.Jakobs.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/MomXXX.24.11.05.Kiara.Lord.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/MomsTeachSex.23.11.24.Justine.Jakobs.Getting.Down.To.Business.XXX.1080p.HEVC.x265.PRT.mkv"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/MrLuckyPOV.24.01.05.Gianna.Dior.And.Kazumi.Squirts.An.All.Star.Threesome.XXX.1080p.HEVC.x265.PRT.mkv"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/MrLuckyPOV.24.01.05.Gianna.Dior.And.Kazumi.Squirts.An.All.Star.Threesome.XXX.1080p.HEVC.x265.PRT.mkv"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/MySistersHotFriend.24.04.09.Willow.Ryder.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/NewSensations.24.11.16.Ellie.Nova.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/NubileFilms.23.10.17.Liz.Jordan.My.Roommate.Has.A.Crush.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/NubileFilms.23.11.20.Hailey.Rose.And.Jade.Maris.Happy.Birthday.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/NubileFilms.24.02.01.Liz.Jordan.February.2024.Fantasy.Of.The.Month.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/NubileFilms.24.03.01.Jade.Maris.March.2024.Fantasy.Of.The.Month.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/NubileFilms.24.12.01.Chloe.Temple.December.2024.Fantasy.Of.The.Month.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/Nubiles-Casting.24.10.28.Chanel.Camryn.Cast.Addison.Vodka.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/PascalsSubSluts.21.06.25.Rebel.Rhyder.Every.Inch.Every.Hole.XXX.1080p.HEVC.x265.PRT.mkv"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/Passion-HD.24.04.03.Aria.Valencia.Good.Help.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{
|
||||
"file_path": "/mnt/box/files/_raw/prt/PissVids.23.05.20.Juis.Wild.Black.Pee.1on1.BBC.ATM.Balls.Deep.No.Pussy.Rough.Sex.Gapes.Pee.Drink.Cum.In.Mouth.Swallow.GL836.XXX.1080p.HEVC.x265.PRT.mp4"
|
||||
},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/PornMegaLoad.24.07.18.Justine.Jakobs.Hardcore.40443.XXX.1080p.HEVC.x265.PRT.mkv"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/PornPlus.24.10.18.Hailey.Rose.Naturally.Gifted.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/Private.23.11.18.Dolly.Dyson.Enjoys.An.Anal.Threesome.With.Mary.Popiense.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/Private.24.11.18.Lia.Lin.Gets.Horny.By.The.Pool.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/RKPrime.24.10.28.Octokuro.Fucking.After.The.Club.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/RickysRoom.23.03.09.Destiny.Mira.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/SexSelector.23.03.17.Graycee.Baybee.XXX.1080p.HEVC.x265.PRT.mkv"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/StrapLez.23.08.17.Alexis.Crystal.And.Lilly.Bella.Everything.She.Wants.XXX.1080p.HEVC.x265.PRT.mkv"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/StrapLez.24.11.14.Cherry.Candle.And.Lilly.Mays.Pleasure.Control.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/StrapLez.24.11.28.Alexis.Crystal.And.Mia.Trejsi.Rhythm.Of.Desire.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/TeensLoveHugeCocks.23.04.28.Kylie.Rocket.Influencers.Delight.XXX.1080p.HEVC.x265.PRT.mkv"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/ThePOVGod.24.10.25.Willow.Ryder.The.Call.Of.That.Booty.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
{"file_path": "/mnt/box/files/_raw/prt/Tiny4K.23.02.02.Liz.Jordan.Ice.Cream.Treat.XXX.1080p.HEVC.x265.PRT.mp4"},
|
||||
]
|
||||
with connect_server("klein") as ssh, connect_db() as con:
|
||||
for row in rows:
|
||||
file_path = row["file_path"]
|
||||
try:
|
||||
print("deleting", file_path)
|
||||
ssh.exec_command(f'rm -rf "{file_path}"')
|
||||
delete_video(con, file_path=file_path)
|
||||
con.commit()
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def connect_server(host: str) -> typing.Generator[paramiko.SSHClient, typing.Any, None]:
|
||||
config = paramiko.SSHConfig.from_path(str(Path(r"~/.ssh/config").expanduser()))
|
||||
key = paramiko.RSAKey.from_private_key_file(filename=str(Path(r"~/.ssh/id_rsa").expanduser()))
|
||||
|
||||
target = config.lookup(host)
|
||||
|
||||
ssh_client = paramiko.SSHClient()
|
||||
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh_client.connect(
|
||||
hostname=target["hostname"],
|
||||
port=target.as_int("port"),
|
||||
username=target["user"],
|
||||
pkey=key,
|
||||
)
|
||||
|
||||
yield ssh_client
|
||||
|
||||
ssh_client.close()
|
||||
|
||||
|
||||
def collect_all():
|
||||
root = Path("/Volumes/FATBLUE/_backup/_vidz/_all")
|
||||
video_files = find_vids(root)
|
||||
|
||||
# re_good = re.compile(r"(?P<people>.+) - (?P<title>.+) --? (?P<studio>\S+) (?P<date_year>\d{4})(?P<date_month>\d{2})(?P<date_day>\d{2})")
|
||||
# re_good = re.compile(r"(?P<people>.+) - (?P<title>.+) --? (?P<studio>\S+)")
|
||||
# re_good = re.compile(r"(?P<people>.+) - (?P<title>.+) --? (?P<date_year>\d{4})(?P<date_month>\d{2})(?P<date_day>\d{2})")
|
||||
re_good = re.compile(r"(?P<people>.+) - (?P<title>[^-]+) (?P<date_year>\d{4})(?P<date_month>\d{2})(?P<date_day>\d{2})")
|
||||
|
||||
for f in sorted(video_files):
|
||||
m = re_good.search(f.stem)
|
||||
if not m:
|
||||
continue
|
||||
params = m.groupdict()
|
||||
title = params["title"]
|
||||
if " " not in title:
|
||||
print(f.stem, title, sep="\t\t")
|
||||
continue
|
||||
# better_name = f"{m['people']} -- @{m['studio']} -- {m['title']} -- {m['date_year']}-{m['date_month']}-{m['date_day']}"
|
||||
better_name = f"{m['people']} -- {m['title']} -- {m['date_year']}-{m['date_month']}-{m['date_day']}"
|
||||
# better_name = f"{m['people']} -- @{m['studio']} -- {m['title']}"
|
||||
print(f.stem, better_name, sep="\n")
|
||||
better_path = f.with_stem(better_name)
|
||||
f.rename(better_path)
|
||||
|
||||
|
||||
def clean_images():
|
||||
from PIL import Image
|
||||
|
||||
for img in Path(r"/Users/abdus/Downloads/temp/Ploy Tigerstam -- OnlyFans Leak").glob("*.jpg"):
|
||||
image = Image.open(img)
|
||||
width, height = image.size
|
||||
|
||||
if width < 2000 or height < 2000:
|
||||
img.unlink()
|
||||
|
||||
|
||||
def main():
|
||||
dry = True
|
||||
try:
|
||||
# clean_images()
|
||||
# collect_all()
|
||||
# save_suggested_names()
|
||||
# delete_remote_files()
|
||||
# transfer()
|
||||
# different_format(dry_run=dry)
|
||||
# local_duped(dry_run=dry)
|
||||
# reencoded_self(dry_run=dry)
|
||||
# dupes()
|
||||
# scan_videos()
|
||||
# list_studios()
|
||||
# lower_resolutions(dry_run=dry)
|
||||
# large_unwanted()
|
||||
unwanted(dry_run=dry)
|
||||
finally:
|
||||
pass
|
||||
# clean_records()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,87 @@
|
||||
import csv
|
||||
import html
|
||||
import sys
|
||||
|
||||
page_css = """
|
||||
body {
|
||||
font-family: consolas, monospace;
|
||||
}
|
||||
table {
|
||||
border-collapse: collapse;
|
||||
width: 100%;
|
||||
}
|
||||
th, td {
|
||||
border: 1px solid black;
|
||||
padding: 8px;
|
||||
text-align: left;
|
||||
}
|
||||
th {
|
||||
background-color: #f2f2f2;
|
||||
}
|
||||
tr:nth-child(even) {
|
||||
background-color: #fdfdfd;
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def csv_to_html(csv_input: str) -> str:
|
||||
"""
|
||||
Converts TSV input from stdin to an HTML table.
|
||||
|
||||
Args:
|
||||
csv_input: The TSV input as a string.
|
||||
|
||||
Returns:
|
||||
A string containing the HTML table, or None if an error occurs.
|
||||
"""
|
||||
try:
|
||||
delimiter = "\t" if "\t" in csv_input else ","
|
||||
reader = csv.reader(csv_input.splitlines(), delimiter=delimiter)
|
||||
header = next(reader) # Get the header row
|
||||
|
||||
header_html = f'{"".join(f"<th>{html.escape(col)}</th>" for col in header)}'
|
||||
rows = [f"<tr>{''.join(f'<td>{html.escape(cell)}</td>' for cell in row)}</tr>" for row in reader]
|
||||
rows_html = "\n".join(rows)
|
||||
|
||||
rendered = f"""
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Query Results</title>
|
||||
<style>{page_css}</style>
|
||||
</head>
|
||||
<body>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>{header_html}</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows_html}
|
||||
</tbody>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
return rendered
|
||||
|
||||
except csv.Error as e:
|
||||
raise ValueError(f"Error parsing TSV: {e}")
|
||||
except StopIteration:
|
||||
return "<p>No data found in TSV input.</p>"
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
tsv_data = sys.stdin.read()
|
||||
html_output = csv_to_html(tsv_data)
|
||||
if html_output:
|
||||
print(html_output)
|
||||
except UnicodeDecodeError as e:
|
||||
raise ValueError(f"Error decoding input: {e}. Ensure your TSV data is encoded correctly (e.g., UTF-8).")
|
||||
except Exception as e:
|
||||
raise Exception(f"An unexpected error occurred: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,167 +0,0 @@
|
||||
#!/usr/bin/env python3.9
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import typing
|
||||
from playwright.sync_api import Playwright, sync_playwright, Browser
|
||||
from typer import Option, Typer
|
||||
|
||||
|
||||
class Storage(typing.Protocol):
|
||||
def get(self, key: str) -> typing.Optional[typing.Any]:
|
||||
...
|
||||
|
||||
def set(self, key: str, value) -> None:
|
||||
...
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileStorage:
|
||||
path: Path
|
||||
|
||||
def get(self, key: str):
|
||||
try:
|
||||
return json.loads(self.path.read_text()).get(key)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
def set(self, key: str, value) -> None:
|
||||
try:
|
||||
data = json.loads(self.path.read_text())
|
||||
except FileNotFoundError:
|
||||
data = {}
|
||||
data[key] = value
|
||||
self.path.write_text(json.dumps(data))
|
||||
|
||||
|
||||
class Emp:
|
||||
def __init__(self, browser: Browser, storage: Storage) -> None:
|
||||
self.browser = browser
|
||||
self.storage = storage
|
||||
|
||||
def ensure_session(self, username: str, password: str) -> None:
|
||||
if self.storage.get("cookies") is None:
|
||||
self.login(username, password)
|
||||
|
||||
def login(self, username: str, password: str) -> typing.Dict[str, str]:
|
||||
"""
|
||||
logins and returns session cookies
|
||||
"""
|
||||
context = self.browser.new_context()
|
||||
|
||||
# Open new page
|
||||
page = context.new_page()
|
||||
|
||||
# Go to https://www.empornium.is/
|
||||
page.goto("https://www.empornium.is/login")
|
||||
|
||||
page.locator('[placeholder="Username"]').fill(username)
|
||||
page.locator('[placeholder="Password"]').fill(password)
|
||||
# Click text=Stay logged in
|
||||
page.locator("text=Stay logged in").click()
|
||||
|
||||
# Click input:has-text("login")
|
||||
page.locator('input:has-text("login")').click()
|
||||
|
||||
cookies = {c["name"]: c["value"] for c in context.cookies()}
|
||||
self.storage.set("cookies", {"sid": cookies["sid"]})
|
||||
|
||||
return {
|
||||
"sid": cookies["sid"],
|
||||
}
|
||||
|
||||
def prepare_post(
|
||||
self,
|
||||
torrent_path: Path,
|
||||
title: str,
|
||||
tags: str,
|
||||
description: str,
|
||||
cover_image_url: str,
|
||||
category: typing.Optional[str] = None,
|
||||
) -> None:
|
||||
cookies = self.storage.get("cookies")
|
||||
assert cookies, "You must login first"
|
||||
|
||||
context = self.browser.new_context()
|
||||
context.add_cookies(
|
||||
[{"name": k, "value": v, "domain": "www.empornium.is", "path": "/"} for k, v in cookies.items()]
|
||||
)
|
||||
|
||||
page = context.new_page()
|
||||
|
||||
page.goto("https://www.empornium.is/upload.php")
|
||||
|
||||
if torrent_path.is_file():
|
||||
page.locator('input[name="file_input"]').set_input_files(torrent_path.expanduser().resolve())
|
||||
page.locator('text="check for dupes"').click()
|
||||
|
||||
# Select category
|
||||
if category:
|
||||
page.locator('select[name="category"]').select_option(label=category)
|
||||
|
||||
page.locator('input[name="title"]').fill(title)
|
||||
page.locator('textarea[name="taglist"]').fill(tags)
|
||||
page.locator('input[name="image"]').fill(cover_image_url)
|
||||
page.locator('textarea[name="desc"]').fill(description)
|
||||
|
||||
# Click text=Preview
|
||||
page.locator("text=Preview").click()
|
||||
|
||||
|
||||
def submit_post():
|
||||
storage = FileStorage(Path("emp.json"))
|
||||
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(headless=False)
|
||||
emp = Emp(browser=browser, storage=storage)
|
||||
emp.ensure_session(username="zzzp", password="9arjs9za2o")
|
||||
|
||||
emp.prepare_post(
|
||||
torrent_path=Path("~/Downloads/v.torrent"),
|
||||
title="A Title",
|
||||
tags="tag.1 tag.2",
|
||||
description="Some description",
|
||||
cover_image_url="https://images.com/image.jpg",
|
||||
category="Anal",
|
||||
)
|
||||
input("Press Enter to continue...")
|
||||
browser.close()
|
||||
|
||||
|
||||
cli = Typer(name="emp")
|
||||
|
||||
|
||||
torrent_cli = Typer(name="torrent")
|
||||
cli.add_typer(torrent_cli)
|
||||
|
||||
|
||||
@torrent_cli.callback("torrent")
|
||||
def make_torrent(paths: typing.List[Path], announce_url: str = Option(..., envvar="ANNOUNCE_URL")):
|
||||
if len(paths) == 1 and paths[0].is_dir():
|
||||
dir_path = paths[0]
|
||||
|
||||
dir_path = dir_path.expanduser().resolve()
|
||||
proc = subprocess.run(
|
||||
[
|
||||
"torrentify",
|
||||
f"-announce={announce_url}",
|
||||
f"-comment=created by zzzp",
|
||||
f"-created-by=zzzp",
|
||||
f"-name={dir_path.name}",
|
||||
str(dir_path),
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
torrent_path = dir_path / f"{dir_path.name}.torrent"
|
||||
torrent_path.write_bytes(proc.stdout)
|
||||
|
||||
|
||||
@torrent_cli.command()
|
||||
def clone():
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
@@ -175,7 +175,7 @@ def upload_file(
|
||||
size = file_path.stat().st_size
|
||||
while True:
|
||||
proc = subprocess.Popen(
|
||||
["rsync", "--bwlimit", "1100", "-rvPa", str(file_path), destination],
|
||||
["rsync", "--bwlimit", "4500", "-rvPa", str(file_path), destination],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
@@ -333,14 +333,14 @@ def parse_args():
|
||||
arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
arger.add_argument("video_path", type=Path, help="path to video file")
|
||||
arger.add_argument("-e", "--encoder", choices=["handbrake", "ffmpeg"], default="handbrake", help="encoder engine")
|
||||
arger.add_argument("-q", "--quality", type=float, default=25, help="x265 quality factor")
|
||||
arger.add_argument("-q", "--quality", type=float, default=35, help="x265 quality factor")
|
||||
arger.add_argument("--output-dir", "-o", dest="output_dir", type=Path, help="Dir to save encoded files")
|
||||
arger.add_argument("--rsync", dest="upload_target", help="rsync encoded file to a host")
|
||||
arger.add_argument("--validate", action="store_true", default=False, help="perform validations before starting")
|
||||
arger.add_argument("--denoise", action="store_true", default=False, help="Enable denoise filter (Handbrake only)")
|
||||
arger.add_argument("--10bit", action="store_true", dest="is_10bit", help="Encode using 10-bit profile")
|
||||
arger.add_argument("--8bit", action="store_false", dest="is_10bit", help="Encode using 8-bit profile")
|
||||
arger.set_defaults(is_10bit=True)
|
||||
arger.add_argument("--8bit", action="store_true", dest="is_10bit", help="Encode using 8-bit profile")
|
||||
arger.set_defaults(is_8bit=True)
|
||||
|
||||
def parse_time(val: str) -> int:
|
||||
parts = val.split(":")
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
import argparse
|
||||
import logging
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
KNOWN_PASSWORDS = """
|
||||
torgo
|
||||
Yuzuki
|
||||
oron.com
|
||||
MonikGonPs
|
||||
koth
|
||||
drdoom-psuzy
|
||||
psuzy
|
||||
suzy
|
||||
idle
|
||||
dan4260
|
||||
""".strip().splitlines(keepends=False)
|
||||
|
||||
|
||||
def find_archive_files(root_dir: Path) -> list[Path]:
|
||||
extractables = []
|
||||
globs = ["*.rar", "*.zip", "*.7z", "*.zip.*", "*.7z.*"]
|
||||
for glob in globs:
|
||||
for f in root_dir.rglob(glob):
|
||||
extractables.append(f)
|
||||
|
||||
extractables = [f for f in extractables if is_extractable(f)]
|
||||
|
||||
return sorted(extractables, key=str)
|
||||
|
||||
|
||||
def is_extractable(archive: Path) -> bool:
|
||||
if re.search(f"\.zip|\.7z", archive.name):
|
||||
zip_part = re.search("(\d+)$", archive.suffix)
|
||||
if zip_part and int(zip_part.group(1)) == 1:
|
||||
return True
|
||||
if not zip_part:
|
||||
return True
|
||||
|
||||
if archive.name.endswith(".rar"):
|
||||
rar_part = re.search(r"\.part(\d+)", archive.name)
|
||||
if not rar_part:
|
||||
return True
|
||||
if rar_part and int(rar_part.group(1)) == 1:
|
||||
if archive.with_name(archive.name.replace("part1", "part2")).is_file():
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def extract_archive_rar(archive: Path, output_dir: Path, password: str | None = None) -> None:
|
||||
args = [
|
||||
"rar",
|
||||
"e",
|
||||
f"-p{password}" if password else "-p-",
|
||||
f"{archive.name}",
|
||||
f"{output_dir}",
|
||||
]
|
||||
try:
|
||||
subprocess.run(args, cwd=archive.parent, capture_output=True, text=True, check=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
output: str = e.stderr or e.stdout
|
||||
if "corrupt file or wrong password" in output.lower():
|
||||
raise ValueError("corrupt or invalid password")
|
||||
elif e.returncode == 3:
|
||||
raise FileNotFoundError("missing parts")
|
||||
elif e.returncode == 10:
|
||||
raise ValueError("not a rar file")
|
||||
elif e.returncode == 11:
|
||||
raise ValueError("invalid password")
|
||||
|
||||
raise
|
||||
|
||||
|
||||
def extract_archive_7z(archive: Path, output_dir: Path, password: str | None = None):
|
||||
args = [
|
||||
"7z",
|
||||
"e",
|
||||
f'-p{password or ""}',
|
||||
f"-o{output_dir}",
|
||||
"-y",
|
||||
f"{archive}",
|
||||
]
|
||||
|
||||
try:
|
||||
subprocess.run(args, capture_output=True, text=True, check=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
output: str = e.stderr or e.stdout
|
||||
if "missing volume" in output.lower():
|
||||
raise FileNotFoundError("missing parts")
|
||||
if e.returncode == 2:
|
||||
raise ValueError("invalid password")
|
||||
|
||||
raise
|
||||
|
||||
|
||||
def extract_archive(archive_path: Path, output_dir: Path, passwords: list[str]) -> None:
|
||||
for p in ["", *passwords]:
|
||||
try:
|
||||
if ".rar" in archive_path.name:
|
||||
extract_archive_rar(archive_path, output_dir, p)
|
||||
else:
|
||||
extract_archive_7z(archive_path, output_dir, p)
|
||||
except Exception as e:
|
||||
logger.error(f"unhandled error: {e}")
|
||||
raise e
|
||||
|
||||
|
||||
def delete_archive(archive: Path):
|
||||
stem = archive.stem.rsplit(".", maxsplit=1)[0]
|
||||
for f in archive.parent.rglob(f"*{archive.suffix}"):
|
||||
if archive.stem.startswith(stem):
|
||||
logger.info(f"deleting: {f.name}")
|
||||
f.unlink()
|
||||
|
||||
|
||||
def parse_args():
|
||||
arger = argparse.ArgumentParser()
|
||||
arger.add_argument("archive_paths", type=Path, nargs="+", help="List of archive paths")
|
||||
arger.add_argument("--cwd", type=Path, default=Path.cwd(), help="working directory")
|
||||
arger.add_argument("--clean", type=bool, default=False, help="delete file(s) after successful extraction", action="store_true")
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
arger.print_help()
|
||||
exit(1)
|
||||
|
||||
return arger.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
cwd: Path = args.cwd
|
||||
|
||||
archives = find_archives(cwd)
|
||||
logger.info(f"found {len(archives)} archives")
|
||||
for f in archives:
|
||||
logger.info(f"\t{f.relative_to(cwd)}")
|
||||
|
||||
typer.confirm("continue", default=True, abort=True)
|
||||
|
||||
failed = []
|
||||
for i, f in enumerate(archives, start=1):
|
||||
progress = f"{i:02}/{len(archives):02}"
|
||||
logger.info(f"\nextracting {progress}: {f.relative_to(cwd)}")
|
||||
try:
|
||||
success = extract_archive(f, passwords=passwords)
|
||||
if success:
|
||||
if clean:
|
||||
delete_archive(f)
|
||||
logger.info(f"done")
|
||||
else:
|
||||
logger.error("failed")
|
||||
except (ValueError, FileNotFoundError) as e:
|
||||
logger.error(str(e))
|
||||
failed.append(f)
|
||||
if failed:
|
||||
logger.info(f"\n\nfailed to extract {len(failed)} archives")
|
||||
for f in failed:
|
||||
logger.info(f"{f.relative_to(cwd)}")
|
||||
logger.info("\n\nfinished.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
typer.run(main)
|
||||
@@ -4,12 +4,10 @@ import dataclasses
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
import typing
|
||||
from datetime import timedelta
|
||||
from math import ceil
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -26,11 +24,28 @@ class FfprobeResult:
|
||||
container: str
|
||||
ar: float
|
||||
sample_ar: float
|
||||
tags: typing.Dict[str, str]
|
||||
tags: dict[str, str]
|
||||
|
||||
@property
|
||||
def resolution(self) -> str:
|
||||
return f'{self.width}x{self.height}'
|
||||
|
||||
@property
|
||||
def has_metadata(self):
|
||||
return any(k in self.tags for k in ['title', 'comment'])
|
||||
|
||||
@property
|
||||
def hd_mode(self) -> str:
|
||||
if 700 < self.height < 800:
|
||||
return '720p'
|
||||
if 900 < self.height < 1200:
|
||||
return '1080p'
|
||||
if 2000 < self.height < 2200:
|
||||
return '4K'
|
||||
raise ValueError('unknown HD mode')
|
||||
|
||||
|
||||
def ffprobe(video: Path) -> FfprobeResult:
|
||||
logger.debug("executing ffprobe")
|
||||
def ffprobe(video_path: Path) -> FfprobeResult:
|
||||
proc = subprocess.run(
|
||||
args=[
|
||||
"ffprobe",
|
||||
@@ -40,7 +55,7 @@ def ffprobe(video: Path) -> FfprobeResult:
|
||||
"-show_format",
|
||||
"-print_format",
|
||||
"json",
|
||||
str(video),
|
||||
str(video_path),
|
||||
],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
@@ -48,8 +63,8 @@ def ffprobe(video: Path) -> FfprobeResult:
|
||||
)
|
||||
try:
|
||||
proc.check_returncode()
|
||||
except subprocess.CalledProcessError:
|
||||
logger.error("failed to run ffprobe", exc_info=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.error(f"failed to run ffprobe. stderr={e.stderr}", exc_info=True)
|
||||
raise
|
||||
output: dict = json.loads(proc.stdout)
|
||||
video_stream: dict = [s for s in output.get("streams", []) if s.get("codec_type") == "video"][0]
|
||||
@@ -71,9 +86,6 @@ def ffprobe(video: Path) -> FfprobeResult:
|
||||
except:
|
||||
sample_ar = 1
|
||||
|
||||
logger.debug(f"dimensions={width}x{height} duration={duration_time}")
|
||||
# total_frames = video_stream.get("nb_frames")
|
||||
|
||||
return FfprobeResult(
|
||||
duration_sec=duration,
|
||||
duration_human=str(duration_time),
|
||||
@@ -83,19 +95,18 @@ def ffprobe(video: Path) -> FfprobeResult:
|
||||
width=width,
|
||||
bitrate=bitrate,
|
||||
height=height,
|
||||
container=video.suffix.lstrip(".").lower(),
|
||||
container=video_path.suffix.lstrip(".").lower(),
|
||||
ar=width / height,
|
||||
sample_ar=sample_ar,
|
||||
tags=tags,
|
||||
# total_frames=total_frames,
|
||||
)
|
||||
|
||||
|
||||
def strip_metadata(video_path: Path, save_path: Path) -> Path:
|
||||
media_info = ffprobe(video_path)
|
||||
# check if the first video stream has hevc codec
|
||||
if media_info["streams"][0]["codec_name"] == "hevc":
|
||||
logger.info(f"Video codec is hevc, adding hvc1 tag")
|
||||
if media_info.codec == "hevc":
|
||||
logging.info(f"Video codec is hevc, adding hvc1 tag")
|
||||
extra_args = ["-tag:v", "hvc1"]
|
||||
else:
|
||||
extra_args = []
|
||||
@@ -106,7 +117,7 @@ def strip_metadata(video_path: Path, save_path: Path) -> Path:
|
||||
'-i',
|
||||
str(video_path),
|
||||
'-c:v', 'copy',
|
||||
'-movflags', '+faststart',
|
||||
# '-movflags', '+faststart',
|
||||
'-map_metadata', '-1',
|
||||
*extra_args,
|
||||
'-c:a', 'copy',
|
||||
@@ -114,32 +125,22 @@ def strip_metadata(video_path: Path, save_path: Path) -> Path:
|
||||
str(save_path),
|
||||
]
|
||||
# fmt: on
|
||||
logger.info(f"calling ffmpeg with {args=}")
|
||||
logging.info(f"calling ffmpeg with {args=}")
|
||||
subprocess.run(args, check=True)
|
||||
return save_path
|
||||
|
||||
|
||||
def make_thumbnail_tile(
|
||||
video: Path,
|
||||
image_path: Path = None,
|
||||
columns: int = 3,
|
||||
interval_seconds: int = 60,
|
||||
tile_width: int = 540,
|
||||
skip_first_sec: int = 10,
|
||||
skip_if_exists: bool = False,
|
||||
) -> Path:
|
||||
if not image_path:
|
||||
image_path = video.parent / f"{video.stem}.thumbnail.jpg"
|
||||
if image_path.is_file() and skip_if_exists:
|
||||
logger.debug("thumbnail already exists")
|
||||
return image_path
|
||||
|
||||
) -> bytes:
|
||||
info = ffprobe(video)
|
||||
|
||||
min_frames = columns * 3
|
||||
|
||||
sec_per_frame = min(interval_seconds, ceil(info.duration_sec / min_frames))
|
||||
|
||||
rows = ceil(info.duration_sec // sec_per_frame / columns)
|
||||
tile = f"{columns}x{rows}"
|
||||
|
||||
@@ -150,25 +151,32 @@ def make_thumbnail_tile(
|
||||
# timestamp_filter = rf"drawtext=r=1:timecode='00\:00\:00\:00':fontsize=16:fontcolor=white:x=10:y=10:box=1:boxcolor=black@0.5"
|
||||
# timestamp_filter = fr"drawtext=text='%{{(pts\\+{skip_first_sec})\:hms}}':fontsize=16:fontcolor=white:x=10:y=10:box=1:boxcolor=black@0.5"
|
||||
|
||||
proc = subprocess.run(
|
||||
args = [
|
||||
# fmt: off
|
||||
args=[
|
||||
"ffmpeg",
|
||||
"-v", "error",
|
||||
"-skip_frame", "nokey",
|
||||
"-ss", f"{skip_first_sec}",
|
||||
"-i", str(video.absolute()),
|
||||
# "-vf", f"fps=1/{sec_per_frame},scale={scale},{timestamp_filter},tile={tile}",
|
||||
"-vf", f"fps=1/{sec_per_frame},scale={scale},tile={tile}",
|
||||
"-frames", "1",
|
||||
"-y", str(image_path.absolute()),
|
||||
],
|
||||
"ffmpeg",
|
||||
"-v", "error",
|
||||
"-skip_frame", "nokey",
|
||||
"-ss", f"{skip_first_sec}",
|
||||
"-i", str(video.absolute()),
|
||||
# "-vf", f"fps=1/{sec_per_frame},scale={scale},{timestamp_filter},tile={tile}",
|
||||
"-vf", f"fps=1/{sec_per_frame},scale={scale},tile={tile}",
|
||||
"-frames", "1",
|
||||
"-c:v",
|
||||
"jpeg2000",
|
||||
"-q:v", "90",
|
||||
"-f", "image2pipe",
|
||||
"-",
|
||||
# fmt: on
|
||||
stdout=subprocess.DEVNULL,
|
||||
]
|
||||
logger.debug(f'calling ffmpeg with args={args}')
|
||||
proc = subprocess.run(
|
||||
args=args,
|
||||
stdout=subprocess.PIPE,
|
||||
timeout=2000,
|
||||
# preexec_fn=os.setpgrp,
|
||||
)
|
||||
proc.check_returncode()
|
||||
return image_path
|
||||
return bytes(proc.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
@@ -177,9 +185,11 @@ if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
for it in sys.argv[1:]:
|
||||
video = Path(it)
|
||||
image_path = video.parent / f"{video.stem}.thumbnail.jpg"
|
||||
|
||||
if not video.exists():
|
||||
continue
|
||||
|
||||
logger.info("creating thumbnail for %s", video)
|
||||
make_thumbnail_tile(video)
|
||||
image_jpeg = make_thumbnail_tile(video)
|
||||
image_path.write_bytes(image_jpeg)
|
||||
|
||||
+5
-5
@@ -58,7 +58,7 @@ def optimize_gif(gif_path: Path) -> Path:
|
||||
subprocess.run(
|
||||
[
|
||||
"gifsicle",
|
||||
"--lossy=98",
|
||||
"--lossy=200",
|
||||
"--optimize=3",
|
||||
"--batch",
|
||||
"-i",
|
||||
@@ -75,9 +75,9 @@ def parse_args(argv: typing.List[str] = None) -> argparse.Namespace:
|
||||
arger.add_argument(
|
||||
"--start", "-s", dest="from_time", type=str, help="Time to start from, e.g. 02:59", required=True
|
||||
)
|
||||
arger.add_argument("--duration", "-d", default=3, type=int, help="Duration of gif in seconds")
|
||||
arger.add_argument("--fps", default=15, type=int, help="Frames per second")
|
||||
arger.add_argument("--width", default=400, type=int, help="Image width")
|
||||
arger.add_argument("--duration", "-d", default=3, type=float, help="Duration of gif in seconds")
|
||||
arger.add_argument("--fps", default=20, type=int, help="Frames per second")
|
||||
arger.add_argument("--width", default=376, type=int, help="Image width")
|
||||
arger.add_argument("--optimize", default=True, action="store_true", help="Reduce GIF filesize")
|
||||
|
||||
if len(argv) == 0:
|
||||
@@ -105,7 +105,7 @@ def main():
|
||||
|
||||
save_path = None
|
||||
if str(args.video_path).startswith("http"):
|
||||
save_path = Path("~/Downloads").expanduser() / f"{md5(str(video_path))}__{from_time.replace(':', '')}.gif"
|
||||
save_path = Path("~/Pictures/Screenshots/").expanduser() / f"{md5(str(video_path))}__{from_time.replace(':', '')}.gif"
|
||||
|
||||
gif_path = create_gif(
|
||||
video_path=video_path,
|
||||
|
||||
+2
-2
@@ -41,7 +41,7 @@ def clean_video(video_path: Path, save_path: Path) -> Path:
|
||||
media_info = ffprobe(video_path)
|
||||
# check if the first video stream has hevc codec
|
||||
if media_info["streams"][0]["codec_name"] == "hevc":
|
||||
logging.info(f"Video codec is hevc, adding hvc1 tag")
|
||||
logging.debug(f"Video codec is hevc, adding hvc1 tag")
|
||||
extra_args = ["-tag:v", "hvc1"]
|
||||
else:
|
||||
extra_args = []
|
||||
@@ -52,7 +52,7 @@ def clean_video(video_path: Path, save_path: Path) -> Path:
|
||||
'-i',
|
||||
str(video_path),
|
||||
'-c:v', 'copy',
|
||||
# '-movflags', '+faststart',
|
||||
# '-movflags', '+faststart',
|
||||
'-map_metadata', '-1',
|
||||
*extra_args,
|
||||
'-c:a', 'copy',
|
||||
|
||||
+87
-58
@@ -10,24 +10,25 @@ import typing
|
||||
from pathlib import Path
|
||||
|
||||
script_path = Path(__file__)
|
||||
db_filename = script_path.with_name(f'{script_path.stem}.known.txt')
|
||||
db_filename = script_path.with_name(f"{script_path.stem}.known.txt")
|
||||
|
||||
|
||||
def save_actors(actors: typing.Iterable[str]):
|
||||
existing = load_actors()
|
||||
all_actors = sorted({*existing, *actors})
|
||||
with db_filename.open('w', encoding='utf-8', newline='\n') as f:
|
||||
with db_filename.open("w", encoding="utf-8", newline="\n") as f:
|
||||
f.writelines(all_actors)
|
||||
|
||||
|
||||
def load_actors() -> set[str]:
|
||||
try:
|
||||
with db_filename.open('w', encoding='utf-8', newline='\n') as f:
|
||||
with db_filename.open(encoding="utf-8", newline="\n") as f:
|
||||
return {line.strip() for line in f}
|
||||
except FileNotFoundError:
|
||||
return set()
|
||||
|
||||
|
||||
re_date_only = re.compile(r"\b(\d{4})\b") # 2024
|
||||
re_date_us = re.compile(r"\b(\d{2})\D(\d{2})\D(\d{2})\b") # 12/31/21
|
||||
re_date_iso = re.compile(r"\b(\d{4})\D?(\d{2})\D?(\d{2})\b") # 2020-12-31
|
||||
re_date_iso_short = re.compile(r"(\d{2})\D?(\d{2})\D?(\d{2})") # 20-12-31"'
|
||||
@@ -53,10 +54,19 @@ class ParsedDate:
|
||||
|
||||
|
||||
def parse_date(filename: str) -> ParsedDate | None:
|
||||
def from_date_only():
|
||||
match = re_date_only.search(filename)
|
||||
y = int(match.group(1))
|
||||
return ParsedDate(
|
||||
datetime.date(y, 1, 1), start=match.start(), end=match.end()
|
||||
)
|
||||
|
||||
def from_us_format():
|
||||
match = re_date_us.search(filename)
|
||||
m, d, y = map(int, match.groups())
|
||||
return ParsedDate(datetime.date(y + 2000, m, d), start=match.start(), end=match.end())
|
||||
return ParsedDate(
|
||||
datetime.date(y + 2000, m, d), start=match.start(), end=match.end()
|
||||
)
|
||||
|
||||
def from_iso():
|
||||
match = re_date_iso.search(filename)
|
||||
@@ -66,7 +76,9 @@ def parse_date(filename: str) -> ParsedDate | None:
|
||||
def from_iso_short():
|
||||
match = re_date_iso_short.search(filename)
|
||||
y, m, d = map(int, match.groups())
|
||||
return ParsedDate(datetime.date(y + 2000, m, d), start=match.start(), end=match.end())
|
||||
return ParsedDate(
|
||||
datetime.date(y + 2000, m, d), start=match.start(), end=match.end()
|
||||
)
|
||||
|
||||
def from_iso_reversed():
|
||||
match = re_date_iso_rev.search(filename)
|
||||
@@ -76,10 +88,13 @@ def parse_date(filename: str) -> ParsedDate | None:
|
||||
def from_iso_reversed_short():
|
||||
match = re_date_iso_rev_short.search(filename)
|
||||
d, m, y = map(int, match.groups())
|
||||
return ParsedDate(datetime.date(y + 2000, m, d), start=match.start(), end=match.end())
|
||||
return ParsedDate(
|
||||
datetime.date(y + 2000, m, d), start=match.start(), end=match.end()
|
||||
)
|
||||
|
||||
candidates = []
|
||||
for fn in [
|
||||
from_date_only,
|
||||
from_us_format,
|
||||
from_iso,
|
||||
from_iso_short,
|
||||
@@ -94,7 +109,9 @@ def parse_date(filename: str) -> ParsedDate | None:
|
||||
today = datetime.date.today()
|
||||
future_threshold = today + datetime.timedelta(days=60)
|
||||
past_threshold = datetime.date(2010, 1, 1)
|
||||
candidates = [it for it in candidates if past_threshold < it.date < future_threshold]
|
||||
candidates = [
|
||||
it for it in candidates if past_threshold < it.date < future_threshold
|
||||
]
|
||||
|
||||
if not candidates:
|
||||
return
|
||||
@@ -112,31 +129,35 @@ class Release:
|
||||
def to_filename(self):
|
||||
parts = []
|
||||
if self.actors:
|
||||
parts.append(', '.join(self.actors))
|
||||
parts.append(", ".join(self.actors))
|
||||
if self.studio:
|
||||
parts.append(f'@{self.studio}')
|
||||
parts.append(f"@{self.studio}")
|
||||
if self.title:
|
||||
parts.append(self.title)
|
||||
if self.released_at:
|
||||
parts.append(self.released_at.isoformat())
|
||||
return ' -- '.join(parts)
|
||||
return " -- ".join(parts)
|
||||
|
||||
|
||||
def trash(path: Path):
|
||||
filename = f'"{path}"'
|
||||
cmd = ['osascript', '-e', f'tell app "Finder" to move (POSIX file {filename}) to trash']
|
||||
cmd = [
|
||||
"osascript",
|
||||
"-e",
|
||||
f'tell app "Finder" to move (POSIX file {filename}) to trash',
|
||||
]
|
||||
subprocess.run(cmd).check_returncode()
|
||||
|
||||
|
||||
def filenames_to_actors():
|
||||
sources = [
|
||||
Path(r'/Users/abdus/Downloads/temp/'),
|
||||
Path(r'/Volumes/BANDAID/_temp/__reenc/'),
|
||||
Path(r'/Volumes/BANDAID/_temp/'),
|
||||
Path(r"/Users/abdus/Downloads/temp/"),
|
||||
Path(r"/Volumes/BANDAID/_temp/__reenc/"),
|
||||
Path(r"/Volumes/BANDAID/_temp/"),
|
||||
]
|
||||
actors = set()
|
||||
for it in sources:
|
||||
for f in it.glob('*.mp4'):
|
||||
for f in it.glob("*.mp4"):
|
||||
if not f.is_file():
|
||||
continue
|
||||
if r := parse_release(f.name):
|
||||
@@ -144,78 +165,83 @@ def filenames_to_actors():
|
||||
return actors
|
||||
|
||||
|
||||
def parse_release(filename: str, known_actors: set[str] | None = None):
|
||||
def parse_release(
|
||||
filename: str, known_actors: set[str] | None = None
|
||||
) -> Release | None:
|
||||
if not known_actors:
|
||||
known_actors = set()
|
||||
filename = re.sub(r'\.\w{3,4}$', '', filename)
|
||||
if any(filename.lower().endswith(ext) for ext in [".mp4", ".mkv"]):
|
||||
filename = re.sub(r"\.\w{3,4}$", "", filename)
|
||||
|
||||
def from_own():
|
||||
remaining = re.sub(r'\s+\[[^]]+]$', '', filename)
|
||||
remaining = re.sub(r'\s+\[([^]]+|\d+\w)(,\s*[^]]+)?]', '', remaining)
|
||||
remaining = re.sub(r'\s*--\s*', ' -- ', remaining)
|
||||
remaining = re.sub(r"\s+\[[^]]+]$", "", filename)
|
||||
remaining = re.sub(r"\s+\[([^]]+|\d+\w)(,\s*[^]]+)?]", "", remaining)
|
||||
remaining = re.sub(r"\s*--\s*", " -- ", remaining)
|
||||
|
||||
match remaining.split(' -- '):
|
||||
case [actors, studio, title, date] if studio.startswith('@'):
|
||||
match remaining.split(" -- "):
|
||||
case [actors, studio, title, date] if studio.startswith("@"):
|
||||
return Release(
|
||||
actors=sorted(actors.split(', ')),
|
||||
studio=studio.removeprefix('@'),
|
||||
actors=sorted(actors.split(", ")),
|
||||
studio=studio.removeprefix("@"),
|
||||
title=title,
|
||||
released_at=datetime.date.fromisoformat(date),
|
||||
)
|
||||
case [actors, studio, date] if studio.startswith('@') and (parsed := parse_date(date)):
|
||||
case [actors, studio, date] if studio.startswith("@") and (
|
||||
parsed := parse_date(date)
|
||||
):
|
||||
return Release(
|
||||
actors=sorted(actors.split(', ')),
|
||||
studio=studio.removeprefix('@'),
|
||||
actors=sorted(actors.split(", ")),
|
||||
studio=studio.removeprefix("@"),
|
||||
title=None,
|
||||
released_at=parsed.date,
|
||||
)
|
||||
case [actors, studio, title] if studio.startswith('@'):
|
||||
case [actors, studio, title] if studio.startswith("@"):
|
||||
return Release(
|
||||
actors=sorted(actors.split(', ')),
|
||||
studio=studio.removeprefix('@'),
|
||||
actors=sorted(actors.split(", ")),
|
||||
studio=studio.removeprefix("@"),
|
||||
title=title,
|
||||
released_at=None,
|
||||
)
|
||||
case [actors, title, date] if (parsed := parse_date(date)):
|
||||
return Release(
|
||||
actors=sorted(actors.split(', ')),
|
||||
actors=sorted(actors.split(", ")),
|
||||
studio=None,
|
||||
title=title,
|
||||
released_at=parsed.date,
|
||||
)
|
||||
case [actors, studio] if studio.startswith('@'):
|
||||
case [actors, studio] if studio.startswith("@"):
|
||||
return Release(
|
||||
actors=sorted(actors.split(', ')),
|
||||
actors=sorted(actors.split(", ")),
|
||||
studio=studio,
|
||||
)
|
||||
case [actors, title]:
|
||||
return Release(
|
||||
actors=sorted(actors.split(', ')),
|
||||
actors=sorted(actors.split(", ")),
|
||||
title=title,
|
||||
)
|
||||
|
||||
def from_prt():
|
||||
assert '.PRT' in filename
|
||||
remaining = re.sub(r'\.(720p|1080p|HEVC|x265|PRT|XXX)', ' ', filename)
|
||||
studio = remaining[:remaining.index('.')]
|
||||
# assert '.PRT' in filename
|
||||
remaining = re.sub(r"\.(720p|1080p|HEVC|x265|PRT|XXX)", " ", filename)
|
||||
studio = remaining[: remaining.index(".")]
|
||||
parsed_date = parse_date(remaining)
|
||||
remaining = remaining[parsed_date.end:]
|
||||
remaining = re.sub(r'[. ]+', ' ', remaining).strip()
|
||||
remaining = remaining[parsed_date.end :]
|
||||
remaining = re.sub(r"[. ]+", " ", remaining).strip()
|
||||
|
||||
actors = []
|
||||
title = None
|
||||
|
||||
match remaining.split(' '):
|
||||
case [a_first, a_last, 'And', b_first, b_last]:
|
||||
actors = [f'{a_first} {a_last}', f'{b_first} {b_last}']
|
||||
case [a_first, a_last, 'And', b_first, b_last, *rest]:
|
||||
actors = [f'{a_first} {a_last}', f'{b_first} {b_last}']
|
||||
title = ' '.join(rest)
|
||||
match remaining.split(" "):
|
||||
case [a_first, a_last, "And", b_first, b_last]:
|
||||
actors = [f"{a_first} {a_last}", f"{b_first} {b_last}"]
|
||||
case [a_first, a_last, "And", b_first, b_last, *rest]:
|
||||
actors = [f"{a_first} {a_last}", f"{b_first} {b_last}"]
|
||||
title = " ".join(rest)
|
||||
case [first, last]:
|
||||
actors = [f'{first} {last}']
|
||||
actors = [f"{first} {last}"]
|
||||
case [first, last, *rest]:
|
||||
actors = [f'{first} {last}']
|
||||
title = ' '.join(rest)
|
||||
actors = [f"{first} {last}"]
|
||||
title = " ".join(rest)
|
||||
case [first]:
|
||||
actors = [first]
|
||||
|
||||
@@ -243,7 +269,9 @@ def parse_release(filename: str, known_actors: set[str] | None = None):
|
||||
|
||||
def parse_args():
|
||||
arger = argparse.ArgumentParser()
|
||||
arger.add_argument('filenames', nargs='+', type=lambda v: Path(v).resolve(), help='Filenames')
|
||||
arger.add_argument(
|
||||
"filenames", nargs="+", type=lambda v: Path(v).resolve(), help="Filenames"
|
||||
)
|
||||
return arger.parse_args()
|
||||
|
||||
|
||||
@@ -254,17 +282,18 @@ class MoveOp:
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
'source': str(self.source),
|
||||
'target': str(self.target),
|
||||
"source": str(self.source),
|
||||
"target": str(self.target),
|
||||
}
|
||||
|
||||
|
||||
def write_undo_script(save_dir: Path, ops: list[MoveOp]):
|
||||
now = datetime.datetime.now().isoformat().replace(':', '')
|
||||
save_path = save_dir / f'undo_{now}.json'
|
||||
with save_path.open('w', encoding='utf-8') as f:
|
||||
now = datetime.datetime.now().isoformat().replace(":", "")
|
||||
save_path = save_dir / f"undo_{now}.json"
|
||||
with save_path.open("w", encoding="utf-8") as f:
|
||||
for it in ops:
|
||||
f.write(json.dumps(it.to_dict()) + '\n')
|
||||
f.write(json.dumps(it.to_dict()) + "\n")
|
||||
|
||||
|
||||
def fix_filename():
|
||||
pass
|
||||
@@ -279,18 +308,18 @@ def main():
|
||||
jobs = []
|
||||
for it in paths:
|
||||
if parsed := parse_release(it.name, known_actors=actors):
|
||||
logging.debug(f'filename={it} release={parsed}')
|
||||
logging.debug(f"filename={it} release={parsed}")
|
||||
suggested = parsed.to_filename()
|
||||
target = it.with_stem(suggested)
|
||||
if target.is_file():
|
||||
logging.error('target already exists')
|
||||
logging.error("target already exists")
|
||||
continue
|
||||
jobs.append(MoveOp(source=it, target=target))
|
||||
|
||||
if not jobs:
|
||||
return
|
||||
|
||||
write_undo_script(paths[0].parent, jobs)
|
||||
# write_undo_script(paths[0].parent, jobs)
|
||||
|
||||
for it in jobs:
|
||||
it.target.hardlink_to(it.source)
|
||||
|
||||
@@ -58,6 +58,10 @@ def test_parse_date(filename: str, date: datetime.date | None):
|
||||
|
||||
|
||||
@pytest.mark.parametrize(['filename', 'expected'], [
|
||||
[
|
||||
'PenthouseGold.23.02.05.Tiffany.Tatum.XXX.720p.HEVC.x265.PRT',
|
||||
Release(studio='DoctorAdventures', actors=['Jamie Michelle'], title='Nurse Jamie Knows Best', released_at=datetime.date(2021, 5, 26)),
|
||||
],
|
||||
[
|
||||
'Rebecca Volpetti -- @RealityKings -- Driving Him Crazy [1080p, x265] -- 2019-04-10',
|
||||
Release(actors=['Rebecca Volpetti'], studio='RealityKings', title='Driving Him Crazy', released_at=datetime.date(2019, 4, 10)),
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import argparse
|
||||
import contextlib
|
||||
import re
|
||||
|
||||
from playwright.sync_api import sync_playwright, Page
|
||||
|
||||
|
||||
def download_filejoker(url: str) -> dict:
|
||||
with launch_browser() as page:
|
||||
page.goto(url, referer='http://planetsuzy.org')
|
||||
page.get_by_role("button", name="Slow Download").click()
|
||||
page.get_by_role("button", name="Get Download Link").click()
|
||||
with page.expect_response(re.compile('https://fs.+.filejoker.net/')) as res:
|
||||
with page.expect_download(timeout=0) as dl:
|
||||
page.get_by_role("button", name="Download File").click(timeout=0)
|
||||
dl.value.cancel()
|
||||
print(dl.value.url)
|
||||
print(dl.value)
|
||||
page.pause()
|
||||
print()
|
||||
|
||||
|
||||
def download_filefox(url: str) -> dict:
|
||||
with launch_browser() as page:
|
||||
page.goto(url, referer='http://planetsuzy.org')
|
||||
page.get_by_role("button", name="Slow Download").click()
|
||||
page.get_by_role("button", name="Slow Download").click(timeout=0)
|
||||
with page.expect_response(re.compile('https://fs.+.filefox.net/')) as res:
|
||||
with page.expect_download(timeout=0) as dl:
|
||||
page.get_by_role("button", name="Download File").click(timeout=0)
|
||||
dl.value.cancel()
|
||||
print(dl.value.url)
|
||||
print(dl.value)
|
||||
page.pause()
|
||||
print()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def launch_browser() -> Page:
|
||||
with sync_playwright() as playwright:
|
||||
with playwright.chromium.launch(headless=False) as browser:
|
||||
with browser.new_context() as context:
|
||||
with context.new_page() as page:
|
||||
yield page
|
||||
|
||||
|
||||
def parse_args():
|
||||
arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
arger.add_argument('url', help='URL to download', type=str)
|
||||
return arger.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
url = args.url
|
||||
download_filejoker(url)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# download_filejoker('https://filejoker.net/ofg59z1pqjjb')
|
||||
download_filefox('https://filefox.cc/qxd2yrxgu4ak')
|
||||
main()
|
||||
-207
@@ -1,207 +0,0 @@
|
||||
import dataclasses
|
||||
import logging
|
||||
import os
|
||||
import webbrowser
|
||||
|
||||
|
||||
def force_import(module: str):
|
||||
import importlib
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
try:
|
||||
return importlib.import_module(module)
|
||||
except ModuleNotFoundError:
|
||||
subprocess.run([sys.executable, "-m", "pip", "install", module])
|
||||
importlib.invalidate_caches()
|
||||
return importlib.import_module(module)
|
||||
|
||||
|
||||
try:
|
||||
import httpx
|
||||
except ImportError:
|
||||
torf = force_import("httpx")
|
||||
|
||||
try:
|
||||
import typer
|
||||
except ImportError:
|
||||
typer = force_import("typer")
|
||||
try:
|
||||
import rich
|
||||
except ImportError:
|
||||
rich = force_import("rich")
|
||||
try:
|
||||
import inquirer
|
||||
except ImportError:
|
||||
inquirer = force_import("inquirer")
|
||||
|
||||
from rich import print
|
||||
import rich.table
|
||||
|
||||
JIRA_EMAIL = os.getenv('JIRA_EMAIL', 'abdussamet.kocak@akinon.com')
|
||||
JIRA_TOKEN = os.getenv('JIRA_TOKEN', 'iAZhlJ1AdGwckbVllhlx48C4')
|
||||
JIRA_URL = os.getenv('JIRA_URL', 'https://omniplatform.atlassian.net/')
|
||||
|
||||
if not JIRA_TOKEN:
|
||||
logging.info('')
|
||||
webbrowser.open('https://id.atlassian.com/manage-profile/security/api-tokens')
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Issue:
|
||||
id: str
|
||||
key: str
|
||||
summary: str
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class PullRequest:
|
||||
id: str
|
||||
name: str
|
||||
repository: str
|
||||
branch: str
|
||||
|
||||
|
||||
class Jira:
|
||||
def __init__(self, email: str, token: str):
|
||||
self.client = httpx.Client(
|
||||
base_url=JIRA_URL,
|
||||
auth=httpx.BasicAuth(email, token),
|
||||
)
|
||||
|
||||
def list_issues_for_release(self) -> list[Issue]:
|
||||
mergeable_status = 'In Review'
|
||||
res = self.client.post(
|
||||
'/rest/api/2/search',
|
||||
json={
|
||||
'jql': f'project = COM AND status = "{mergeable_status}" AND "Team[Dropdown]" = Backend ORDER BY created DESC'
|
||||
},
|
||||
)
|
||||
|
||||
res.raise_for_status()
|
||||
data = res.json()
|
||||
development_type = ['Story', 'Bug', 'Task']
|
||||
return [
|
||||
Issue(id=it['id'], key=it['key'], summary=it['fields']['summary'])
|
||||
for it in data['issues']
|
||||
if it['fields']['issuetype']['name'] in development_type
|
||||
]
|
||||
|
||||
def get_approved_prs(self, issue_id: str):
|
||||
res = self.client.post(
|
||||
f'/jsw/graphql',
|
||||
params={'operation': 'DevDetailsDialog'},
|
||||
json={
|
||||
"operationName": "DevDetailsDialog",
|
||||
"query": """
|
||||
query DevDetailsDialog ($issueId: ID!) {
|
||||
developmentInformation(issueId: $issueId){
|
||||
details {
|
||||
instanceTypes {
|
||||
repository {
|
||||
name
|
||||
branches {
|
||||
name
|
||||
pullRequests {
|
||||
name
|
||||
status
|
||||
lastUpdate
|
||||
}
|
||||
reviews {
|
||||
state
|
||||
id
|
||||
}
|
||||
}
|
||||
pullRequests {
|
||||
id
|
||||
name
|
||||
branchName
|
||||
status
|
||||
reviewers{
|
||||
name
|
||||
isApproved
|
||||
}
|
||||
}
|
||||
}
|
||||
danglingPullRequests {
|
||||
id
|
||||
name
|
||||
branchName
|
||||
status
|
||||
reviewers{
|
||||
name
|
||||
isApproved
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
""",
|
||||
"variables": {"issueId": issue_id},
|
||||
},
|
||||
)
|
||||
res.raise_for_status()
|
||||
data = res.json()['data']
|
||||
if not data['developmentInformation']['details']['instanceTypes']:
|
||||
return []
|
||||
repo_name = data['developmentInformation']['details']['instanceTypes'][0]['repository'][0]['name']
|
||||
prs = data['developmentInformation']['details']['instanceTypes'][0]['danglingPullRequests']
|
||||
if not prs:
|
||||
repo_name = data['developmentInformation']['details']['instanceTypes'][0]['repository'][0]['name']
|
||||
prs = data['developmentInformation']['details']['instanceTypes'][0]['repository'][0]['pullRequests']
|
||||
|
||||
return [
|
||||
PullRequest(
|
||||
id=it['id'],
|
||||
name=it['name'],
|
||||
repository=repo_name,
|
||||
branch=it['branchName'],
|
||||
)
|
||||
for it in prs
|
||||
if it['status'] != 'MERGED'
|
||||
]
|
||||
|
||||
|
||||
console = rich.console.Console()
|
||||
|
||||
|
||||
def render_issue(issue: Issue, prs: list[PullRequest]):
|
||||
t = rich.table.Table(
|
||||
title=f'{issue.key} -- {issue.summary}',
|
||||
caption_justify='left',
|
||||
)
|
||||
t.add_column("Repository", no_wrap=True)
|
||||
t.add_column("Branch", no_wrap=True)
|
||||
t.add_column("Pull Request")
|
||||
|
||||
for it in prs:
|
||||
t.add_row(it.repository, it.branch, it.name)
|
||||
|
||||
console.print(t)
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
if not JIRA_EMAIL:
|
||||
logging.error('JIRA_EMAIL is not set')
|
||||
exit(1)
|
||||
if not JIRA_TOKEN:
|
||||
logging.error('JIRA_TOKEN is not set')
|
||||
exit(1)
|
||||
|
||||
j = Jira(email=JIRA_EMAIL, token=JIRA_TOKEN)
|
||||
|
||||
logging.info('fetching issues in "ready to release" status')
|
||||
issues = j.list_issues_for_release()
|
||||
for it in issues:
|
||||
logging.info(f'fetching pull requests for issue {it.key}')
|
||||
prs = j.get_approved_prs(it.id)
|
||||
if not prs:
|
||||
logging.info(f'no non-merged pull requests found for issue {it.key}')
|
||||
continue
|
||||
render_issue(it, prs)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,5 @@
|
||||
module github.com/abdusco/playground
|
||||
|
||||
go 1.23.0
|
||||
|
||||
require golang.org/x/sync v0.10.0
|
||||
@@ -0,0 +1,2 @@
|
||||
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
import http.server
|
||||
import random
|
||||
import socketserver
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import typing
|
||||
import webbrowser
|
||||
|
||||
# language=html
|
||||
inject_script = """
|
||||
<script>
|
||||
// Send periodic heartbeat
|
||||
let heartbeatInterval = setInterval(() => {
|
||||
fetch('/heartbeat', {method: 'POST'})
|
||||
.catch(() => {}); // Ignore errors
|
||||
}, 1000);
|
||||
|
||||
// Multiple event handlers for different close scenarios
|
||||
function sendShutdown() {
|
||||
clearInterval(heartbeatInterval);
|
||||
// Synchronous request to ensure it gets sent
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open('POST', '/shutdown', false); // false makes it synchronous
|
||||
try {
|
||||
xhr.send();
|
||||
} catch (e) {}
|
||||
}
|
||||
|
||||
window.addEventListener('beforeunload', sendShutdown);
|
||||
window.addEventListener('unload', sendShutdown);
|
||||
// setTimeout(sendShutdown, 5000);
|
||||
|
||||
// Handle visibility change (tab hidden/visible)
|
||||
document.addEventListener('visibilitychange', () => {
|
||||
if (document.visibilityState === 'hidden') {
|
||||
sendShutdown();
|
||||
}
|
||||
});
|
||||
</script>
|
||||
"""
|
||||
|
||||
|
||||
def handler_for_html(html: str) -> typing.Type[http.server.SimpleHTTPRequestHandler]:
|
||||
class Handler(http.server.SimpleHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
if self.path == "/":
|
||||
self.send_response(200)
|
||||
self.send_header("Content-type", "text/html")
|
||||
self.end_headers()
|
||||
self.wfile.write(html.encode())
|
||||
else:
|
||||
self.send_error(404)
|
||||
|
||||
def do_POST(self):
|
||||
if self.path == "/shutdown":
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.server.last_heartbeat = 0 # Force shutdown
|
||||
self.server.shutdown()
|
||||
elif self.path == "/heartbeat":
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.server.last_heartbeat = time.time()
|
||||
else:
|
||||
self.send_error(404)
|
||||
|
||||
def log_message(self, format, *args):
|
||||
# Suppress logging
|
||||
pass
|
||||
|
||||
return Handler
|
||||
|
||||
|
||||
class ServerWithHeartbeat(socketserver.TCPServer):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.last_heartbeat = time.time()
|
||||
|
||||
|
||||
def monitor_heartbeat(server: ServerWithHeartbeat, timeout: int = 2):
|
||||
while True:
|
||||
time.sleep(0.5)
|
||||
if time.time() - server.last_heartbeat > timeout:
|
||||
print("No heartbeat received, shutting down...")
|
||||
server.shutdown()
|
||||
break
|
||||
|
||||
|
||||
def run_server(handler: http.server.SimpleHTTPRequestHandler, port: int = 8000):
|
||||
with ServerWithHeartbeat(("", port), handler) as httpd:
|
||||
print(f"Serving at port {port}")
|
||||
|
||||
# Start heartbeat monitor in separate thread
|
||||
monitor_thread = threading.Thread(target=monitor_heartbeat, args=(httpd,))
|
||||
monitor_thread.daemon = True
|
||||
monitor_thread.start()
|
||||
|
||||
httpd.serve_forever()
|
||||
|
||||
|
||||
def main():
|
||||
html = sys.stdin.read()
|
||||
if "<body>" in html:
|
||||
html_to_display = html.replace("</body>", f"{inject_script}</body>")
|
||||
else:
|
||||
html_to_display = f"<html><body>{html}{inject_script}</body></html>"
|
||||
|
||||
handler = handler_for_html(html_to_display)
|
||||
|
||||
# Start the server in a separate thread
|
||||
port = random.randint(20000, 65535)
|
||||
server_thread = threading.Thread(target=run_server, kwargs=dict(handler=handler, port=port))
|
||||
server_thread.daemon = True
|
||||
server_thread.start()
|
||||
|
||||
# Wait a moment for the server to start
|
||||
time.sleep(0.5)
|
||||
|
||||
# Open the web browser
|
||||
webbrowser.open(f"http://localhost:{port}")
|
||||
|
||||
# Wait for the server thread to finish
|
||||
server_thread.join()
|
||||
print("Server stopped")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
print("\nReceived keyboard interrupt, exiting...")
|
||||
raise SystemExit(0)
|
||||
Executable
+119
@@ -0,0 +1,119 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// language=html
|
||||
const injectScript = `
|
||||
<script>
|
||||
const sendShutdown = () => navigator.sendBeacon('/shutdown');
|
||||
|
||||
window.addEventListener('beforeunload', sendShutdown);
|
||||
window.addEventListener('unload', sendShutdown);
|
||||
</script>
|
||||
`
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func newHandler(html string, onShutdown func()) *http.ServeMux {
|
||||
handler := http.NewServeMux()
|
||||
|
||||
handler.HandleFunc("POST /shutdown", func(w http.ResponseWriter, r *http.Request) {
|
||||
onShutdown()
|
||||
})
|
||||
|
||||
handler.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.Write([]byte(html))
|
||||
})
|
||||
|
||||
return handler
|
||||
}
|
||||
|
||||
func run() error {
|
||||
html, err := io.ReadAll(os.Stdin)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read HTML from stdin: %w", err)
|
||||
}
|
||||
|
||||
htmlStr := string(html)
|
||||
if strings.Contains(htmlStr, "</body>") {
|
||||
htmlStr = strings.ReplaceAll(string(html), "</body>", injectScript+"</body>")
|
||||
} else {
|
||||
htmlStr += injectScript
|
||||
}
|
||||
|
||||
listener, err := net.Listen("tcp", ":0")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create listener: %w", err)
|
||||
}
|
||||
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
port := listener.Addr().(*net.TCPAddr).Port
|
||||
|
||||
server := &http.Server{Addr: listener.Addr().String()}
|
||||
server.Handler = newHandler(htmlStr, func() {
|
||||
cancel()
|
||||
})
|
||||
|
||||
servedURL := fmt.Sprintf("http://localhost:%d", port)
|
||||
|
||||
go func() {
|
||||
log.Printf("serving at %s", servedURL)
|
||||
if err := server.Serve(listener); err != nil && err != http.ErrServerClosed {
|
||||
log.Fatalf("failed to serve: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
// wait for server to start
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
if err := openBrowser(servedURL); err != nil {
|
||||
log.Fatalf("failed to open browser: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
<-ctx.Done()
|
||||
|
||||
log.Printf("shutting down server")
|
||||
|
||||
if err := server.Shutdown(ctx); err != nil {
|
||||
return fmt.Errorf("failed to shutdown server: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func openBrowser(url string) error {
|
||||
var cmd *exec.Cmd
|
||||
switch runtime.GOOS {
|
||||
case "darwin":
|
||||
cmd = exec.Command("open", url)
|
||||
case "linux":
|
||||
cmd = exec.Command("xdg-open", url)
|
||||
case "windows":
|
||||
cmd = exec.Command("cmd", "/c", "start", url)
|
||||
default:
|
||||
return fmt.Errorf("invalid URL: %s", url)
|
||||
}
|
||||
return cmd.Run()
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def find_editor() -> str:
|
||||
editor = os.getenv("EDITOR")
|
||||
if editor:
|
||||
return editor
|
||||
|
||||
for candidate in ["micro", "nano"]:
|
||||
if shutil.which(candidate):
|
||||
return candidate
|
||||
|
||||
return "nano"
|
||||
|
||||
|
||||
def edit_text(text: str) -> str | None:
|
||||
with tempfile.NamedTemporaryFile(suffix=".tmp", delete=False) as tf:
|
||||
temp_file_path = Path(tf.name)
|
||||
temp_file_path.write_text(text)
|
||||
|
||||
old_modtime = temp_file_path.stat().st_mtime
|
||||
subprocess.call([*find_editor().split(), str(temp_file_path)])
|
||||
new_modtime = temp_file_path.stat().st_mtime
|
||||
if new_modtime == old_modtime:
|
||||
logging.warning("editor exited without saving")
|
||||
return None
|
||||
|
||||
return temp_file_path.read_text()
|
||||
|
||||
|
||||
def move_to_trash(file_path: Path):
|
||||
if not file_path.is_file():
|
||||
raise FileNotFoundError
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
"osascript",
|
||||
"-e",
|
||||
f'tell app "Finder" to move POSIX file "{file_path}" to trash',
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def rename_files(files: list[Path], cwd: Path | None = None):
|
||||
if not cwd:
|
||||
cwd = files[0].parent
|
||||
|
||||
originals = [f"{f.relative_to(cwd)}" for f in files]
|
||||
edited_raw = edit_text("\n".join(originals))
|
||||
if not edited_raw:
|
||||
return
|
||||
edited = edited_raw.splitlines(keepends=False)
|
||||
|
||||
deletables = []
|
||||
moveables = []
|
||||
for source, target in zip(originals, edited):
|
||||
if target.startswith("#") or target == "":
|
||||
deletables.append(Path(source))
|
||||
continue
|
||||
if source != target:
|
||||
moveables.append((cwd / source, cwd / target))
|
||||
|
||||
for source, target in moveables:
|
||||
logging.info(f"moving {source} -> {target}")
|
||||
source.rename(target)
|
||||
|
||||
for f in deletables:
|
||||
try:
|
||||
logging.info(f"trashing {f}")
|
||||
move_to_trash(f)
|
||||
except:
|
||||
logging.error(f"failed to trash {f}")
|
||||
|
||||
|
||||
def parse_args():
|
||||
arger = argparse.ArgumentParser()
|
||||
arger.add_argument("filenames", type=Path, nargs="+", help="File paths")
|
||||
arger.add_argument("--cwd", type=Path, help="Current working directory")
|
||||
return arger.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
rename_files(args.filenames, cwd=args.cwd)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
main()
|
||||
@@ -0,0 +1,57 @@
|
||||
import argparse
|
||||
|
||||
import httpx
|
||||
|
||||
OPENAPI_TOKEN = "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"
|
||||
|
||||
|
||||
http = httpx.Client(
|
||||
base_url='https://api.openai.com/v1/',
|
||||
headers={
|
||||
'Authorization': f'Bearer {OPENAPI_TOKEN}',
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
|
||||
def generate_examples(text: str) -> str:
|
||||
payload = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": """
|
||||
You are to provide colloquial and idiomatic German translations of given text. List 10 alternatives.
|
||||
""",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": text,
|
||||
},
|
||||
],
|
||||
"temperature": 0.7,
|
||||
"max_tokens": 256,
|
||||
"top_p": 1,
|
||||
"frequency_penalty": 1.06,
|
||||
"presence_penalty": 0.42,
|
||||
"model": "gpt-3.5-turbo",
|
||||
"stream": False,
|
||||
}
|
||||
res = http.post('/chat/completions', json=payload)
|
||||
res.raise_for_status()
|
||||
return res.json()['choices'][0]['message']['content'].strip()
|
||||
|
||||
|
||||
def parse_args():
|
||||
arger = argparse.ArgumentParser()
|
||||
arger.add_argument('prompt', help='Phrase to generate sentences for')
|
||||
return arger.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
response = generate_examples(args.prompt)
|
||||
print(response)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+16
-1
@@ -36,6 +36,17 @@ def download_with_aria2(url: str, cwd: Path = Path.cwd(), extra_args: list[str]
|
||||
subprocess.run(args, text=True, cwd=str(cwd.resolve()))
|
||||
|
||||
|
||||
def download_torrent(torrent_path: Path):
|
||||
res = http.put('/torrents/addTorrent', content=torrent_path.read_bytes())
|
||||
res.raise_for_status()
|
||||
data = res.json()
|
||||
|
||||
res = http.get(f'/torrents/info/{data["id"]}')
|
||||
res.raise_for_status()
|
||||
data = res.json()
|
||||
print()
|
||||
|
||||
|
||||
def parse_args():
|
||||
arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
arger.add_argument('url', help='URL to unrestrict & download')
|
||||
@@ -66,4 +77,8 @@ def main():
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
download_torrent(
|
||||
Path(
|
||||
'/Users/abdus/Downloads/[Empornium][PornWorld] All Her Holes Pounded Hard [Veronica Leal] [1080p] {Se7enSeas} [x265].torrent'
|
||||
)
|
||||
)
|
||||
|
||||
Executable
+75
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3.10
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
|
||||
def parse_args():
|
||||
arger = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
arger.add_argument('dirs', help='Path to the directories', type=Path, nargs='+')
|
||||
return arger.parse_args()
|
||||
|
||||
|
||||
def natural_sort_key(path: Path) -> tuple[str, ...]:
|
||||
s = path.stem
|
||||
return tuple(f'{int(p):010d}' if p.isnumeric() else p for p in re.findall(r'(\D+|\d+)', s))
|
||||
|
||||
|
||||
def rename_children(cwd: Path) -> None:
|
||||
assert cwd.is_dir()
|
||||
|
||||
files = [f for f in cwd.glob('*') if f.is_file()]
|
||||
files = sorted(files, key=natural_sort_key)
|
||||
|
||||
new_stem = cwd.name
|
||||
|
||||
jobs = []
|
||||
for i, f in enumerate(files):
|
||||
target = f.with_stem(f'{new_stem}__{i:04d}')
|
||||
if f.suffix == '.jpeg':
|
||||
target = target.with_suffix('.jpg')
|
||||
if target.is_file():
|
||||
logging.error(f'target already exists: {target}')
|
||||
continue
|
||||
jobs.append(MoveOp(source=f, target=target))
|
||||
f.rename(target)
|
||||
|
||||
if not jobs:
|
||||
return
|
||||
|
||||
write_undo_script(Path.cwd(), jobs)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class MoveOp:
|
||||
source: Path
|
||||
target: Path
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
'source': str(self.source),
|
||||
'target': str(self.target),
|
||||
}
|
||||
|
||||
|
||||
def write_undo_script(save_dir: Path, ops: list[MoveOp]):
|
||||
now = datetime.datetime.now().isoformat().replace(':', '')
|
||||
save_path = save_dir / f'undo_{now}.json'
|
||||
with save_path.open('w', encoding='utf-8') as f:
|
||||
for it in ops:
|
||||
f.write(json.dumps(it.to_dict()) + '\n')
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
for d in args.dirs:
|
||||
rename_children(d)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+6
-1
@@ -2,5 +2,10 @@ beautifulsoup4==4.11.2
|
||||
boto3==1.19.1
|
||||
fastapi==0.86.0
|
||||
httpx==0.23.3
|
||||
pytest==6.2.5
|
||||
pytest==7.2.1
|
||||
typer==0.6.1
|
||||
|
||||
inquirer==3.1.2
|
||||
playwright==1.31.1
|
||||
torf==4.1.4
|
||||
bs4==0.0.1
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env python3
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# dependencies = [
|
||||
# "psycopg[binary]",
|
||||
# ]
|
||||
# ///
|
||||
import argparse
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
import typing
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
import psycopg.rows
|
||||
|
||||
import ffmpeg
|
||||
|
||||
|
||||
def hash_partial(f: Path) -> str:
|
||||
sha1 = hashlib.sha1()
|
||||
chunk_size = 1024 * 1024 * 10 # 10MB chunk size
|
||||
|
||||
total_read = 0
|
||||
with f.open("rb") as file:
|
||||
while chunk := file.read(chunk_size):
|
||||
total_read += len(chunk)
|
||||
sha1.update(chunk)
|
||||
break # Only reads the first 10MB
|
||||
|
||||
return f"sha1:{total_read}:{sha1.hexdigest()}"
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def connect_db() -> typing.Generator[psycopg.Connection, typing.Any, typing.Any]:
|
||||
with psycopg.connect("postgres://abdus:abdus@db.abdus.dev:5444/smut?sslmode=disable") as conn:
|
||||
conn.row_factory = psycopg.rows.dict_row
|
||||
yield conn
|
||||
|
||||
|
||||
def video_exists(con: psycopg.Connection, file_path: str) -> bool:
|
||||
stmt = con.execute(
|
||||
"UPDATE videos SET last_seen_at = NOW() WHERE file_path = %s RETURNING true AS exists",
|
||||
[file_path],
|
||||
)
|
||||
row = stmt.fetchone()
|
||||
return row["exists"]
|
||||
|
||||
|
||||
def delete_video(con: psycopg.Connection, file_path: str) -> bool:
|
||||
con.execute("DELETE FROM videos WHERE file_path = %s", [file_path])
|
||||
|
||||
|
||||
def save_video(
|
||||
con: psycopg.Connection,
|
||||
category: str,
|
||||
file_path: str,
|
||||
ffprobe: dict,
|
||||
hash_partial: str,
|
||||
):
|
||||
sql = """INSERT INTO videos(category, file_path, ffprobe, hash_partial) VALUES (%s, %s, %s, %s) ON CONFLICT(file_path) SET last_seen_at = NOW();"""
|
||||
con.execute(
|
||||
sql,
|
||||
[
|
||||
category,
|
||||
file_path,
|
||||
json.dumps(ffprobe),
|
||||
hash_partial,
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def save_error(con: psycopg.Connection, file_path: str, stderr: str):
|
||||
sql = """INSERT INTO scan_state(file_path, stderr) VALUES (%s, %s) ON conflict do nothing"""
|
||||
con.execute(
|
||||
sql,
|
||||
[
|
||||
file_path,
|
||||
stderr,
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def scan_videos(con: psycopg.Connection, video_paths: list[Path]) -> None:
|
||||
for i, f in enumerate(video_paths):
|
||||
if "/_picks/" in str(f):
|
||||
continue
|
||||
progress = f"[{i + 1}/{len(video_paths)}]"
|
||||
if f.is_dir():
|
||||
continue
|
||||
|
||||
file_path = str(f.absolute())
|
||||
|
||||
if video_exists(con, file_path=file_path):
|
||||
logging.debug(f"{progress} video already exists {f=}")
|
||||
continue
|
||||
|
||||
logging.debug(f"probing {f=}")
|
||||
try:
|
||||
probed = ffmpeg.ffprobe(f)
|
||||
except subprocess.CalledProcessError as e:
|
||||
save_error(con, file_path=file_path, stderr=e.stderr)
|
||||
continue
|
||||
|
||||
file_hash = hash_partial(f)
|
||||
|
||||
save_video(
|
||||
category=category_for_file(f),
|
||||
con=con,
|
||||
file_path=file_path,
|
||||
ffprobe=probed.__dict__,
|
||||
hash_partial=file_hash,
|
||||
)
|
||||
logging.info(f"{progress} saved {f=}")
|
||||
con.commit()
|
||||
|
||||
|
||||
def category_for_file(file_path: Path) -> str:
|
||||
if str(file_path).startswith("/Volumes/"):
|
||||
return f"downloaded.{file_path.parts[1].lower()}"
|
||||
if str(file_path).startswith("/mnt/box"):
|
||||
return "box"
|
||||
raise ValueError(f"unknown category for {file_path}")
|
||||
|
||||
|
||||
def find_copies(con: psycopg.Connection, video_path: Path) -> list[str]:
|
||||
probed = ffmpeg.ffprobe(video_path)
|
||||
|
||||
sql = """
|
||||
SELECT
|
||||
file_path
|
||||
FROM videos
|
||||
WHERE
|
||||
file_path != %(file_path)s
|
||||
AND (
|
||||
hash_partial = %(hash_partial)s
|
||||
OR (abs(duration_sec - %(duration_sec)s) < 0.1 AND abs(size_bytes - %(size_bytes)s) < 3000000)
|
||||
);
|
||||
"""
|
||||
|
||||
stmt: psycopg.Cursor = con.execute(
|
||||
sql,
|
||||
{
|
||||
"file_path": str(video_path.absolute()),
|
||||
"hash_partial": hash_partial(video_path),
|
||||
"duration_sec": probed.duration_sec,
|
||||
"size_bytes": probed.size_bytes,
|
||||
},
|
||||
)
|
||||
rows = stmt.fetchall()
|
||||
|
||||
return [it["file_path"] for it in rows]
|
||||
|
||||
|
||||
def parse_args():
|
||||
arger = argparse.ArgumentParser()
|
||||
arger.add_argument("video_paths", nargs="+", type=Path, help="Path to the videos")
|
||||
arger.add_argument("--debug", action="store_true", help="Path to the videos")
|
||||
|
||||
return arger.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(format=logging.BASIC_FORMAT, level=logging.INFO)
|
||||
args = parse_args()
|
||||
if args.debug:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
|
||||
video_paths: list[Path] = args.video_paths
|
||||
if not video_paths:
|
||||
logging.info("no files to scan")
|
||||
return
|
||||
|
||||
with connect_db() as con:
|
||||
scan_videos(con=con, video_paths=video_paths)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
-190
@@ -1,190 +0,0 @@
|
||||
import functools
|
||||
import hashlib
|
||||
import json
|
||||
import textwrap
|
||||
import typing
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import kubernetes.config
|
||||
import yaml
|
||||
|
||||
|
||||
def make_client() -> kubernetes.client.CoreV1Api:
|
||||
kubernetes.config.load_kube_config('/Users/abdus/Desktop/kubeconfig-kube.yml')
|
||||
return kubernetes.client.CoreV1Api()
|
||||
|
||||
|
||||
class Kubernetes:
|
||||
def __init__(self, client: kubernetes.client.CoreV1Api, namespace='default'):
|
||||
self.client = client
|
||||
self.custom_objects = kubernetes.client.CustomObjectsApi(self.client.api_client)
|
||||
self.namespace = namespace
|
||||
|
||||
@classmethod
|
||||
def from_kubeconfig(cls, kubeconfig_path: str, namespace: str) -> 'Kubernetes':
|
||||
kubernetes.config.load_kube_config(kubeconfig_path)
|
||||
return cls(kubernetes.client.CoreV1Api(), namespace=namespace)
|
||||
|
||||
def read_logs(self, pod_name: str, container_name: typing.Optional[str] = None, timestamps: bool = False) -> str:
|
||||
return self.client.read_namespaced_pod_log(
|
||||
pod_name,
|
||||
self.namespace,
|
||||
container=container_name,
|
||||
timestamps=timestamps,
|
||||
pretty=True,
|
||||
)
|
||||
|
||||
def get_tekton_object(self, plural: str, name: str = None):
|
||||
getter = functools.partial(
|
||||
self.custom_objects.get_namespaced_custom_object,
|
||||
group='tekton.dev',
|
||||
version='v1beta1',
|
||||
namespace=self.namespace,
|
||||
)
|
||||
return getter(plural=plural, name=name)
|
||||
|
||||
def list_tekton_object(self, plural: str):
|
||||
getter = functools.partial(
|
||||
self.custom_objects.list_namespaced_custom_object,
|
||||
group='tekton.dev',
|
||||
version='v1beta1',
|
||||
namespace=self.namespace,
|
||||
)
|
||||
return getter(plural=plural)['items']
|
||||
|
||||
def create_from_yaml(self, yaml_text: str) -> list[dict]:
|
||||
specs = list(yaml.safe_load_all(yaml_text))
|
||||
return kubernetes.utils.create_from_yaml(
|
||||
kubernetes.client.CustomObjectsApi(self.client.api_client),
|
||||
yaml_objects=specs,
|
||||
namespace=self.namespace,
|
||||
)
|
||||
|
||||
|
||||
def read_logs():
|
||||
task_hello_world = k8s.get_tekton_object(plural='tasks', name='echo-hello-world')
|
||||
taskruns = k8s.list_tekton_object(plural='taskruns')
|
||||
|
||||
pod_name = taskruns['items'][0]['status']['podName']
|
||||
container_name = taskruns['items'][0]['status']['containerName']
|
||||
logs = k8s.read_logs(pod_name, container_name)
|
||||
print(logs)
|
||||
|
||||
|
||||
k8s = Kubernetes.from_kubeconfig('/Users/abdus/Desktop/kubeconfig.yaml', namespace='api-server')
|
||||
|
||||
|
||||
def get_pipelineru_status():
|
||||
result = k8s.get_tekton_object('pipelineruns', name='pr-2rr6t')
|
||||
status = result['status']['conditions'][0]['status']
|
||||
reason = result['status']['conditions'][0]['reason']
|
||||
failed_taskruns = [
|
||||
name for name, it in result['status']['taskRuns'].items() if it['status']['conditions'][0]['status'] == 'False'
|
||||
]
|
||||
|
||||
logs = []
|
||||
for it in result['status']['taskRuns'].values():
|
||||
task_name = it['pipelineTaskName']
|
||||
task_status = it['status']['conditions'][0]['reason']
|
||||
pod_name = it['status']['podName']
|
||||
|
||||
if task_status != 'Failed':
|
||||
continue
|
||||
|
||||
# fetch all logs for all steps to get a better picture of the error
|
||||
# (regardless of whether it is successful or not)
|
||||
for i, step in enumerate(it['status']['steps'], start=1):
|
||||
container_name = step['container']
|
||||
header = f'# {task_name}, step {i}: {step["name"]}'
|
||||
logs.append('#' * len(header))
|
||||
logs.append(header)
|
||||
logs.append('#' * len(header))
|
||||
step_logs = k8s.read_logs(pod_name, container_name, timestamps=True)
|
||||
logs.append(step_logs)
|
||||
|
||||
log_text = '\n'.join(logs)
|
||||
return {
|
||||
'pipelinerun_name': result['metadata']['name'],
|
||||
'namespace': result['metadata']['namespace'],
|
||||
'status': status,
|
||||
'reason': reason,
|
||||
'failed_taskruns': failed_taskruns,
|
||||
'logs': log_text,
|
||||
}
|
||||
|
||||
|
||||
def create_taskrun():
|
||||
k = kubernetes.client.CustomObjectsApi()
|
||||
res = k.create_namespaced_custom_object(
|
||||
group='tekton.dev',
|
||||
version='v1beta1',
|
||||
namespace='default',
|
||||
plural='taskruns',
|
||||
body={
|
||||
'apiVersion': 'tekton.dev/v1beta1',
|
||||
'kind': 'TaskRun',
|
||||
'metadata': {'name': 'testing1234' + uuid.uuid4().hex},
|
||||
'spec': {
|
||||
'taskRef': {'name': 'task-with-json'},
|
||||
'params': [
|
||||
{
|
||||
'name': 'json_value',
|
||||
'value': '{"foo": "bar", "nested": {"a": "1"}}',
|
||||
}
|
||||
],
|
||||
},
|
||||
},
|
||||
)
|
||||
print(res)
|
||||
|
||||
|
||||
def migrate_pipelines():
|
||||
k = kubernetes.client.CustomObjectsApi()
|
||||
|
||||
pipeline_yaml = Path('sample_pipeline.yaml').read_text()
|
||||
parsed_pipeline = yaml.safe_load(pipeline_yaml)
|
||||
|
||||
params = dict(group='tekton.dev', version='v1beta1', namespace='default', plural='pipelines')
|
||||
k.delete_namespaced_custom_object(
|
||||
**params,
|
||||
name=parsed_pipeline['metadata']['name'],
|
||||
)
|
||||
res = k.create_namespaced_custom_object(
|
||||
**params,
|
||||
body=parsed_pipeline,
|
||||
)
|
||||
return parsed_pipeline
|
||||
|
||||
|
||||
def trigger_pipeline():
|
||||
k = kubernetes.client.CustomObjectsApi()
|
||||
pipeline = migrate_pipelines()
|
||||
pipeline_name = pipeline['metadata']['name']
|
||||
|
||||
payload = {
|
||||
'image': 'nginx',
|
||||
'json_value': {'nested': {'nested2': {'a': 1}}},
|
||||
}
|
||||
|
||||
res = k.create_namespaced_custom_object(
|
||||
group='tekton.dev',
|
||||
version='v1beta1',
|
||||
namespace='default',
|
||||
plural='pipelineruns',
|
||||
body={
|
||||
'apiVersion': 'tekton.dev/v1beta1',
|
||||
'kind': 'PipelineRun',
|
||||
'metadata': {'name': 'deploy-app-' + uuid.uuid4().hex},
|
||||
'spec': {
|
||||
'pipelineRef': {'name': pipeline_name},
|
||||
'params': [
|
||||
{'name': k, 'value': json.dumps(v) if not isinstance(v, str) else v} for k, v in payload.items()
|
||||
],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python3.11
|
||||
import argparse
|
||||
import io
|
||||
from pathlib import Path
|
||||
|
||||
import torf
|
||||
|
||||
|
||||
def parse_args() -> dict:
|
||||
arger = argparse.ArgumentParser()
|
||||
arger.add_argument('torrent_path', type=Path, help='Path to torrent file')
|
||||
arger.add_argument('--name', type=Path, help='New filename', required=True)
|
||||
return dict(arger.parse_args().__dict__)
|
||||
|
||||
|
||||
def modify_torrent(
|
||||
torrent_file: bytes,
|
||||
name: str,
|
||||
announce_url: str,
|
||||
created_by: str = '',
|
||||
comment: str = '',
|
||||
private: bool = True,
|
||||
) -> bytes:
|
||||
t = torf.Torrent.read_stream(io.BytesIO(torrent_file))
|
||||
|
||||
if len(t.files) > 1:
|
||||
raise ValueError('Torrent contains multiple files')
|
||||
|
||||
t.name = name
|
||||
t.private = private
|
||||
t.comment = comment
|
||||
t.created_by = created_by
|
||||
t.trackers.clear()
|
||||
t.trackers.insert(0, announce_url)
|
||||
|
||||
return t.dump()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
torrent_path: Path = args.pop('torrent_path')
|
||||
modded = modify_torrent(
|
||||
**args,
|
||||
torrent_file=torrent_path.read_bytes(),
|
||||
announce_url='http://tracker.empornium.sx:2710/tegqucis10uanp672qh6nhn393xlkncs/announce',
|
||||
created_by='zzzp',
|
||||
comment='created by zzzp',
|
||||
private=True,
|
||||
)
|
||||
save_path = torrent_path.with_stem(f'{torrent_path.stem}.mod')
|
||||
save_path.write_bytes(modded)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+8
-2
@@ -4,6 +4,7 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import select
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
@@ -210,7 +211,10 @@ def upload_image(image_path: Path) -> str:
|
||||
return image_url
|
||||
|
||||
|
||||
ParsedFilename = typing.TypedDict('ParsedFilename', {'actors': list[str], 'studio': str, 'date': str, 'title': str, 'tags': list[str]})
|
||||
ParsedFilename = typing.TypedDict(
|
||||
'ParsedFilename', {'actors': list[str], 'studio': str, 'date': str, 'title': str, 'tags': list[str]}
|
||||
)
|
||||
|
||||
|
||||
def parse_filename(filename: str) -> ParsedFilename:
|
||||
parsed = {
|
||||
@@ -244,6 +248,7 @@ def parse_filename(filename: str) -> ParsedFilename:
|
||||
|
||||
return parsed
|
||||
|
||||
|
||||
def generate_title(parsed: ParsedFilename) -> str:
|
||||
parts = []
|
||||
if actors := parsed.get('actors'):
|
||||
@@ -473,7 +478,8 @@ def main():
|
||||
thumbnail_path = video_path.with_suffix(".jpg")
|
||||
if old_thumb_path.is_file():
|
||||
old_thumb_path.rename(thumbnail_path)
|
||||
ffmpeg.make_thumbnail_tile(video_path, image_path=thumbnail_path, skip_if_exists=True)
|
||||
image_jpeg = ffmpeg.make_thumbnail_tile(video_path)
|
||||
thumbnail_path.write_bytes(image_jpeg)
|
||||
logging.info(f"Saved thumbnails at {thumbnail_path}")
|
||||
|
||||
post_bbcode = generate_post_bbcode(video_path, thumbnail_path)
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
from cmath import exp
|
||||
from torrentify import generate_title, parse_filename, ParsedFilename
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
["filename", "expected"],
|
||||
[
|
||||
[
|
||||
"To Ki -- @Studio -- 2022-10-04",
|
||||
{
|
||||
"actors": ["To Ki"],
|
||||
"date": "2022-10-04",
|
||||
"studio": "Studio",
|
||||
"tags": ["to.ki", "studio.com", "2022", "2022.10"],
|
||||
},
|
||||
],
|
||||
[
|
||||
"To Ki, Ki To -- @Studio -- Title -- 2022-10-04",
|
||||
{
|
||||
"actors": ["To Ki", "Ki To"],
|
||||
"date": "2022-10-04",
|
||||
"studio": "Studio",
|
||||
"title": "Title",
|
||||
"tags": ["to.ki", "ki.to", "studio.com", "2022", "2022.10"],
|
||||
},
|
||||
],
|
||||
],
|
||||
)
|
||||
def test_parse_filename(filename: str, expected: dict):
|
||||
parsed = parse_filename(filename)
|
||||
print(filename, parsed)
|
||||
assert parsed == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
["parsed", "expected"],
|
||||
[
|
||||
[
|
||||
{
|
||||
"actors": ["To Ki"],
|
||||
"date": "2022-10-04",
|
||||
"studio": "Studio",
|
||||
},
|
||||
'To Ki -- @Studio -- 2022-10-04',
|
||||
],
|
||||
[
|
||||
{
|
||||
"actors": ["To Ki", "Ki To"],
|
||||
"date": "2022-10-04",
|
||||
"title": "Title",
|
||||
"studio": "Studio",
|
||||
},
|
||||
'To Ki, Ki To -- @Studio -- Title -- 2022-10-04',
|
||||
],
|
||||
],
|
||||
)
|
||||
def test_generate_title(parsed: ParsedFilename, expected: str):
|
||||
generated = generate_title(parsed)
|
||||
assert generated == expected
|
||||
Reference in New Issue
Block a user