HugeFrog24/jailbirdz-dl
1
0
Fork 0
mirror of https://github.com/HugeFrog24/jailbirdz-dl.git synced 2026-10-27 17:25:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Better input dir handling

Browse Source
This commit is contained in:
HugeFrog24
2026-09-04 18:16:43 +02:00
parent 3d21ac07ca
commit 1d83988811
2 changed files with 158 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+5 -2
View File
@@ -89,10 +89,13 @@ Options:
--poll-interval SECS State poll interval in seconds (default: 30)
--skip-wait Upload without waiting for transcoding
--nsfw Mark videos as NSFW
-n, --dry-run Print what would be uploaded
-n, --dry-run Print what would be uploaded (checks the channel, uploads nothing)
--scan-all Scan every subdirectory of --input, not just the site folders
```
Uploads in resumable 10 MB chunks. After each batch, waits for transcoding and object storage to complete before uploading the next batch — this prevents disk exhaustion on the PeerTube server. Videos already present on the channel (matched by name) are skipped. Progress is tracked in `.uploaded` inside the input directory.
Uploads in resumable 10 MB chunks. After each batch, waits for transcoding and object storage to complete before uploading the next batch — this prevents disk exhaustion on the PeerTube server. Videos already present on the channel (matched by name) are skipped. If `--input` contains any of the configured site folders (`jailbirdz`, `pinkcuffs`, `femuniverse`), only those are scanned — so the input directory can be a shared drive holding unrelated purchases alongside them. Anything skipped is listed on startup; `--scan-all` includes it. A directory with no site folders is walked in full.
Progress is tracked in `.uploaded` inside the input directory, so a new `--input` directory starts with an empty ledger — the channel itself is reconciled against on every run (dry runs included) to fill it back in.
## CI / Nightly Indexing
upload.py
+110 -28
View File
@@ -8,7 +8,7 @@ Usage:
python upload.py # upload from ./downloads
python upload.py -i /mnt/vol/dl # custom input dir
python upload.py --batch-size 2 # upload 2, wait, repeat
python upload.py --dry-run # preview without uploading
python upload.py --dry-run # preview (checks the channel, uploads nothing)
python upload.py --skip-wait # upload without waiting
Required (CLI flag or env var):
@@ -32,6 +32,7 @@ import requests
from dotenv import load_dotenv
from check_clashes import fmt_size, url_to_filename, VIDEO_EXTS, load_video_map
from config import SITES
from download import (
collect_urls,
get_paths_for_mode,
@@ -448,14 +449,43 @@ def build_path_to_meta(
return result
def find_videos(input_dir: str) -> set[Path]:
"""Walk input_dir and return a set of relative paths for all video files."""
def site_dirs_in(input_dir: str) -> set[str]:
"""Names of the per-site subdirectories that actually exist under input_dir."""
root = Path(input_dir)
if not root.is_dir():
return set()
return {site for site in SITES if (root / site).is_dir()}
def unmanaged_dirs_in(input_dir: str, site_dirs: set[str]) -> list[str]:
"""Top-level subdirectories of input_dir that belong to no configured site."""
root = Path(input_dir)
if not root.is_dir():
return []
return sorted(
d.name
for d in root.iterdir()
if d.is_dir() and not d.name.startswith(".") and d.name not in site_dirs
)
def find_videos(input_dir: str, only_dirs: set[str] | None = None) -> set[Path]:
"""Walk input_dir and return a set of relative paths for all video files.
When *only_dirs* is given, just those top-level directories are descended
into; loose files sitting directly in input_dir are always included. That
keeps a shared parent directory — an external drive holding other purchases
next to the site folders — from dragging in unrelated albums.
"""
top = Path(input_dir)
found = set()
for root, dirs, files in os.walk(input_dir):
for cur, dirs, files in os.walk(str(top)):
dirs[:] = [d for d in dirs if not d.startswith(".")]
if only_dirs is not None and Path(cur) == top:
dirs[:] = [d for d in dirs if d in only_dirs]
for f in files:
if Path(f).suffix.lower() in VIDEO_EXTS:
found.add((Path(root) / f).relative_to(input_dir))
found.add((Path(cur) / f).relative_to(top))
return found
@@ -532,6 +562,11 @@ def main() -> None:
ap.add_argument(
"--dry-run", "-n", action="store_true", help="Preview what would be uploaded"
)
ap.add_argument(
"--scan-all",
action="store_true",
help="Scan every subdirectory of --input, not just the configured site folders",
)
args = ap.parse_args()
url = args.url or os.environ.get("PEERTUBE_URL")
@@ -539,7 +574,6 @@ def main() -> None:
channel = args.channel or os.environ.get("PEERTUBE_CHANNEL")
password = args.password or os.environ.get("PEERTUBE_PASSWORD")
if not args.dry_run:
missing = [
label
for label, val in [
@@ -550,7 +584,8 @@ def main() -> None:
]
if not val
]
if missing:
have_creds = not missing
if missing and not args.dry_run:
for label in missing:
print(f"[!] Required: {label}")
sys.exit(1)
@@ -558,6 +593,22 @@ def main() -> None:
# ── load metadata & scan disk ──
video_map = load_video_map()
path_meta = build_path_to_meta(video_map, args.input)
# --input may be a shared directory that holds unrelated purchases beside
# the per-site folders. Once any site folder is present, treat the layout
# as site-partitioned and ignore everything else; a flat directory (no site
# folders at all) is still walked in full.
site_dirs = set() if args.scan_all else site_dirs_in(args.input)
if site_dirs:
skipped = unmanaged_dirs_in(args.input, site_dirs)
if skipped:
print(
f"[!] Ignoring {len(skipped)} unmanaged folder(s) in {args.input}: "
f"{', '.join(skipped)}"
)
print(" (pass --scan-all to upload those too)")
on_disk = find_videos(args.input, site_dirs)
else:
on_disk = find_videos(args.input)
unmatched = on_disk - set(path_meta.keys())
@@ -568,11 +619,12 @@ def main() -> None:
for rel in unmatched:
path_meta[rel] = {"title": "", "description": ""}
ledger = Path(args.input) / UPLOADED_FILE
uploaded = load_uploaded(args.input)
pending = sorted(rel for rel in on_disk if rel not in uploaded)
print(f"[+] {len(on_disk)} video files in {args.input}/")
print(f"[+] {len(uploaded)} already uploaded")
print(f"[+] {len(uploaded)} already uploaded (per {ledger})")
print(f"[+] {len(pending)} pending")
print(f"[+] Batch size: {args.batch_size}")
@@ -580,24 +632,22 @@ def main() -> None:
print("\nAll videos already uploaded.")
return
# ── dry run ──
if args.dry_run:
total_bytes = 0
for rel in pending:
meta = path_meta.get(rel, {})
name = make_pt_name(meta.get("title", ""), rel.name)
sz = (Path(args.input) / rel).stat().st_size
total_bytes += sz
print(f" [{fmt_size(sz):>10}] {name}")
print(f"\n Total: {fmt_size(total_bytes)} across {len(pending)} videos")
return
# ── connect to the channel ────────────────────────────────────────
# The .uploaded ledger lives inside --input, so pointing at a fresh
# directory (an external drive, say) starts from an empty one. The channel
# is the real source of truth, so reconcile against it before deciding what
# is pending — dry runs included, whenever credentials are available.
base = ""
token = ""
channel_id = 0
existing: set[str] = set()
if have_creds:
assert url is not None
assert username is not None
assert channel is not None
assert password is not None
# ── authenticate ──
base = url.rstrip("/")
if not base.startswith("http"):
base = "https://" + base
@@ -612,13 +662,20 @@ def main() -> None:
name_counts = get_channel_video_names(base, token, channel)
existing = set(name_counts)
total = sum(name_counts.values())
print(f"[+] Found {total} video(s) on channel ({len(name_counts)} unique name(s))")
print(
f"[+] Found {total} video(s) on channel ({len(name_counts)} unique name(s))"
)
dupes = {name: count for name, count in name_counts.items() if count > 1}
if dupes:
print(f"[!] {len(dupes)} duplicate name(s) detected on channel:")
for name, count in sorted(dupes.items()):
print(f" x{count} {name}")
else:
print(
"\n[!] No PeerTube credentials given — skipping the channel check; "
"everything below is reported as pending."
)
# ── pre-reconcile: sweep all pending against channel names ────────
# The main upload loop discovers already-uploaded videos lazily as it
@@ -627,20 +684,39 @@ def main() -> None:
# has processed everything. Doing a full sweep here, before any
# upload starts, gives an accurate count up-front and pre-populates
# .uploaded so that interrupted/re-run sessions skip them instantly
# without re-checking each time.
pre_matched = []
for rel in pending:
if _channel_match(rel, path_meta, existing)[0]:
pre_matched.append(rel)
# without re-checking each time. A dry run sweeps but writes nothing.
if existing:
pre_matched = [
rel for rel in pending if _channel_match(rel, path_meta, existing)[0]
]
if pre_matched:
suffix = "" if args.dry_run else " — marking uploaded"
print(
f"\n[+] Pre-sweep: {len(pre_matched)} local file(s) already on channel — marking uploaded"
f"\n[+] Pre-sweep: {len(pre_matched)} local file(s) already on channel{suffix}"
)
if not args.dry_run:
for rel in pre_matched:
mark_uploaded(args.input, rel)
pending = [rel for rel in pending if rel not in set(pre_matched)]
matched = set(pre_matched)
pending = [rel for rel in pending if rel not in matched]
print(f"[+] {len(pending)} left to upload\n")
if not pending:
print("All videos already uploaded.")
return
# ── dry run ──
if args.dry_run:
total_bytes = 0
for rel in pending:
meta = path_meta.get(rel, {})
name = make_pt_name(meta.get("title", ""), rel.name)
sz = (Path(args.input) / rel).stat().st_size
total_bytes += sz
print(f" [{fmt_size(sz):>10}] {name}")
print(f"\n Total: {fmt_size(total_bytes)} across {len(pending)} videos")
return
nsfw = args.nsfw
total_up = 0
batch: list[tuple[str, str]] = [] # [(uuid, name), ...]
@@ -703,4 +779,10 @@ def main() -> None:
if __name__ == "__main__":
try:
main()
except BrokenPipeError:
# Downstream reader closed the pipe (e.g. `| head`). Point stdout at
# devnull so interpreter shutdown does not raise a second time.
os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno())
sys.exit(0)