From 1d83988811ffdeab26a1051c2c3d80390ca28911 Mon Sep 17 00:00:00 2001 From: HugeFrog24 <62775760+HugeFrog24@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:16:43 +0200 Subject: [PATCH] Better input dir handling --- README.md | 7 +- upload.py | 224 +++++++++++++++++++++++++++++++++++++----------------- 2 files changed, 158 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index 7b9b17c..727746d 100644 --- a/README.md +++ b/README.md @@ -89,10 +89,13 @@ Options: --poll-interval SECS State poll interval in seconds (default: 30) --skip-wait Upload without waiting for transcoding --nsfw Mark videos as NSFW - -n, --dry-run Print what would be uploaded + -n, --dry-run Print what would be uploaded (checks the channel, uploads nothing) + --scan-all Scan every subdirectory of --input, not just the site folders ``` -Uploads in resumable 10 MB chunks. After each batch, waits for transcoding and object storage to complete before uploading the next batch — this prevents disk exhaustion on the PeerTube server. Videos already present on the channel (matched by name) are skipped. Progress is tracked in `.uploaded` inside the input directory. +Uploads in resumable 10 MB chunks. After each batch, waits for transcoding and object storage to complete before uploading the next batch — this prevents disk exhaustion on the PeerTube server. Videos already present on the channel (matched by name) are skipped. If `--input` contains any of the configured site folders (`jailbirdz`, `pinkcuffs`, `femuniverse`), only those are scanned — so the input directory can be a shared drive holding unrelated purchases alongside them. Anything skipped is listed on startup; `--scan-all` includes it. A directory with no site folders is walked in full. + +Progress is tracked in `.uploaded` inside the input directory, so a new `--input` directory starts with an empty ledger — the channel itself is reconciled against on every run (dry runs included) to fill it back in. ## CI / Nightly Indexing diff --git a/upload.py b/upload.py index b527278..f5ee0df 100644 --- a/upload.py +++ b/upload.py @@ -8,7 +8,7 @@ Usage: python upload.py # upload from ./downloads python upload.py -i /mnt/vol/dl # custom input dir python upload.py --batch-size 2 # upload 2, wait, repeat - python upload.py --dry-run # preview without uploading + python upload.py --dry-run # preview (checks the channel, uploads nothing) python upload.py --skip-wait # upload without waiting Required (CLI flag or env var): @@ -32,6 +32,7 @@ import requests from dotenv import load_dotenv from check_clashes import fmt_size, url_to_filename, VIDEO_EXTS, load_video_map +from config import SITES from download import ( collect_urls, get_paths_for_mode, @@ -448,14 +449,43 @@ def build_path_to_meta( return result -def find_videos(input_dir: str) -> set[Path]: - """Walk input_dir and return a set of relative paths for all video files.""" +def site_dirs_in(input_dir: str) -> set[str]: + """Names of the per-site subdirectories that actually exist under input_dir.""" + root = Path(input_dir) + if not root.is_dir(): + return set() + return {site for site in SITES if (root / site).is_dir()} + + +def unmanaged_dirs_in(input_dir: str, site_dirs: set[str]) -> list[str]: + """Top-level subdirectories of input_dir that belong to no configured site.""" + root = Path(input_dir) + if not root.is_dir(): + return [] + return sorted( + d.name + for d in root.iterdir() + if d.is_dir() and not d.name.startswith(".") and d.name not in site_dirs + ) + + +def find_videos(input_dir: str, only_dirs: set[str] | None = None) -> set[Path]: + """Walk input_dir and return a set of relative paths for all video files. + + When *only_dirs* is given, just those top-level directories are descended + into; loose files sitting directly in input_dir are always included. That + keeps a shared parent directory — an external drive holding other purchases + next to the site folders — from dragging in unrelated albums. + """ + top = Path(input_dir) found = set() - for root, dirs, files in os.walk(input_dir): + for cur, dirs, files in os.walk(str(top)): dirs[:] = [d for d in dirs if not d.startswith(".")] + if only_dirs is not None and Path(cur) == top: + dirs[:] = [d for d in dirs if d in only_dirs] for f in files: if Path(f).suffix.lower() in VIDEO_EXTS: - found.add((Path(root) / f).relative_to(input_dir)) + found.add((Path(cur) / f).relative_to(top)) return found @@ -532,6 +562,11 @@ def main() -> None: ap.add_argument( "--dry-run", "-n", action="store_true", help="Preview what would be uploaded" ) + ap.add_argument( + "--scan-all", + action="store_true", + help="Scan every subdirectory of --input, not just the configured site folders", + ) args = ap.parse_args() url = args.url or os.environ.get("PEERTUBE_URL") @@ -539,26 +574,42 @@ def main() -> None: channel = args.channel or os.environ.get("PEERTUBE_CHANNEL") password = args.password or os.environ.get("PEERTUBE_PASSWORD") - if not args.dry_run: - missing = [ - label - for label, val in [ - ("--url / PEERTUBE_URL", url), - ("--username / PEERTUBE_USER", username), - ("--channel / PEERTUBE_CHANNEL", channel), - ("--password / PEERTUBE_PASSWORD", password), - ] - if not val + missing = [ + label + for label, val in [ + ("--url / PEERTUBE_URL", url), + ("--username / PEERTUBE_USER", username), + ("--channel / PEERTUBE_CHANNEL", channel), + ("--password / PEERTUBE_PASSWORD", password), ] - if missing: - for label in missing: - print(f"[!] Required: {label}") - sys.exit(1) + if not val + ] + have_creds = not missing + if missing and not args.dry_run: + for label in missing: + print(f"[!] Required: {label}") + sys.exit(1) # ── load metadata & scan disk ── video_map = load_video_map() path_meta = build_path_to_meta(video_map, args.input) - on_disk = find_videos(args.input) + + # --input may be a shared directory that holds unrelated purchases beside + # the per-site folders. Once any site folder is present, treat the layout + # as site-partitioned and ignore everything else; a flat directory (no site + # folders at all) is still walked in full. + site_dirs = set() if args.scan_all else site_dirs_in(args.input) + if site_dirs: + skipped = unmanaged_dirs_in(args.input, site_dirs) + if skipped: + print( + f"[!] Ignoring {len(skipped)} unmanaged folder(s) in {args.input}: " + f"{', '.join(skipped)}" + ) + print(" (pass --scan-all to upload those too)") + on_disk = find_videos(args.input, site_dirs) + else: + on_disk = find_videos(args.input) unmatched = on_disk - set(path_meta.keys()) if unmatched: @@ -568,11 +619,12 @@ def main() -> None: for rel in unmatched: path_meta[rel] = {"title": "", "description": ""} + ledger = Path(args.input) / UPLOADED_FILE uploaded = load_uploaded(args.input) pending = sorted(rel for rel in on_disk if rel not in uploaded) print(f"[+] {len(on_disk)} video files in {args.input}/") - print(f"[+] {len(uploaded)} already uploaded") + print(f"[+] {len(uploaded)} already uploaded (per {ledger})") print(f"[+] {len(pending)} pending") print(f"[+] Batch size: {args.batch_size}") @@ -580,6 +632,79 @@ def main() -> None: print("\nAll videos already uploaded.") return + # ── connect to the channel ──────────────────────────────────────── + # The .uploaded ledger lives inside --input, so pointing at a fresh + # directory (an external drive, say) starts from an empty one. The channel + # is the real source of truth, so reconcile against it before deciding what + # is pending — dry runs included, whenever credentials are available. + base = "" + token = "" + channel_id = 0 + existing: set[str] = set() + + if have_creds: + assert url is not None + assert username is not None + assert channel is not None + assert password is not None + + base = url.rstrip("/") + if not base.startswith("http"): + base = "https://" + base + + print(f"\n[+] Authenticating with {base} ...") + token = get_oauth_token(base, username, password) + print(f"[+] Authenticated as {username}") + + channel_id = get_channel_id(base, token, channel) + print(f"[+] Channel: {channel} (id {channel_id})") + + name_counts = get_channel_video_names(base, token, channel) + existing = set(name_counts) + total = sum(name_counts.values()) + print( + f"[+] Found {total} video(s) on channel ({len(name_counts)} unique name(s))" + ) + + dupes = {name: count for name, count in name_counts.items() if count > 1} + if dupes: + print(f"[!] {len(dupes)} duplicate name(s) detected on channel:") + for name, count in sorted(dupes.items()): + print(f" x{count} {name}") + else: + print( + "\n[!] No PeerTube credentials given — skipping the channel check; " + "everything below is reported as pending." + ) + + # ── pre-reconcile: sweep all pending against channel names ──────── + # The main upload loop discovers already-uploaded videos lazily as it + # walks the sorted pending list — meaning on a fresh run (no .uploaded + # file) you won't know how many files are genuinely new until the loop + # has processed everything. Doing a full sweep here, before any + # upload starts, gives an accurate count up-front and pre-populates + # .uploaded so that interrupted/re-run sessions skip them instantly + # without re-checking each time. A dry run sweeps but writes nothing. + if existing: + pre_matched = [ + rel for rel in pending if _channel_match(rel, path_meta, existing)[0] + ] + if pre_matched: + suffix = "" if args.dry_run else " — marking uploaded" + print( + f"\n[+] Pre-sweep: {len(pre_matched)} local file(s) already on channel{suffix}" + ) + if not args.dry_run: + for rel in pre_matched: + mark_uploaded(args.input, rel) + matched = set(pre_matched) + pending = [rel for rel in pending if rel not in matched] + print(f"[+] {len(pending)} left to upload\n") + + if not pending: + print("All videos already uploaded.") + return + # ── dry run ── if args.dry_run: total_bytes = 0 @@ -592,55 +717,6 @@ def main() -> None: print(f"\n Total: {fmt_size(total_bytes)} across {len(pending)} videos") return - assert url is not None - assert username is not None - assert channel is not None - assert password is not None - - # ── authenticate ── - base = url.rstrip("/") - if not base.startswith("http"): - base = "https://" + base - - print(f"\n[+] Authenticating with {base} ...") - token = get_oauth_token(base, username, password) - print(f"[+] Authenticated as {username}") - - channel_id = get_channel_id(base, token, channel) - print(f"[+] Channel: {channel} (id {channel_id})") - - name_counts = get_channel_video_names(base, token, channel) - existing = set(name_counts) - total = sum(name_counts.values()) - print(f"[+] Found {total} video(s) on channel ({len(name_counts)} unique name(s))") - - dupes = {name: count for name, count in name_counts.items() if count > 1} - if dupes: - print(f"[!] {len(dupes)} duplicate name(s) detected on channel:") - for name, count in sorted(dupes.items()): - print(f" x{count} {name}") - - # ── pre-reconcile: sweep all pending against channel names ──────── - # The main upload loop discovers already-uploaded videos lazily as it - # walks the sorted pending list — meaning on a fresh run (no .uploaded - # file) you won't know how many files are genuinely new until the loop - # has processed everything. Doing a full sweep here, before any - # upload starts, gives an accurate count up-front and pre-populates - # .uploaded so that interrupted/re-run sessions skip them instantly - # without re-checking each time. - pre_matched = [] - for rel in pending: - if _channel_match(rel, path_meta, existing)[0]: - pre_matched.append(rel) - if pre_matched: - print( - f"\n[+] Pre-sweep: {len(pre_matched)} local file(s) already on channel — marking uploaded" - ) - for rel in pre_matched: - mark_uploaded(args.input, rel) - pending = [rel for rel in pending if rel not in set(pre_matched)] - print(f"[+] {len(pending)} left to upload\n") - nsfw = args.nsfw total_up = 0 batch: list[tuple[str, str]] = [] # [(uuid, name), ...] @@ -703,4 +779,10 @@ def main() -> None: if __name__ == "__main__": - main() + try: + main() + except BrokenPipeError: + # Downstream reader closed the pipe (e.g. `| head`). Point stdout at + # devnull so interpreter shutdown does not raise a second time. + os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno()) + sys.exit(0)