Better input dir handling

README.md

@@ -89,10 +89,13 @@ Options:
       --poll-interval SECS  State poll interval in seconds (default: 30)
       --skip-wait           Upload without waiting for transcoding
       --nsfw                Mark videos as NSFW
-  -n, --dry-run             Print what would be uploaded
+  -n, --dry-run             Print what would be uploaded (checks the channel, uploads nothing)
+      --scan-all            Scan every subdirectory of --input, not just the site folders
 ```
 
-Uploads in resumable 10 MB chunks. After each batch, waits for transcoding and object storage to complete before uploading the next batch — this prevents disk exhaustion on the PeerTube server. Videos already present on the channel (matched by name) are skipped. Progress is tracked in `.uploaded` inside the input directory.
+Uploads in resumable 10 MB chunks. After each batch, waits for transcoding and object storage to complete before uploading the next batch — this prevents disk exhaustion on the PeerTube server. Videos already present on the channel (matched by name) are skipped. If `--input` contains any of the configured site folders (`jailbirdz`, `pinkcuffs`, `femuniverse`), only those are scanned — so the input directory can be a shared drive holding unrelated purchases alongside them. Anything skipped is listed on startup; `--scan-all` includes it. A directory with no site folders is walked in full.
+
+Progress is tracked in `.uploaded` inside the input directory, so a new `--input` directory starts with an empty ledger — the channel itself is reconciled against on every run (dry runs included) to fill it back in.
 
 ## CI / Nightly Indexing
 

upload.py

@@ -8,7 +8,7 @@ Usage:
     python upload.py                      # upload from ./downloads
     python upload.py -i /mnt/vol/dl      # custom input dir
     python upload.py --batch-size 2      # upload 2, wait, repeat
-    python upload.py --dry-run           # preview without uploading
+    python upload.py --dry-run           # preview (checks the channel, uploads nothing)
     python upload.py --skip-wait         # upload without waiting
 
 Required (CLI flag or env var):
@@ -32,6 +32,7 @@ import requests
 from dotenv import load_dotenv
 
 from check_clashes import fmt_size, url_to_filename, VIDEO_EXTS, load_video_map
+from config import SITES
 from download import (
     collect_urls,
     get_paths_for_mode,
@@ -448,14 +449,43 @@ def build_path_to_meta(
     return result
 
 
-def find_videos(input_dir: str) -> set[Path]:
+def site_dirs_in(input_dir: str) -> set[str]:
-    """Walk input_dir and return a set of relative paths for all video files."""
+    """Names of the per-site subdirectories that actually exist under input_dir."""
+    root = Path(input_dir)
+    if not root.is_dir():
+        return set()
+    return {site for site in SITES if (root / site).is_dir()}
+
+
+def unmanaged_dirs_in(input_dir: str, site_dirs: set[str]) -> list[str]:
+    """Top-level subdirectories of input_dir that belong to no configured site."""
+    root = Path(input_dir)
+    if not root.is_dir():
+        return []
+    return sorted(
+        d.name
+        for d in root.iterdir()
+        if d.is_dir() and not d.name.startswith(".") and d.name not in site_dirs
+    )
+
+
+def find_videos(input_dir: str, only_dirs: set[str] | None = None) -> set[Path]:
+    """Walk input_dir and return a set of relative paths for all video files.
+
+    When *only_dirs* is given, just those top-level directories are descended
+    into; loose files sitting directly in input_dir are always included.  That
+    keeps a shared parent directory — an external drive holding other purchases
+    next to the site folders — from dragging in unrelated albums.
+    """
+    top = Path(input_dir)
     found = set()
-    for root, dirs, files in os.walk(input_dir):
+    for cur, dirs, files in os.walk(str(top)):
         dirs[:] = [d for d in dirs if not d.startswith(".")]
+        if only_dirs is not None and Path(cur) == top:
+            dirs[:] = [d for d in dirs if d in only_dirs]
         for f in files:
             if Path(f).suffix.lower() in VIDEO_EXTS:
-                found.add((Path(root) / f).relative_to(input_dir))
+                found.add((Path(cur) / f).relative_to(top))
     return found
 
 
@@ -532,6 +562,11 @@ def main() -> None:
     ap.add_argument(
         "--dry-run", "-n", action="store_true", help="Preview what would be uploaded"
     )
+    ap.add_argument(
+        "--scan-all",
+        action="store_true",
+        help="Scan every subdirectory of --input, not just the configured site folders",
+    )
     args = ap.parse_args()
 
     url = args.url or os.environ.get("PEERTUBE_URL")
@@ -539,7 +574,6 @@ def main() -> None:
     channel = args.channel or os.environ.get("PEERTUBE_CHANNEL")
     password = args.password or os.environ.get("PEERTUBE_PASSWORD")
 
-    if not args.dry_run:
     missing = [
         label
         for label, val in [
@@ -550,7 +584,8 @@ def main() -> None:
         ]
         if not val
     ]
-        if missing:
+    have_creds = not missing
+    if missing and not args.dry_run:
         for label in missing:
             print(f"[!] Required: {label}")
         sys.exit(1)
@@ -558,6 +593,22 @@ def main() -> None:
     # ── load metadata & scan disk ──
     video_map = load_video_map()
     path_meta = build_path_to_meta(video_map, args.input)
+
+    # --input may be a shared directory that holds unrelated purchases beside
+    # the per-site folders.  Once any site folder is present, treat the layout
+    # as site-partitioned and ignore everything else; a flat directory (no site
+    # folders at all) is still walked in full.
+    site_dirs = set() if args.scan_all else site_dirs_in(args.input)
+    if site_dirs:
+        skipped = unmanaged_dirs_in(args.input, site_dirs)
+        if skipped:
+            print(
+                f"[!] Ignoring {len(skipped)} unmanaged folder(s) in {args.input}: "
+                f"{', '.join(skipped)}"
+            )
+            print("    (pass --scan-all to upload those too)")
+        on_disk = find_videos(args.input, site_dirs)
+    else:
         on_disk = find_videos(args.input)
 
     unmatched = on_disk - set(path_meta.keys())
@@ -568,11 +619,12 @@ def main() -> None:
         for rel in unmatched:
             path_meta[rel] = {"title": "", "description": ""}
 
+    ledger = Path(args.input) / UPLOADED_FILE
     uploaded = load_uploaded(args.input)
     pending = sorted(rel for rel in on_disk if rel not in uploaded)
 
     print(f"[+] {len(on_disk)} video files in {args.input}/")
-    print(f"[+] {len(uploaded)} already uploaded")
+    print(f"[+] {len(uploaded)} already uploaded (per {ledger})")
     print(f"[+] {len(pending)} pending")
     print(f"[+] Batch size: {args.batch_size}")
 
@@ -580,24 +632,22 @@ def main() -> None:
         print("\nAll videos already uploaded.")
         return
 
-    # ── dry run ──
+    # ── connect to the channel ────────────────────────────────────────
-    if args.dry_run:
+    # The .uploaded ledger lives inside --input, so pointing at a fresh
-        total_bytes = 0
+    # directory (an external drive, say) starts from an empty one.  The channel
-        for rel in pending:
+    # is the real source of truth, so reconcile against it before deciding what
-            meta = path_meta.get(rel, {})
+    # is pending — dry runs included, whenever credentials are available.
-            name = make_pt_name(meta.get("title", ""), rel.name)
+    base = ""
-            sz = (Path(args.input) / rel).stat().st_size
+    token = ""
-            total_bytes += sz
+    channel_id = 0
-            print(f"  [{fmt_size(sz):>10}]  {name}")
+    existing: set[str] = set()
-        print(f"\n  Total: {fmt_size(total_bytes)} across {len(pending)} videos")
-        return
 
+    if have_creds:
         assert url is not None
         assert username is not None
         assert channel is not None
         assert password is not None
 
-    # ── authenticate ──
         base = url.rstrip("/")
         if not base.startswith("http"):
             base = "https://" + base
@@ -612,13 +662,20 @@ def main() -> None:
         name_counts = get_channel_video_names(base, token, channel)
         existing = set(name_counts)
         total = sum(name_counts.values())
-    print(f"[+] Found {total} video(s) on channel ({len(name_counts)} unique name(s))")
+        print(
+            f"[+] Found {total} video(s) on channel ({len(name_counts)} unique name(s))"
+        )
 
         dupes = {name: count for name, count in name_counts.items() if count > 1}
         if dupes:
             print(f"[!] {len(dupes)} duplicate name(s) detected on channel:")
             for name, count in sorted(dupes.items()):
                 print(f"    x{count}  {name}")
+    else:
+        print(
+            "\n[!] No PeerTube credentials given — skipping the channel check; "
+            "everything below is reported as pending."
+        )
 
     # ── pre-reconcile: sweep all pending against channel names ────────
     # The main upload loop discovers already-uploaded videos lazily as it
@@ -627,20 +684,39 @@ def main() -> None:
     # has processed everything.  Doing a full sweep here, before any
     # upload starts, gives an accurate count up-front and pre-populates
     # .uploaded so that interrupted/re-run sessions skip them instantly
-    # without re-checking each time.
+    # without re-checking each time.  A dry run sweeps but writes nothing.
-    pre_matched = []
+    if existing:
-    for rel in pending:
+        pre_matched = [
-        if _channel_match(rel, path_meta, existing)[0]:
+            rel for rel in pending if _channel_match(rel, path_meta, existing)[0]
-            pre_matched.append(rel)
+        ]
         if pre_matched:
+            suffix = "" if args.dry_run else " — marking uploaded"
             print(
-            f"\n[+] Pre-sweep: {len(pre_matched)} local file(s) already on channel — marking uploaded"
+                f"\n[+] Pre-sweep: {len(pre_matched)} local file(s) already on channel{suffix}"
             )
+            if not args.dry_run:
                 for rel in pre_matched:
                     mark_uploaded(args.input, rel)
-        pending = [rel for rel in pending if rel not in set(pre_matched)]
+            matched = set(pre_matched)
+            pending = [rel for rel in pending if rel not in matched]
             print(f"[+] {len(pending)} left to upload\n")
 
+    if not pending:
+        print("All videos already uploaded.")
+        return
+
+    # ── dry run ──
+    if args.dry_run:
+        total_bytes = 0
+        for rel in pending:
+            meta = path_meta.get(rel, {})
+            name = make_pt_name(meta.get("title", ""), rel.name)
+            sz = (Path(args.input) / rel).stat().st_size
+            total_bytes += sz
+            print(f"  [{fmt_size(sz):>10}]  {name}")
+        print(f"\n  Total: {fmt_size(total_bytes)} across {len(pending)} videos")
+        return
+
     nsfw = args.nsfw
     total_up = 0
     batch: list[tuple[str, str]] = []  # [(uuid, name), ...]
@@ -703,4 +779,10 @@ def main() -> None:
 
 
 if __name__ == "__main__":
+    try:
         main()
+    except BrokenPipeError:
+        # Downstream reader closed the pipe (e.g. `| head`).  Point stdout at
+        # devnull so interpreter shutdown does not raise a second time.
+        os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno())
+        sys.exit(0)