chore: nightly index update [skip ci]

README.md

@@ -128,10 +128,11 @@ Lists filenames that map to more than one source URL, with sizes.
 ### Estimate total download size
 
 ```bash
-python total_size.py
+python total_size.py           # read cached sizes and print summary
+python total_size.py --write   # probe uncached/stale URLs and refresh the cache
 ```
 
-Fetches `Content-Length` for every video URL in `video_map.json` and prints a size summary. Does not download anything.
+Reads cached file sizes from `video_map.json` and prints a summary (total, smallest, largest, average). The default mode never hits the network. Use `--write` to probe any missing or stale entries and persist the results.
 
 ## Data files
 

download.py

@@ -18,6 +18,7 @@ from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Any
 import requests
+import time
 
 from check_clashes import (
     make_session,
@@ -28,6 +29,7 @@ from check_clashes import (
     build_url_referers,
     fetch_sizes,
     load_video_map,
+    save_video_map,
     is_valid_url,
     VIDEO_MAP_FILE,
 )
@@ -288,6 +290,25 @@ def build_url_title_map(video_map: dict[str, Any]) -> dict[str, str]:
     return url_title
 
 
+def _persist_fetched_sizes(newly_fetched: dict[str, int | None]) -> None:
+    """Write newly probed sizes back to video_map.json (successful probes only)."""
+    now = int(time.time())
+    for site_key in SITES:
+        vm_site = load_video_map(site_key)
+        changed = False
+        for entry in vm_site.values():
+            for vid in entry.get("videos", []):
+                if vid["url"] in newly_fetched and vid.get("size") is None and newly_fetched[vid["url"]] is not None:
+                    vid["size"] = newly_fetched[vid["url"]]
+                    vid["size_checked_at"] = now
+                    changed = True
+        if changed:
+            save_video_map(vm_site, site_key)
+    n_saved = sum(1 for s in newly_fetched.values() if s is not None)
+    if n_saved:
+        print(f"[+] Cached {n_saved} newly probed size(s).")
+
+
 def build_url_to_site() -> dict[str, str]:
     """Return {cdn_video_url: site_key} by loading each site's map in turn."""
     result: dict[str, str] = {}
@@ -423,16 +444,16 @@ def main() -> None:
         if vid.get("size") is not None
     }
 
+    newly_fetched: dict[str, int | None] = {}
     uncached_pending = [u for u in pending if u not in cached_sizes]
     session = make_session()
     if uncached_pending:
         print(
             f"\n[+] Fetching remote file sizes ({len(uncached_pending)} uncached, {len(pending) - len(uncached_pending)} cached)…"
         )
-        remote_sizes: dict[str, int | None] = {
+        fetched_pending = fetch_sizes(uncached_pending, workers=20, url_referers=url_referers)
-            **cached_sizes,
+        newly_fetched.update(fetched_pending)
-            **fetch_sizes(uncached_pending, workers=20, url_referers=url_referers),
+        remote_sizes: dict[str, int | None] = {**cached_sizes, **fetched_pending}
-        }
     else:
         print(f"\n[+] All {len(pending)} pending sizes cached — skipping probe.")
         remote_sizes = dict(cached_sizes)
@@ -447,10 +468,9 @@ def main() -> None:
             print(
                 f"[+] Verifying {len(already)} existing files ({len(uncached_already)} uncached)…"
             )
-            already_sizes: dict[str, int | None] = {
+            fetched_already = fetch_sizes(uncached_already, workers=20, url_referers=url_referers)
-                **cached_sizes,
+            newly_fetched.update(fetched_already)
-                **fetch_sizes(uncached_already, workers=20, url_referers=url_referers),
+            already_sizes: dict[str, int | None] = {**cached_sizes, **fetched_already}
-            }
         else:
             print(f"[+] Verifying {len(already)} existing files (all sizes cached)…")
             already_sizes = dict(cached_sizes)
@@ -472,6 +492,9 @@ def main() -> None:
     if mismatched:
         print(f"[!] {mismatched} file(s) will be re-downloaded due to size mismatch")
 
+    if newly_fetched:
+        _persist_fetched_sizes(newly_fetched)
+
     print(f"\n[⚡] Downloading with {args.workers} threads…\n")
 
     completed = 0

grab_cookie.py

@@ -14,10 +14,12 @@ import os
 from pathlib import Path
 from typing import Literal
 import requests
+from dotenv import load_dotenv
 from config import SITES
 
 ENV_FILE = Path(".env")
 COOKIE_PREFIX = "wordpress_logged_in_"
+load_dotenv(dotenv_path=ENV_FILE)
 
 
 def update_env(
@@ -72,6 +74,9 @@ def login_and_get_cookie(
             "Referer": f"{base_url}/",
             "Origin": base_url,
             "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:147.0) Gecko/20100101 Firefox/147.0",
+            "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+            "X-Requested-With": "XMLHttpRequest",
+            "Accept": "*/*",
         },
         timeout=30,
     )

total_size.py

@@ -59,7 +59,7 @@ def _is_stale(vid: dict[str, Any], now: int) -> bool:
     """True if the cached size is absent or older than SIZE_CACHE_TTL seconds."""
     if vid.get("size") is None:
         return True
-    return (now - vid.get("size_checked_at", 0)) >= SIZE_CACHE_TTL
+    return (now - int(vid.get("size_checked_at", 0))) >= SIZE_CACHE_TTL
 
 
 # --------------- CLI ---------------

video_map.json

@@ -16416,6 +16416,16 @@
         }
       ],
       "scraped_at": 1772204332
+    },
+    "https://www.jailbirdz.com/pinkcuffs-videos/luvv-gingers-jail-escape-full-recapture-pt1/": {
+      "title": "LUVV & Ginger’s Jail Escape – Full Recapture pt1",
+      "description": "LUVV and Ginger begin the video nude and restrained inside a cell, both secured in hinge handcuffs and shackles. The brunette, LUVV, nervously complains about the restraints while Ginger tries to stay calm. During a surprise cell inspection, Officer Luna Lux orders them to stand against the wall and conducts a detailed search, checking the bed, mattress, and every corner of the cell. While Luna is distracted, Ginger quietly swipes the key without any prior planning.\n\nAfter the officer finishes and leaves, Ginger reveals the key. The two quickly work together to unlock the hinge cuffs, then remove the shackles, freeing themselves completely. Finding the cell unsecured, they seize the opportunity, throw on dark green prison jumpsuits, and slip away. When Luna returns and discovers the empty cell, she immediately realizes what happened and rushes out to find them.\n\nThe scene shifts to the pair sitting outside at a table, visibly shaken and unsure of their next move. Without phones, proper clothes, or a safe place to go, panic starts to set in as they debate what to do. Their brief taste of freedom ends quickly when Luna tracks them down. She commands them to put their hands on their heads, cross their ankles, and lie flat on the ground before securing both behind their backs in handcuffs once again. After reading them their rights, she warns them that their attempted escape will only add more time to their charges, bringing their failed breakout to a firm and controlled conclusion.",
+      "videos": [
+        {
+          "url": "https://d1nqppmcfahrr7.cloudfront.net/2026/LunaLux/LUVVGingerJailEscapept1.mp4"
+        }
+      ],
+      "scraped_at": 1772426905
     }
   },
   "pinkcuffs": {