Admin Replay: re-run the pipeline over past calls in a sandbox (#170)

Correlation has only ever been measured through live AI windows: days of
wall time per change, and the 09-20→22 window was invalidated outright by
unfunded AI accounts (#169). Recordings are kept regardless of AI, so the
traffic to measure against already exists.

- internal/replay.py: runs a time range of real calls through the live
  pipeline code in original order, clock pinned per call, into
  replay_runs/{run_id}/calls|incidents. Modes: audio (re-transcribe),
  transcripts (re-extract), reuse (correlation only from a prior run's
  scenes). Simulates the idle-resolve and orphan-recorrelation sweeps on
  virtual time. No alerts, summaries, vocab, AI-health alerts or pending
  terms. One run at a time, <=5000 calls, <=7 days.
- firestore.py: ContextVar sandbox redirect for calls/incidents.
- clock.py: ContextVar-pinnable now(), used on the correlation path.
- feature_flags.py: ContextVar flag override so replay runs with live AI off.
- upload.py: scene loop extracted to _extract_and_correlate, shared by the
  live pipeline and replay so replay measures the code that runs live.
- resolved_via on every incident resolve, so a real clear can be told
  from the idle timeout — live and in replay.
- routers/replay.py + /admin Replay tab: estimate, start, compare runs,
  drill into incidents with audio.

Reviewed by drb-correlation-review; its leak and fidelity findings are
fixed and covered by tests. c2-core: 456 pass. Frontend typecheck not run
(no Node on the authoring box).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

drb-c2-core/app/internal/firestore.py

@@ -1,5 +1,6 @@
 import asyncio
 import time as _time
+from contextvars import ContextVar
 from typing import Optional, Any
 import firebase_admin
 from firebase_admin import credentials, firestore as fs
@@ -40,23 +41,61 @@ _init_firebase()
 db = fs.client(database_id=settings.firestore_database)
 
 
+# ---------------------------------------------------------------------------
+# Replay sandbox (app/internal/replay.py)
+# ---------------------------------------------------------------------------
+# While a replay run is executing, every read and write the pipeline makes to
+# `calls` or `incidents` is redirected to that run's own subcollections under
+# replay_runs/{run_id}/, so re-running the pipeline over past traffic can never
+# touch a live call or incident. A subcollection keeps the same collection ID
+# ("calls"/"incidents"), so the composite indexes prod queries depend on apply
+# to it unchanged. Everything else (systems, nodes, config) is read from prod
+# as-is. ContextVar for the same reason as app/internal/clock.py: the redirect
+# follows the replay task and never a concurrent live upload.
+SANDBOXED_COLLECTIONS = frozenset({"calls", "incidents"})
+_sandbox_root: ContextVar[Optional[str]] = ContextVar("drb_fstore_sandbox", default=None)
+
+
+def enter_sandbox(root: Optional[str]):
+    """Redirect calls/incidents under `root` (e.g. "replay_runs/<id>") for this context."""
+    return _sandbox_root.set(root)
+
+
+def exit_sandbox(token) -> None:
+    _sandbox_root.reset(token)
+
+
+def in_sandbox() -> bool:
+    """True inside a replay run. Anything that writes live state OTHER than
+    calls/incidents (AI health alerts, pending-term queues) checks this and
+    stands down — the redirect below only covers the two sandboxed collections."""
+    return _sandbox_root.get() is not None
+
+
+def _path(collection: str) -> str:
+    root = _sandbox_root.get()
+    if root and collection in SANDBOXED_COLLECTIONS:
+        return f"{root}/{collection}"
+    return collection
+
+
 # ---------------------------------------------------------------------------
 # Thin async wrappers — firebase-admin is synchronous, run in thread executor
 # ---------------------------------------------------------------------------
 
 async def doc_set(collection: str, doc_id: str, data: dict, merge: bool = True) -> None:
-    ref = db.collection(collection).document(doc_id)
+    ref = db.collection(_path(collection)).document(doc_id)
     await asyncio.to_thread(ref.set, data, merge=merge)
 
 
 async def doc_get(collection: str, doc_id: str) -> Optional[dict]:
-    ref = db.collection(collection).document(doc_id)
+    ref = db.collection(_path(collection)).document(doc_id)
     snap = await asyncio.to_thread(ref.get)
     return snap.to_dict() if snap.exists else None
 
 
 async def doc_update(collection: str, doc_id: str, data: dict) -> None:
-    ref = db.collection(collection).document(doc_id)
+    ref = db.collection(_path(collection)).document(doc_id)
     await asyncio.to_thread(ref.update, data)
 
 
@@ -66,7 +105,7 @@ async def collection_list(collection: str, **filters) -> list[dict]:
     Optional keyword filters: field=value pairs passed as equality where-clauses.
     """
     def _query():
-        ref = db.collection(collection)
+        ref = db.collection(_path(collection))
         for field, value in filters.items():
             ref = ref.where(filter=FieldFilter(field, "==", value))
         return [doc.to_dict() for doc in ref.stream()]
@@ -103,7 +142,7 @@ async def collection_where(
     unscoped equality-only lookups can keep using collection_list().
     """
     def _query():
-        ref = db.collection(collection)
+        ref = db.collection(_path(collection))
         for field, op, value in conditions:
             ref = ref.where(filter=FieldFilter(field, op, value))
         for field, direction in (order_by or []):
@@ -118,7 +157,7 @@ async def collection_where(
 
 
 async def doc_delete(collection: str, doc_id: str) -> None:
-    ref = db.collection(collection).document(doc_id)
+    ref = db.collection(_path(collection)).document(doc_id)
     await asyncio.to_thread(ref.delete)
 
 
@@ -128,7 +167,7 @@ async def doc_get_cached(collection: str, doc_id: str, ttl: float = 300.0) -> Op
     Use for documents that change rarely (systems config, node assignments).
     Default TTL is 5 minutes — a write will be visible within that window.
     """
-    key = f"{collection}/{doc_id}"
+    key = f"{_path(collection)}/{doc_id}"
     now = _time.monotonic()
     entry = _doc_cache.get(key)
     if entry and now < entry[0]: