Admin Replay: re-run the pipeline over past calls in a sandbox (#170)

Correlation has only ever been measured through live AI windows: days of
wall time per change, and the 09-20→22 window was invalidated outright by
unfunded AI accounts (#169). Recordings are kept regardless of AI, so the
traffic to measure against already exists.

- internal/replay.py: runs a time range of real calls through the live
  pipeline code in original order, clock pinned per call, into
  replay_runs/{run_id}/calls|incidents. Modes: audio (re-transcribe),
  transcripts (re-extract), reuse (correlation only from a prior run's
  scenes). Simulates the idle-resolve and orphan-recorrelation sweeps on
  virtual time. No alerts, summaries, vocab, AI-health alerts or pending
  terms. One run at a time, <=5000 calls, <=7 days.
- firestore.py: ContextVar sandbox redirect for calls/incidents.
- clock.py: ContextVar-pinnable now(), used on the correlation path.
- feature_flags.py: ContextVar flag override so replay runs with live AI off.
- upload.py: scene loop extracted to _extract_and_correlate, shared by the
  live pipeline and replay so replay measures the code that runs live.
- resolved_via on every incident resolve, so a real clear can be told
  from the idle timeout — live and in replay.
- routers/replay.py + /admin Replay tab: estimate, start, compare runs,
  drill into incidents with audio.

Reviewed by drb-correlation-review; its leak and fidelity findings are
fixed and covered by tests. c2-core: 456 pass. Frontend typecheck not run
(no Node on the authoring box).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

drb-c2-core/app/routers/replay.py

@@ -0,0 +1,181 @@
+"""
+Admin replay routes — the backend for the /admin Replay tab.
+
+See app/internal/replay.py for what a run is and why it exists. Every route is
+admin-only: a run spends real AI credits.
+"""
+from datetime import datetime, timezone
+from typing import Literal, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from pydantic import BaseModel
+
+from app.internal import replay
+from app.internal.audit import write_audit
+from app.internal.auth import describe_actor, require_admin_token, resolve_caller_org_id
+from app.internal.logger import logger
+
+router = APIRouter(prefix="/admin/replay", tags=["admin"])
+
+
+def _parse_ts(value: Optional[str], field: str) -> datetime:
+    if not value:
+        raise HTTPException(400, f"{field} is required.")
+    try:
+        dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError:
+        raise HTTPException(400, f"{field} is not an ISO-8601 timestamp.")
+    return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
+
+
+async def _org(decoded: dict) -> str:
+    # Same fallback as /calls/search: a platform admin resolves to "every
+    # org", which is not a scope a replay can run in.
+    org_id = await resolve_caller_org_id(decoded) or decoded.get("org_id")
+    if not org_id:
+        raise HTTPException(403, "No organization scope for this caller.")
+    return org_id
+
+
+async def _own_run(run_id: str, org_id: str) -> dict:
+    run = await replay.get_run(run_id)
+    if not run or run.get("org_id") != org_id:
+        raise HTTPException(404, f"Replay run '{run_id}' not found.")
+    return run
+
+
+@router.get("/estimate")
+async def estimate_run(
+    date_from: str = Query(...),
+    date_to: str = Query(...),
+    mode: Literal["audio", "transcripts", "reuse"] = Query("transcripts"),
+    system_ids: Optional[str] = Query(None, description="comma-separated"),
+    decoded: dict = Depends(require_admin_token),
+):
+    """How many calls a run over this range would process, and a rough cost."""
+    org_id = await _org(decoded)
+    sids = [s for s in (system_ids or "").split(",") if s] or None
+    calls, truncated = await replay.select_calls(
+        org_id, _parse_ts(date_from, "date_from"), _parse_ts(date_to, "date_to"), sids,
+    )
+    return {**replay.estimate(calls, mode), "truncated": truncated, "max_calls": replay.MAX_CALLS}
+
+
+class StartRun(BaseModel):
+    date_from: str
+    date_to: str
+    mode: Literal["audio", "transcripts", "reuse"] = "transcripts"
+    system_ids: Optional[list[str]] = None
+    source_run_id: Optional[str] = None
+    label: str = ""
+
+
+@router.post("")
+async def start_run(body: StartRun, decoded: dict = Depends(require_admin_token)):
+    org_id = await _org(decoded)
+    actor_uid, actor_email = describe_actor(decoded)
+    try:
+        run = await replay.start_run(
+            org_id=org_id,
+            date_from=_parse_ts(body.date_from, "date_from"),
+            date_to=_parse_ts(body.date_to, "date_to"),
+            mode=body.mode,
+            system_ids=body.system_ids or None,
+            source_run_id=body.source_run_id,
+            label=body.label[:120],
+            actor=actor_email or actor_uid,
+        )
+    except replay.ReplayBusy as e:
+        raise HTTPException(409, str(e))
+    except ValueError as e:
+        raise HTTPException(400, str(e))
+    try:
+        await write_audit(actor_uid, actor_email, "replay.start", details={
+            "run_id": run["run_id"], "mode": run["mode"], "calls": run["progress"]["total"],
+            "est_cost_usd": run["estimate"]["est_cost_usd"],
+        })
+    except Exception as e:
+        logger.error(f"Replay: audit write failed ({e}) — run {run['run_id']} continues")
+    return run
+
+
+@router.get("")
+async def list_runs(decoded: dict = Depends(require_admin_token)):
+    org_id = await _org(decoded)
+    return {"runs": await replay.list_runs(org_id), "active_run_id": replay.active_run_id()}
+
+
+@router.get("/{run_id}")
+async def get_run(run_id: str, decoded: dict = Depends(require_admin_token)):
+    return await _own_run(run_id, await _org(decoded))
+
+
+@router.post("/{run_id}/cancel")
+async def cancel_run(run_id: str, decoded: dict = Depends(require_admin_token)):
+    await _own_run(run_id, await _org(decoded))
+    if not replay.request_cancel(run_id):
+        raise HTTPException(409, "That run is not running.")
+    return {"ok": True}
+
+
+@router.delete("/{run_id}")
+async def delete_run(run_id: str, decoded: dict = Depends(require_admin_token)):
+    await _own_run(run_id, await _org(decoded))
+    try:
+        await replay.delete_run(run_id)
+    except replay.ReplayBusy as e:
+        raise HTTPException(409, str(e))
+    return {"ok": True}
+
+
+def _call_row(c: dict) -> dict:
+    scenes = c.get("scenes") or {}
+    paths = [((s.get("corr_debug") or {}).get("corr_path")) for _, s in sorted(scenes.items())]
+    return {
+        "call_id":        c.get("call_id"),
+        "started_at":     c.get("started_at"),
+        "talkgroup_name": c.get("talkgroup_name"),
+        "transcript":     c.get("transcript_corrected") or c.get("transcript"),
+        "units":          c.get("units"),
+        "cleared_units":  c.get("cleared_units"),
+        "location":       c.get("location"),
+        "skip_reason":    c.get("skip_reason"),
+        "corr_path":      [p for p in paths if p] or ([c["corr_path"]] if c.get("corr_path") else []),
+        "incident_ids":   c.get("incident_ids") or [],
+    }
+
+
+@router.get("/{run_id}/incidents")
+async def run_incidents(run_id: str, decoded: dict = Depends(require_admin_token)):
+    """
+    A run's sandbox, shaped for reading: every incident with its calls in
+    order, plus the calls that never linked. Embeddings stay out.
+    """
+    await _own_run(run_id, await _org(decoded))
+    incidents, calls = await replay.sandbox_contents(run_id)
+    by_id = {c.get("call_id"): _call_row(c) for c in calls}
+    out = []
+    for inc in sorted(incidents, key=lambda i: str(i.get("started_at") or "")):
+        rows = [by_id[cid] for cid in (inc.get("call_ids") or []) if cid in by_id]
+        rows.sort(key=lambda r: str(r["started_at"] or ""))
+        out.append({
+            "incident_id":   inc.get("incident_id"),
+            "title":         inc.get("title"),
+            "type":          inc.get("type"),
+            "severity":      inc.get("severity"),
+            "status":        inc.get("status"),
+            "resolved_via":  inc.get("resolved_via"),
+            "started_at":    inc.get("started_at"),
+            "updated_at":    inc.get("updated_at"),
+            "resolved_at":   inc.get("resolved_at"),
+            "location":      inc.get("location"),
+            "location_coords": inc.get("location_coords"),
+            "units":         inc.get("units"),
+            "units_active":  inc.get("units_active"),
+            "units_cleared": inc.get("units_cleared"),
+            "talkgroup_ids": inc.get("talkgroup_ids"),
+            "calls":         rows,
+        })
+    orphans = sorted((r for r in by_id.values() if not r["incident_ids"]),
+                     key=lambda r: str(r["started_at"] or ""))
+    return {"incidents": out, "orphans": orphans}