logan/server-26
1
0
Fork 0
Code Issues 59 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Admin Replay: re-run the pipeline over past calls in a sandbox (#170)

Browse Source 
Correlation has only ever been measured through live AI windows: days of
wall time per change, and the 09-20→22 window was invalidated outright by
unfunded AI accounts (#169). Recordings are kept regardless of AI, so the
traffic to measure against already exists.

- internal/replay.py: runs a time range of real calls through the live
  pipeline code in original order, clock pinned per call, into
  replay_runs/{run_id}/calls|incidents. Modes: audio (re-transcribe),
  transcripts (re-extract), reuse (correlation only from a prior run's
  scenes). Simulates the idle-resolve and orphan-recorrelation sweeps on
  virtual time. No alerts, summaries, vocab, AI-health alerts or pending
  terms. One run at a time, <=5000 calls, <=7 days.
- firestore.py: ContextVar sandbox redirect for calls/incidents.
- clock.py: ContextVar-pinnable now(), used on the correlation path.
- feature_flags.py: ContextVar flag override so replay runs with live AI off.
- upload.py: scene loop extracted to _extract_and_correlate, shared by the
  live pipeline and replay so replay measures the code that runs live.
- resolved_via on every incident resolve, so a real clear can be told
  from the idle timeout — live and in replay.
- routers/replay.py + /admin Replay tab: estimate, start, compare runs,
  drill into incidents with audio.

Reviewed by drb-correlation-review; its leak and fidelity findings are
fixed and covered by tests. c2-core: 456 pass. Frontend typecheck not run
(no Node on the authoring box).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Logan Cusano
2026-09-26 15:23:57 -04:00
co-authored by Claude Opus 5.5
parent e79b8bc37d
commit aff3f16d32
19 changed files with 1976 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
drb-c2-core/app/routers/calls.py
+1
View File
@@ -443,6 +443,7 @@ async def patch_transcript(
"call_ids": [],
"status": "resolved",
"resolved_at": datetime.now(timezone.utc).isoformat(),
"resolved_via": "emptied_by_correction",
"summary_stale": True,
})
await fstore.doc_set("calls", call_id, {"incident_ids": [], "incident_id": None})
drb-c2-core/app/routers/incidents.py
+1
View File
@@ -170,6 +170,7 @@ async def unlink_call_from_incident(incident_id: str, call_id: str, _: dict = De
if not remaining:
updates["status"] = "resolved"
updates["resolved_at"] = datetime.now(timezone.utc).isoformat()
updates["resolved_via"] = "emptied_by_admin"
await fstore.doc_update("incidents", incident_id, updates)
call = await fstore.doc_get("calls", call_id)
drb-c2-core/app/routers/replay.py
+181
View File
@@ -0,0 +1,181 @@
"""
Admin replay routes — the backend for the /admin Replay tab.
See app/internal/replay.py for what a run is and why it exists. Every route is
admin-only: a run spends real AI credits.
"""
from datetime import datetime, timezone
from typing import Literal, Optional
from fastapi import APIRouter, Depends, HTTPException, Query
from pydantic import BaseModel
from app.internal import replay
from app.internal.audit import write_audit
from app.internal.auth import describe_actor, require_admin_token, resolve_caller_org_id
from app.internal.logger import logger
router = APIRouter(prefix="/admin/replay", tags=["admin"])
def _parse_ts(value: Optional[str], field: str) -> datetime:
if not value:
raise HTTPException(400, f"{field} is required.")
try:
dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
raise HTTPException(400, f"{field} is not an ISO-8601 timestamp.")
return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
async def _org(decoded: dict) -> str:
# Same fallback as /calls/search: a platform admin resolves to "every
# org", which is not a scope a replay can run in.
org_id = await resolve_caller_org_id(decoded) or decoded.get("org_id")
if not org_id:
raise HTTPException(403, "No organization scope for this caller.")
return org_id
async def _own_run(run_id: str, org_id: str) -> dict:
run = await replay.get_run(run_id)
if not run or run.get("org_id") != org_id:
raise HTTPException(404, f"Replay run '{run_id}' not found.")
return run
@router.get("/estimate")
async def estimate_run(
date_from: str = Query(...),
date_to: str = Query(...),
mode: Literal["audio", "transcripts", "reuse"] = Query("transcripts"),
system_ids: Optional[str] = Query(None, description="comma-separated"),
decoded: dict = Depends(require_admin_token),
):
"""How many calls a run over this range would process, and a rough cost."""
org_id = await _org(decoded)
sids = [s for s in (system_ids or "").split(",") if s] or None
calls, truncated = await replay.select_calls(
org_id, _parse_ts(date_from, "date_from"), _parse_ts(date_to, "date_to"), sids,
)
return {**replay.estimate(calls, mode), "truncated": truncated, "max_calls": replay.MAX_CALLS}
class StartRun(BaseModel):
date_from: str
date_to: str
mode: Literal["audio", "transcripts", "reuse"] = "transcripts"
system_ids: Optional[list[str]] = None
source_run_id: Optional[str] = None
label: str = ""
@router.post("")
async def start_run(body: StartRun, decoded: dict = Depends(require_admin_token)):
org_id = await _org(decoded)
actor_uid, actor_email = describe_actor(decoded)
try:
run = await replay.start_run(
org_id=org_id,
date_from=_parse_ts(body.date_from, "date_from"),
date_to=_parse_ts(body.date_to, "date_to"),
mode=body.mode,
system_ids=body.system_ids or None,
source_run_id=body.source_run_id,
label=body.label[:120],
actor=actor_email or actor_uid,
)
except replay.ReplayBusy as e:
raise HTTPException(409, str(e))
except ValueError as e:
raise HTTPException(400, str(e))
try:
await write_audit(actor_uid, actor_email, "replay.start", details={
"run_id": run["run_id"], "mode": run["mode"], "calls": run["progress"]["total"],
"est_cost_usd": run["estimate"]["est_cost_usd"],
})
except Exception as e:
logger.error(f"Replay: audit write failed ({e}) — run {run['run_id']} continues")
return run
@router.get("")
async def list_runs(decoded: dict = Depends(require_admin_token)):
org_id = await _org(decoded)
return {"runs": await replay.list_runs(org_id), "active_run_id": replay.active_run_id()}
@router.get("/{run_id}")
async def get_run(run_id: str, decoded: dict = Depends(require_admin_token)):
return await _own_run(run_id, await _org(decoded))
@router.post("/{run_id}/cancel")
async def cancel_run(run_id: str, decoded: dict = Depends(require_admin_token)):
await _own_run(run_id, await _org(decoded))
if not replay.request_cancel(run_id):
raise HTTPException(409, "That run is not running.")
return {"ok": True}
@router.delete("/{run_id}")
async def delete_run(run_id: str, decoded: dict = Depends(require_admin_token)):
await _own_run(run_id, await _org(decoded))
try:
await replay.delete_run(run_id)
except replay.ReplayBusy as e:
raise HTTPException(409, str(e))
return {"ok": True}
def _call_row(c: dict) -> dict:
scenes = c.get("scenes") or {}
paths = [((s.get("corr_debug") or {}).get("corr_path")) for _, s in sorted(scenes.items())]
return {
"call_id": c.get("call_id"),
"started_at": c.get("started_at"),
"talkgroup_name": c.get("talkgroup_name"),
"transcript": c.get("transcript_corrected") or c.get("transcript"),
"units": c.get("units"),
"cleared_units": c.get("cleared_units"),
"location": c.get("location"),
"skip_reason": c.get("skip_reason"),
"corr_path": [p for p in paths if p] or ([c["corr_path"]] if c.get("corr_path") else []),
"incident_ids": c.get("incident_ids") or [],
}
@router.get("/{run_id}/incidents")
async def run_incidents(run_id: str, decoded: dict = Depends(require_admin_token)):
"""
A run's sandbox, shaped for reading: every incident with its calls in
order, plus the calls that never linked. Embeddings stay out.
"""
await _own_run(run_id, await _org(decoded))
incidents, calls = await replay.sandbox_contents(run_id)
by_id = {c.get("call_id"): _call_row(c) for c in calls}
out = []
for inc in sorted(incidents, key=lambda i: str(i.get("started_at") or "")):
rows = [by_id[cid] for cid in (inc.get("call_ids") or []) if cid in by_id]
rows.sort(key=lambda r: str(r["started_at"] or ""))
out.append({
"incident_id": inc.get("incident_id"),
"title": inc.get("title"),
"type": inc.get("type"),
"severity": inc.get("severity"),
"status": inc.get("status"),
"resolved_via": inc.get("resolved_via"),
"started_at": inc.get("started_at"),
"updated_at": inc.get("updated_at"),
"resolved_at": inc.get("resolved_at"),
"location": inc.get("location"),
"location_coords": inc.get("location_coords"),
"units": inc.get("units"),
"units_active": inc.get("units_active"),
"units_cleared": inc.get("units_cleared"),
"talkgroup_ids": inc.get("talkgroup_ids"),
"calls": rows,
})
orphans = sorted((r for r in by_id.values() if not r["incident_ids"]),
key=lambda r: str(r["started_at"] or ""))
return {"incidents": out, "orphans": orphans}
drb-c2-core/app/routers/upload.py
+126 -86
View File
@@ -1,11 +1,11 @@
import secrets
from typing import Optional
from datetime import datetime, timezone
from fastapi import APIRouter, BackgroundTasks, UploadFile, File, Form, HTTPException, Security
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from app.internal.storage import upload_audio
from app.internal import dedup
from app.internal import firestore as fstore
from app.internal import clock
from app.internal.logger import logger
from app.config import settings
@@ -140,7 +140,7 @@ def _recent_incident_on_same_talkgroup(ctx: dict) -> bool:
if tg_id is None or not system_id:
return False
tg_str = str(tg_id)
now = ctx.get("now") or datetime.now(timezone.utc)
now = ctx.get("now") or clock.now()
idle_limit = settings.tg_dispatch_thin_idle_minutes
for inc in ctx.get("recent") or []:
if system_id not in (inc.get("system_ids") or []):
@@ -374,7 +374,8 @@ async def _run_extraction_pipeline(
if scene["resolved"] and incident_id:
await fstore.doc_set("incidents", incident_id, {
"status": "resolved",
"resolved_at": datetime.now(timezone.utc).isoformat(),
"resolved_at": clock.now().isoformat(),
"resolved_via": "llm_closure",
})
await incident_correlator.maybe_resolve_parent(incident_id)
logger.info(f"Auto-resolved incident {incident_id} (LLM closure detection)")
@@ -396,6 +397,113 @@ async def _run_extraction_pipeline(
)
async def _extract_and_correlate(
call_id: str,
node_id: str,
system_id: Optional[str],
talkgroup_id: Optional[int],
talkgroup_name: Optional[str],
transcript: Optional[str],
segments: Optional[list[dict]] = None,
scenes: Optional[list[dict]] = None,
) -> tuple[list[str], list[str], list[dict]]:
"""
Steps 2-3 of the intelligence pipeline for one call: scene extraction
(skipped when `scenes` is passed in), then per-scene correlation, then the
no-scene thin fallback. Returns (incident_ids, merged tags, scenes).
Shared by the live pipeline below and by replay (app/internal/replay.py),
so a replay run measures exactly the code that runs live rather than a
copy of it that can drift. Caller owns the correlation feature-flag check
and alerting.
"""
from app.internal import intelligence, incident_correlator
# Step 2: Scene detection + intelligence extraction
if scenes is None:
scenes = []
if transcript:
scenes = await intelligence.extract_scenes(
call_id, transcript, talkgroup_name,
talkgroup_id=talkgroup_id, system_id=system_id, segments=segments,
node_id=node_id,
)
# Step 3: Correlate each scene independently.
# A single recording can produce multiple incidents on a busy channel.
incident_ids: list[str] = []
all_tags: list[str] = []
# server-26#96: scene_index is threaded through so each scene's
# corr_debug/transcript lands in its own entry of the call doc's
# `scenes` map instead of clobbering every other scene's write.
for scene_index, scene in enumerate(scenes):
all_tags.extend(scene["tags"])
is_reassignment = bool(scene.get("reassignment"))
corr_units = [] if is_reassignment else scene.get("units")
incident_id = await _correlate_with_consensus(
call_id=call_id,
node_id=node_id,
system_id=system_id,
talkgroup_id=talkgroup_id,
talkgroup_name=talkgroup_name,
tags=scene["tags"],
incident_type=scene["incident_type"],
location=scene["location"],
location_coords=scene["location_coords"],
units=corr_units,
vehicles=scene.get("vehicles"),
cleared_units=scene.get("cleared_units"),
reassignment=is_reassignment,
embedding=scene.get("embedding"),
severity=scene.get("severity"),
transcript=scene.get("transcript"),
scene_index=scene_index,
)
if incident_id and incident_id not in incident_ids:
incident_ids.append(incident_id)
if scene["resolved"] and incident_id:
await fstore.doc_set("incidents", incident_id, {
"status": "resolved",
"resolved_at": clock.now().isoformat(),
"resolved_via": "llm_closure",
})
await incident_correlator.maybe_resolve_parent(incident_id)
logger.info(f"Auto-resolved incident {incident_id} (LLM closure detection)")
# Correlator also runs for calls with no scenes (unclassified) to attempt
# talkgroup-based linking even when no transcript could be produced.
# transcript_too_short (<=5 words: "10-8", "show me clear", a unit
# check-in) still carries a real transcript and talkgroup — exactly the
# brief follow-up/clearance traffic an incident needs, and the thin-path
# merge below already requires a same-talkgroup, recently-active
# incident before attaching anything, same guard already trusted for
# no-transcript calls. Previously excluded here, so these calls never
# attached to anything at all. garbage_transcript (Whisper
# hallucination) has no real content behind it and stays excluded.
if not scenes:
_call_doc = await fstore.doc_get("calls", call_id)
skip_reason = (_call_doc or {}).get("skip_reason")
if not skip_reason or skip_reason == "transcript_too_short":
incident_id = await _correlate_with_consensus(
call_id=call_id,
node_id=node_id,
system_id=system_id,
talkgroup_id=talkgroup_id,
talkgroup_name=talkgroup_name,
tags=[],
incident_type=None,
location=None,
location_coords=None,
)
if incident_id:
incident_ids.append(incident_id)
if incident_ids:
await fstore.doc_set("calls", call_id, {"incident_ids": incident_ids})
return incident_ids, all_tags, scenes
async def _run_intelligence_pipeline(
call_id: str,
node_id: str,
@@ -411,7 +519,7 @@ async def _run_intelligence_pipeline(
3. Correlate each scene with existing incidents (or create new ones)
4. Check alert rules and dispatch notifications
"""
from app.internal import transcription, intelligence, incident_correlator, alerter, talkgroups
from app.internal import transcription, alerter, talkgroups
# server-26#131: mark that real-time processing has started for this call
# BEFORE any of the slow steps below (STT, scene extraction, correlation).
@@ -427,7 +535,7 @@ async def _run_intelligence_pipeline(
# calls). Best-effort: a write failure here must not abort the pipeline.
try:
await fstore.doc_set("calls", call_id, {
"intelligence_started_at": datetime.now(timezone.utc).isoformat()
"intelligence_started_at": clock.now().isoformat()
})
except Exception as e:
logger.warning(f"Could not mark intelligence_started_at for call {call_id}: {e}")
@@ -466,90 +574,22 @@ async def _run_intelligence_pipeline(
scope = "globally" if not flags["stt_enabled"] else f"system {system_id}"
logger.info(f"STT disabled ({scope}) — skipping transcription for call {call_id}")
# Step 2: Scene detection + intelligence extraction
scenes: list[dict] = []
if _flag("correlation_enabled"):
if transcript:
scenes = await intelligence.extract_scenes(
call_id, transcript, talkgroup_name,
talkgroup_id=talkgroup_id, system_id=system_id, segments=segments,
node_id=node_id,
)
else:
scope = "globally" if not flags["correlation_enabled"] else f"system {system_id}"
logger.info(f"Correlation disabled ({scope}) — skipping scene extraction and correlation for call {call_id}")
# Step 3: Correlate each scene independently.
# A single recording can produce multiple incidents on a busy channel.
# Steps 2-3: scene extraction + correlation.
incident_ids: list[str] = []
all_tags: list[str] = []
if _flag("correlation_enabled"):
# server-26#96: scene_index is threaded through so each scene's
# corr_debug/transcript lands in its own entry of the call doc's
# `scenes` map instead of clobbering every other scene's write.
for scene_index, scene in enumerate(scenes):
all_tags.extend(scene["tags"])
is_reassignment = bool(scene.get("reassignment"))
corr_units = [] if is_reassignment else scene.get("units")
incident_id = await _correlate_with_consensus(
call_id=call_id,
node_id=node_id,
system_id=system_id,
talkgroup_id=talkgroup_id,
talkgroup_name=talkgroup_name,
tags=scene["tags"],
incident_type=scene["incident_type"],
location=scene["location"],
location_coords=scene["location_coords"],
units=corr_units,
vehicles=scene.get("vehicles"),
cleared_units=scene.get("cleared_units"),
reassignment=is_reassignment,
embedding=scene.get("embedding"),
severity=scene.get("severity"),
transcript=scene.get("transcript"),
scene_index=scene_index,
)
if incident_id and incident_id not in incident_ids:
incident_ids.append(incident_id)
if scene["resolved"] and incident_id:
await fstore.doc_set("incidents", incident_id, {
"status": "resolved",
"resolved_at": datetime.now(timezone.utc).isoformat(),
})
await incident_correlator.maybe_resolve_parent(incident_id)
logger.info(f"Auto-resolved incident {incident_id} (LLM closure detection)")
# Correlator also runs for calls with no scenes (unclassified) to attempt
# talkgroup-based linking even when no transcript could be produced.
# transcript_too_short (<=5 words: "10-8", "show me clear", a unit
# check-in) still carries a real transcript and talkgroup — exactly the
# brief follow-up/clearance traffic an incident needs, and the thin-path
# merge below already requires a same-talkgroup, recently-active
# incident before attaching anything, same guard already trusted for
# no-transcript calls. Previously excluded here, so these calls never
# attached to anything at all. garbage_transcript (Whisper
# hallucination) has no real content behind it and stays excluded.
if not scenes:
_call_doc = await fstore.doc_get("calls", call_id)
skip_reason = (_call_doc or {}).get("skip_reason")
if not skip_reason or skip_reason == "transcript_too_short":
incident_id = await _correlate_with_consensus(
call_id=call_id,
node_id=node_id,
system_id=system_id,
talkgroup_id=talkgroup_id,
talkgroup_name=talkgroup_name,
tags=[],
incident_type=None,
location=None,
location_coords=None,
)
if incident_id:
incident_ids.append(incident_id)
if incident_ids:
await fstore.doc_set("calls", call_id, {"incident_ids": incident_ids})
incident_ids, all_tags, _ = await _extract_and_correlate(
call_id=call_id,
node_id=node_id,
system_id=system_id,
talkgroup_id=talkgroup_id,
talkgroup_name=talkgroup_name,
transcript=transcript,
segments=segments,
)
else:
scope = "globally" if not flags["correlation_enabled"] else f"system {system_id}"
logger.info(f"Correlation disabled ({scope}) — skipping scene extraction and correlation for call {call_id}")
# Step 4: Alert dispatch (always runs — talkgroup ID rules don't need a transcript)
await alerter.check_and_dispatch(