Admin Replay: re-run the pipeline over past calls in a sandbox (#170)
Correlation has only ever been measured through live AI windows: days of wall time per change, and the 09-20→22 window was invalidated outright by unfunded AI accounts (#169). Recordings are kept regardless of AI, so the traffic to measure against already exists. - internal/replay.py: runs a time range of real calls through the live pipeline code in original order, clock pinned per call, into replay_runs/{run_id}/calls|incidents. Modes: audio (re-transcribe), transcripts (re-extract), reuse (correlation only from a prior run's scenes). Simulates the idle-resolve and orphan-recorrelation sweeps on virtual time. No alerts, summaries, vocab, AI-health alerts or pending terms. One run at a time, <=5000 calls, <=7 days. - firestore.py: ContextVar sandbox redirect for calls/incidents. - clock.py: ContextVar-pinnable now(), used on the correlation path. - feature_flags.py: ContextVar flag override so replay runs with live AI off. - upload.py: scene loop extracted to _extract_and_correlate, shared by the live pipeline and replay so replay measures the code that runs live. - resolved_via on every incident resolve, so a real clear can be told from the idle timeout — live and in replay. - routers/replay.py + /admin Replay tab: estimate, start, compare runs, drill into incidents with audio. Reviewed by drb-correlation-review; its leak and fidelity findings are fixed and covered by tests. c2-core: 456 pass. Frontend typecheck not run (no Node on the authoring box). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Logan Cusano
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
e79b8bc37d
commit
aff3f16d32
@@ -1,11 +1,11 @@
|
||||
import secrets
|
||||
from typing import Optional
|
||||
from datetime import datetime, timezone
|
||||
from fastapi import APIRouter, BackgroundTasks, UploadFile, File, Form, HTTPException, Security
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
from app.internal.storage import upload_audio
|
||||
from app.internal import dedup
|
||||
from app.internal import firestore as fstore
|
||||
from app.internal import clock
|
||||
from app.internal.logger import logger
|
||||
from app.config import settings
|
||||
|
||||
@@ -140,7 +140,7 @@ def _recent_incident_on_same_talkgroup(ctx: dict) -> bool:
|
||||
if tg_id is None or not system_id:
|
||||
return False
|
||||
tg_str = str(tg_id)
|
||||
now = ctx.get("now") or datetime.now(timezone.utc)
|
||||
now = ctx.get("now") or clock.now()
|
||||
idle_limit = settings.tg_dispatch_thin_idle_minutes
|
||||
for inc in ctx.get("recent") or []:
|
||||
if system_id not in (inc.get("system_ids") or []):
|
||||
@@ -374,7 +374,8 @@ async def _run_extraction_pipeline(
|
||||
if scene["resolved"] and incident_id:
|
||||
await fstore.doc_set("incidents", incident_id, {
|
||||
"status": "resolved",
|
||||
"resolved_at": datetime.now(timezone.utc).isoformat(),
|
||||
"resolved_at": clock.now().isoformat(),
|
||||
"resolved_via": "llm_closure",
|
||||
})
|
||||
await incident_correlator.maybe_resolve_parent(incident_id)
|
||||
logger.info(f"Auto-resolved incident {incident_id} (LLM closure detection)")
|
||||
@@ -396,6 +397,113 @@ async def _run_extraction_pipeline(
|
||||
)
|
||||
|
||||
|
||||
async def _extract_and_correlate(
|
||||
call_id: str,
|
||||
node_id: str,
|
||||
system_id: Optional[str],
|
||||
talkgroup_id: Optional[int],
|
||||
talkgroup_name: Optional[str],
|
||||
transcript: Optional[str],
|
||||
segments: Optional[list[dict]] = None,
|
||||
scenes: Optional[list[dict]] = None,
|
||||
) -> tuple[list[str], list[str], list[dict]]:
|
||||
"""
|
||||
Steps 2-3 of the intelligence pipeline for one call: scene extraction
|
||||
(skipped when `scenes` is passed in), then per-scene correlation, then the
|
||||
no-scene thin fallback. Returns (incident_ids, merged tags, scenes).
|
||||
|
||||
Shared by the live pipeline below and by replay (app/internal/replay.py),
|
||||
so a replay run measures exactly the code that runs live rather than a
|
||||
copy of it that can drift. Caller owns the correlation feature-flag check
|
||||
and alerting.
|
||||
"""
|
||||
from app.internal import intelligence, incident_correlator
|
||||
|
||||
# Step 2: Scene detection + intelligence extraction
|
||||
if scenes is None:
|
||||
scenes = []
|
||||
if transcript:
|
||||
scenes = await intelligence.extract_scenes(
|
||||
call_id, transcript, talkgroup_name,
|
||||
talkgroup_id=talkgroup_id, system_id=system_id, segments=segments,
|
||||
node_id=node_id,
|
||||
)
|
||||
|
||||
# Step 3: Correlate each scene independently.
|
||||
# A single recording can produce multiple incidents on a busy channel.
|
||||
incident_ids: list[str] = []
|
||||
all_tags: list[str] = []
|
||||
# server-26#96: scene_index is threaded through so each scene's
|
||||
# corr_debug/transcript lands in its own entry of the call doc's
|
||||
# `scenes` map instead of clobbering every other scene's write.
|
||||
for scene_index, scene in enumerate(scenes):
|
||||
all_tags.extend(scene["tags"])
|
||||
is_reassignment = bool(scene.get("reassignment"))
|
||||
corr_units = [] if is_reassignment else scene.get("units")
|
||||
incident_id = await _correlate_with_consensus(
|
||||
call_id=call_id,
|
||||
node_id=node_id,
|
||||
system_id=system_id,
|
||||
talkgroup_id=talkgroup_id,
|
||||
talkgroup_name=talkgroup_name,
|
||||
tags=scene["tags"],
|
||||
incident_type=scene["incident_type"],
|
||||
location=scene["location"],
|
||||
location_coords=scene["location_coords"],
|
||||
units=corr_units,
|
||||
vehicles=scene.get("vehicles"),
|
||||
cleared_units=scene.get("cleared_units"),
|
||||
reassignment=is_reassignment,
|
||||
embedding=scene.get("embedding"),
|
||||
severity=scene.get("severity"),
|
||||
transcript=scene.get("transcript"),
|
||||
scene_index=scene_index,
|
||||
)
|
||||
if incident_id and incident_id not in incident_ids:
|
||||
incident_ids.append(incident_id)
|
||||
if scene["resolved"] and incident_id:
|
||||
await fstore.doc_set("incidents", incident_id, {
|
||||
"status": "resolved",
|
||||
"resolved_at": clock.now().isoformat(),
|
||||
"resolved_via": "llm_closure",
|
||||
})
|
||||
await incident_correlator.maybe_resolve_parent(incident_id)
|
||||
logger.info(f"Auto-resolved incident {incident_id} (LLM closure detection)")
|
||||
|
||||
# Correlator also runs for calls with no scenes (unclassified) to attempt
|
||||
# talkgroup-based linking even when no transcript could be produced.
|
||||
# transcript_too_short (<=5 words: "10-8", "show me clear", a unit
|
||||
# check-in) still carries a real transcript and talkgroup — exactly the
|
||||
# brief follow-up/clearance traffic an incident needs, and the thin-path
|
||||
# merge below already requires a same-talkgroup, recently-active
|
||||
# incident before attaching anything, same guard already trusted for
|
||||
# no-transcript calls. Previously excluded here, so these calls never
|
||||
# attached to anything at all. garbage_transcript (Whisper
|
||||
# hallucination) has no real content behind it and stays excluded.
|
||||
if not scenes:
|
||||
_call_doc = await fstore.doc_get("calls", call_id)
|
||||
skip_reason = (_call_doc or {}).get("skip_reason")
|
||||
if not skip_reason or skip_reason == "transcript_too_short":
|
||||
incident_id = await _correlate_with_consensus(
|
||||
call_id=call_id,
|
||||
node_id=node_id,
|
||||
system_id=system_id,
|
||||
talkgroup_id=talkgroup_id,
|
||||
talkgroup_name=talkgroup_name,
|
||||
tags=[],
|
||||
incident_type=None,
|
||||
location=None,
|
||||
location_coords=None,
|
||||
)
|
||||
if incident_id:
|
||||
incident_ids.append(incident_id)
|
||||
|
||||
if incident_ids:
|
||||
await fstore.doc_set("calls", call_id, {"incident_ids": incident_ids})
|
||||
|
||||
return incident_ids, all_tags, scenes
|
||||
|
||||
|
||||
async def _run_intelligence_pipeline(
|
||||
call_id: str,
|
||||
node_id: str,
|
||||
@@ -411,7 +519,7 @@ async def _run_intelligence_pipeline(
|
||||
3. Correlate each scene with existing incidents (or create new ones)
|
||||
4. Check alert rules and dispatch notifications
|
||||
"""
|
||||
from app.internal import transcription, intelligence, incident_correlator, alerter, talkgroups
|
||||
from app.internal import transcription, alerter, talkgroups
|
||||
|
||||
# server-26#131: mark that real-time processing has started for this call
|
||||
# BEFORE any of the slow steps below (STT, scene extraction, correlation).
|
||||
@@ -427,7 +535,7 @@ async def _run_intelligence_pipeline(
|
||||
# calls). Best-effort: a write failure here must not abort the pipeline.
|
||||
try:
|
||||
await fstore.doc_set("calls", call_id, {
|
||||
"intelligence_started_at": datetime.now(timezone.utc).isoformat()
|
||||
"intelligence_started_at": clock.now().isoformat()
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not mark intelligence_started_at for call {call_id}: {e}")
|
||||
@@ -466,90 +574,22 @@ async def _run_intelligence_pipeline(
|
||||
scope = "globally" if not flags["stt_enabled"] else f"system {system_id}"
|
||||
logger.info(f"STT disabled ({scope}) — skipping transcription for call {call_id}")
|
||||
|
||||
# Step 2: Scene detection + intelligence extraction
|
||||
scenes: list[dict] = []
|
||||
if _flag("correlation_enabled"):
|
||||
if transcript:
|
||||
scenes = await intelligence.extract_scenes(
|
||||
call_id, transcript, talkgroup_name,
|
||||
talkgroup_id=talkgroup_id, system_id=system_id, segments=segments,
|
||||
node_id=node_id,
|
||||
)
|
||||
else:
|
||||
scope = "globally" if not flags["correlation_enabled"] else f"system {system_id}"
|
||||
logger.info(f"Correlation disabled ({scope}) — skipping scene extraction and correlation for call {call_id}")
|
||||
|
||||
# Step 3: Correlate each scene independently.
|
||||
# A single recording can produce multiple incidents on a busy channel.
|
||||
# Steps 2-3: scene extraction + correlation.
|
||||
incident_ids: list[str] = []
|
||||
all_tags: list[str] = []
|
||||
if _flag("correlation_enabled"):
|
||||
# server-26#96: scene_index is threaded through so each scene's
|
||||
# corr_debug/transcript lands in its own entry of the call doc's
|
||||
# `scenes` map instead of clobbering every other scene's write.
|
||||
for scene_index, scene in enumerate(scenes):
|
||||
all_tags.extend(scene["tags"])
|
||||
is_reassignment = bool(scene.get("reassignment"))
|
||||
corr_units = [] if is_reassignment else scene.get("units")
|
||||
incident_id = await _correlate_with_consensus(
|
||||
call_id=call_id,
|
||||
node_id=node_id,
|
||||
system_id=system_id,
|
||||
talkgroup_id=talkgroup_id,
|
||||
talkgroup_name=talkgroup_name,
|
||||
tags=scene["tags"],
|
||||
incident_type=scene["incident_type"],
|
||||
location=scene["location"],
|
||||
location_coords=scene["location_coords"],
|
||||
units=corr_units,
|
||||
vehicles=scene.get("vehicles"),
|
||||
cleared_units=scene.get("cleared_units"),
|
||||
reassignment=is_reassignment,
|
||||
embedding=scene.get("embedding"),
|
||||
severity=scene.get("severity"),
|
||||
transcript=scene.get("transcript"),
|
||||
scene_index=scene_index,
|
||||
)
|
||||
if incident_id and incident_id not in incident_ids:
|
||||
incident_ids.append(incident_id)
|
||||
if scene["resolved"] and incident_id:
|
||||
await fstore.doc_set("incidents", incident_id, {
|
||||
"status": "resolved",
|
||||
"resolved_at": datetime.now(timezone.utc).isoformat(),
|
||||
})
|
||||
await incident_correlator.maybe_resolve_parent(incident_id)
|
||||
logger.info(f"Auto-resolved incident {incident_id} (LLM closure detection)")
|
||||
|
||||
# Correlator also runs for calls with no scenes (unclassified) to attempt
|
||||
# talkgroup-based linking even when no transcript could be produced.
|
||||
# transcript_too_short (<=5 words: "10-8", "show me clear", a unit
|
||||
# check-in) still carries a real transcript and talkgroup — exactly the
|
||||
# brief follow-up/clearance traffic an incident needs, and the thin-path
|
||||
# merge below already requires a same-talkgroup, recently-active
|
||||
# incident before attaching anything, same guard already trusted for
|
||||
# no-transcript calls. Previously excluded here, so these calls never
|
||||
# attached to anything at all. garbage_transcript (Whisper
|
||||
# hallucination) has no real content behind it and stays excluded.
|
||||
if not scenes:
|
||||
_call_doc = await fstore.doc_get("calls", call_id)
|
||||
skip_reason = (_call_doc or {}).get("skip_reason")
|
||||
if not skip_reason or skip_reason == "transcript_too_short":
|
||||
incident_id = await _correlate_with_consensus(
|
||||
call_id=call_id,
|
||||
node_id=node_id,
|
||||
system_id=system_id,
|
||||
talkgroup_id=talkgroup_id,
|
||||
talkgroup_name=talkgroup_name,
|
||||
tags=[],
|
||||
incident_type=None,
|
||||
location=None,
|
||||
location_coords=None,
|
||||
)
|
||||
if incident_id:
|
||||
incident_ids.append(incident_id)
|
||||
|
||||
if incident_ids:
|
||||
await fstore.doc_set("calls", call_id, {"incident_ids": incident_ids})
|
||||
incident_ids, all_tags, _ = await _extract_and_correlate(
|
||||
call_id=call_id,
|
||||
node_id=node_id,
|
||||
system_id=system_id,
|
||||
talkgroup_id=talkgroup_id,
|
||||
talkgroup_name=talkgroup_name,
|
||||
transcript=transcript,
|
||||
segments=segments,
|
||||
)
|
||||
else:
|
||||
scope = "globally" if not flags["correlation_enabled"] else f"system {system_id}"
|
||||
logger.info(f"Correlation disabled ({scope}) — skipping scene extraction and correlation for call {call_id}")
|
||||
|
||||
# Step 4: Alert dispatch (always runs — talkgroup ID rules don't need a transcript)
|
||||
await alerter.check_and_dispatch(
|
||||
|
||||
Reference in New Issue
Block a user