Files
server-26/drb-c2-core/app/routers/incidents.py
T
Logan CusanoandClaude Opus 5.5 aff3f16d32 Admin Replay: re-run the pipeline over past calls in a sandbox (#170)
Correlation has only ever been measured through live AI windows: days of
wall time per change, and the 09-20→22 window was invalidated outright by
unfunded AI accounts (#169). Recordings are kept regardless of AI, so the
traffic to measure against already exists.

- internal/replay.py: runs a time range of real calls through the live
  pipeline code in original order, clock pinned per call, into
  replay_runs/{run_id}/calls|incidents. Modes: audio (re-transcribe),
  transcripts (re-extract), reuse (correlation only from a prior run's
  scenes). Simulates the idle-resolve and orphan-recorrelation sweeps on
  virtual time. No alerts, summaries, vocab, AI-health alerts or pending
  terms. One run at a time, <=5000 calls, <=7 days.
- firestore.py: ContextVar sandbox redirect for calls/incidents.
- clock.py: ContextVar-pinnable now(), used on the correlation path.
- feature_flags.py: ContextVar flag override so replay runs with live AI off.
- upload.py: scene loop extracted to _extract_and_correlate, shared by the
  live pipeline and replay so replay measures the code that runs live.
- resolved_via on every incident resolve, so a real clear can be told
  from the idle timeout — live and in replay.
- routers/replay.py + /admin Replay tab: estimate, start, compare runs,
  drill into incidents with audio.

Reviewed by drb-correlation-review; its leak and fidelity findings are
fixed and covered by tests. c2-core: 456 pass. Frontend typecheck not run
(no Node on the authoring box).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-26 15:23:57 -04:00

187 lines
7.1 KiB
Python

import uuid
from datetime import datetime, timezone
from typing import Optional
from fastapi import APIRouter, BackgroundTasks, HTTPException, Depends
from app.models import IncidentCreate, IncidentUpdate
from app.internal import firestore as fstore
from app.internal.auth import (
require_admin_token,
require_service_or_firebase_token,
resolve_caller_org_id,
summarize_limiter,
)
router = APIRouter(prefix="/incidents", tags=["incidents"])
@router.get("")
async def list_incidents(
status: Optional[str] = None,
type: Optional[str] = None,
decoded: dict = Depends(require_service_or_firebase_token),
):
filters = {}
if status:
filters["status"] = status
if type:
filters["type"] = type
org_id = await resolve_caller_org_id(decoded)
if org_id is not None:
filters["org_id"] = org_id
return await fstore.collection_list("incidents", **filters)
@router.post("/summarize")
async def summarize_all_stale(
background_tasks: BackgroundTasks,
_: dict = Depends(require_admin_token),
):
"""Immediately run the summarizer pass on all stale incidents (don't wait for the next interval)."""
from app.internal.summarizer import _run_summary_pass
background_tasks.add_task(_run_summary_pass)
return {"ok": True}
@router.get("/{incident_id}")
async def get_incident(incident_id: str, decoded: dict = Depends(require_service_or_firebase_token)):
doc = await fstore.doc_get("incidents", incident_id)
if not doc:
raise HTTPException(404, f"Incident '{incident_id}' not found.")
org_id = await resolve_caller_org_id(decoded)
if org_id is not None and doc.get("org_id") != org_id:
raise HTTPException(404, f"Incident '{incident_id}' not found.")
return doc
@router.post("")
async def create_incident(body: IncidentCreate, _: dict = Depends(require_admin_token)):
now = datetime.now(timezone.utc).isoformat()
incident_id = str(uuid.uuid4())
doc = {
"incident_id": incident_id,
"title": body.title,
"type": body.type,
"status": body.status,
"location": body.location,
"call_ids": body.call_ids,
"summary": body.summary,
"tags": body.tags,
"started_at": now,
"updated_at": now,
}
await fstore.doc_set("incidents", incident_id, doc, merge=False)
return doc
@router.put("/{incident_id}")
async def update_incident(incident_id: str, body: IncidentUpdate, _: dict = Depends(require_admin_token)):
doc = await fstore.doc_get("incidents", incident_id)
if not doc:
raise HTTPException(404, f"Incident '{incident_id}' not found.")
updates = body.model_dump(exclude_none=True)
updates["updated_at"] = datetime.now(timezone.utc).isoformat()
await fstore.doc_update("incidents", incident_id, updates)
return {**doc, **updates}
@router.delete("/{incident_id}")
async def delete_incident(incident_id: str, _: dict = Depends(require_admin_token)):
doc = await fstore.doc_get("incidents", incident_id)
if not doc:
raise HTTPException(404, f"Incident '{incident_id}' not found.")
await fstore.doc_delete("incidents", incident_id)
return {"ok": True}
@router.post("/{incident_id}/summarize")
async def summarize_incident(
incident_id: str,
background_tasks: BackgroundTasks,
decoded: dict = Depends(require_admin_token),
):
"""Immediately run the summarizer for a specific incident."""
from app.internal.summarizer import _summarize_incident
from app.internal.feature_flags import get_flags
inc = await fstore.doc_get("incidents", incident_id)
if not inc:
raise HTTPException(404, f"Incident '{incident_id}' not found.")
flags = await get_flags()
if not flags["summaries_enabled"]:
return {"ok": False, "incident_id": incident_id, "summaries_enabled": False}
# Rate limit by incident ID to prevent repeated expensive LLM calls
summarize_limiter.check(incident_id)
background_tasks.add_task(_summarize_incident, inc)
return {"ok": True, "incident_id": incident_id, "summaries_enabled": True}
@router.post("/{incident_id}/calls/{call_id}")
async def link_call_to_incident(incident_id: str, call_id: str, _: dict = Depends(require_admin_token)):
"""Manually attach a call to an incident (the /calls page's attribution action)."""
doc = await fstore.doc_get("incidents", incident_id)
if not doc:
raise HTTPException(404, f"Incident '{incident_id}' not found.")
call = await fstore.doc_get("calls", call_id)
if not call:
raise HTTPException(404, f"Call '{call_id}' not found.")
call_ids = list(doc.get("call_ids") or [])
if call_id not in call_ids:
call_ids.append(call_id)
await fstore.doc_update("incidents", incident_id, {
"call_ids": call_ids,
"updated_at": datetime.now(timezone.utc).isoformat(),
# A manually attached call changes what the incident is about.
"summary_stale": True,
})
# incident_ids is the canonical link — it is what the correlator writes and
# what the frontend queries with array-contains. This route only ever set
# the legacy scalar incident_id, so a manually attached call stayed
# invisible on the incident's own page.
incident_ids = list(call.get("incident_ids") or ([call["incident_id"]] if call.get("incident_id") else []))
if incident_id not in incident_ids:
incident_ids.append(incident_id)
await fstore.doc_update("calls", call_id, {
"incident_ids": incident_ids,
"incident_id": incident_id,
"corr_path": "manual",
})
return {"ok": True, "incident_ids": incident_ids}
@router.delete("/{incident_id}/calls/{call_id}")
async def unlink_call_from_incident(incident_id: str, call_id: str, _: dict = Depends(require_admin_token)):
"""
Detach a call from an incident — the other half of manual attribution.
An incident left with no calls is resolved rather than deleted, matching
what calls.py's transcript correction does when it empties one.
"""
doc = await fstore.doc_get("incidents", incident_id)
if not doc:
raise HTTPException(404, f"Incident '{incident_id}' not found.")
remaining = [c for c in (doc.get("call_ids") or []) if c != call_id]
updates: dict = {
"call_ids": remaining,
"updated_at": datetime.now(timezone.utc).isoformat(),
"summary_stale": True,
}
if not remaining:
updates["status"] = "resolved"
updates["resolved_at"] = datetime.now(timezone.utc).isoformat()
updates["resolved_via"] = "emptied_by_admin"
await fstore.doc_update("incidents", incident_id, updates)
call = await fstore.doc_get("calls", call_id)
if call:
incident_ids = [
i for i in (call.get("incident_ids") or ([call["incident_id"]] if call.get("incident_id") else []))
if i != incident_id
]
await fstore.doc_update("calls", call_id, {
"incident_ids": incident_ids,
"incident_id": incident_ids[0] if incident_ids else None,
})
return {"ok": True, "incident_emptied": not remaining}