Correlation has only ever been measured through live AI windows: days of wall time per change, and the 09-20→22 window was invalidated outright by unfunded AI accounts (#169). Recordings are kept regardless of AI, so the traffic to measure against already exists. - internal/replay.py: runs a time range of real calls through the live pipeline code in original order, clock pinned per call, into replay_runs/{run_id}/calls|incidents. Modes: audio (re-transcribe), transcripts (re-extract), reuse (correlation only from a prior run's scenes). Simulates the idle-resolve and orphan-recorrelation sweeps on virtual time. No alerts, summaries, vocab, AI-health alerts or pending terms. One run at a time, <=5000 calls, <=7 days. - firestore.py: ContextVar sandbox redirect for calls/incidents. - clock.py: ContextVar-pinnable now(), used on the correlation path. - feature_flags.py: ContextVar flag override so replay runs with live AI off. - upload.py: scene loop extracted to _extract_and_correlate, shared by the live pipeline and replay so replay measures the code that runs live. - resolved_via on every incident resolve, so a real clear can be told from the idle timeout — live and in replay. - routers/replay.py + /admin Replay tab: estimate, start, compare runs, drill into incidents with audio. Reviewed by drb-correlation-review; its leak and fidelity findings are fixed and covered by tests. c2-core: 456 pass. Frontend typecheck not run (no Node on the authoring box). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
187 lines
7.1 KiB
Python
187 lines
7.1 KiB
Python
import uuid
|
|
from datetime import datetime, timezone
|
|
from typing import Optional
|
|
from fastapi import APIRouter, BackgroundTasks, HTTPException, Depends
|
|
from app.models import IncidentCreate, IncidentUpdate
|
|
from app.internal import firestore as fstore
|
|
from app.internal.auth import (
|
|
require_admin_token,
|
|
require_service_or_firebase_token,
|
|
resolve_caller_org_id,
|
|
summarize_limiter,
|
|
)
|
|
|
|
router = APIRouter(prefix="/incidents", tags=["incidents"])
|
|
|
|
|
|
@router.get("")
|
|
async def list_incidents(
|
|
status: Optional[str] = None,
|
|
type: Optional[str] = None,
|
|
decoded: dict = Depends(require_service_or_firebase_token),
|
|
):
|
|
filters = {}
|
|
if status:
|
|
filters["status"] = status
|
|
if type:
|
|
filters["type"] = type
|
|
org_id = await resolve_caller_org_id(decoded)
|
|
if org_id is not None:
|
|
filters["org_id"] = org_id
|
|
return await fstore.collection_list("incidents", **filters)
|
|
|
|
|
|
@router.post("/summarize")
|
|
async def summarize_all_stale(
|
|
background_tasks: BackgroundTasks,
|
|
_: dict = Depends(require_admin_token),
|
|
):
|
|
"""Immediately run the summarizer pass on all stale incidents (don't wait for the next interval)."""
|
|
from app.internal.summarizer import _run_summary_pass
|
|
background_tasks.add_task(_run_summary_pass)
|
|
return {"ok": True}
|
|
|
|
|
|
@router.get("/{incident_id}")
|
|
async def get_incident(incident_id: str, decoded: dict = Depends(require_service_or_firebase_token)):
|
|
doc = await fstore.doc_get("incidents", incident_id)
|
|
if not doc:
|
|
raise HTTPException(404, f"Incident '{incident_id}' not found.")
|
|
org_id = await resolve_caller_org_id(decoded)
|
|
if org_id is not None and doc.get("org_id") != org_id:
|
|
raise HTTPException(404, f"Incident '{incident_id}' not found.")
|
|
return doc
|
|
|
|
|
|
@router.post("")
|
|
async def create_incident(body: IncidentCreate, _: dict = Depends(require_admin_token)):
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
incident_id = str(uuid.uuid4())
|
|
doc = {
|
|
"incident_id": incident_id,
|
|
"title": body.title,
|
|
"type": body.type,
|
|
"status": body.status,
|
|
"location": body.location,
|
|
"call_ids": body.call_ids,
|
|
"summary": body.summary,
|
|
"tags": body.tags,
|
|
"started_at": now,
|
|
"updated_at": now,
|
|
}
|
|
await fstore.doc_set("incidents", incident_id, doc, merge=False)
|
|
return doc
|
|
|
|
|
|
@router.put("/{incident_id}")
|
|
async def update_incident(incident_id: str, body: IncidentUpdate, _: dict = Depends(require_admin_token)):
|
|
doc = await fstore.doc_get("incidents", incident_id)
|
|
if not doc:
|
|
raise HTTPException(404, f"Incident '{incident_id}' not found.")
|
|
updates = body.model_dump(exclude_none=True)
|
|
updates["updated_at"] = datetime.now(timezone.utc).isoformat()
|
|
await fstore.doc_update("incidents", incident_id, updates)
|
|
return {**doc, **updates}
|
|
|
|
|
|
@router.delete("/{incident_id}")
|
|
async def delete_incident(incident_id: str, _: dict = Depends(require_admin_token)):
|
|
doc = await fstore.doc_get("incidents", incident_id)
|
|
if not doc:
|
|
raise HTTPException(404, f"Incident '{incident_id}' not found.")
|
|
await fstore.doc_delete("incidents", incident_id)
|
|
return {"ok": True}
|
|
|
|
|
|
@router.post("/{incident_id}/summarize")
|
|
async def summarize_incident(
|
|
incident_id: str,
|
|
background_tasks: BackgroundTasks,
|
|
decoded: dict = Depends(require_admin_token),
|
|
):
|
|
"""Immediately run the summarizer for a specific incident."""
|
|
from app.internal.summarizer import _summarize_incident
|
|
from app.internal.feature_flags import get_flags
|
|
inc = await fstore.doc_get("incidents", incident_id)
|
|
if not inc:
|
|
raise HTTPException(404, f"Incident '{incident_id}' not found.")
|
|
flags = await get_flags()
|
|
if not flags["summaries_enabled"]:
|
|
return {"ok": False, "incident_id": incident_id, "summaries_enabled": False}
|
|
# Rate limit by incident ID to prevent repeated expensive LLM calls
|
|
summarize_limiter.check(incident_id)
|
|
background_tasks.add_task(_summarize_incident, inc)
|
|
return {"ok": True, "incident_id": incident_id, "summaries_enabled": True}
|
|
|
|
|
|
@router.post("/{incident_id}/calls/{call_id}")
|
|
async def link_call_to_incident(incident_id: str, call_id: str, _: dict = Depends(require_admin_token)):
|
|
"""Manually attach a call to an incident (the /calls page's attribution action)."""
|
|
doc = await fstore.doc_get("incidents", incident_id)
|
|
if not doc:
|
|
raise HTTPException(404, f"Incident '{incident_id}' not found.")
|
|
call = await fstore.doc_get("calls", call_id)
|
|
if not call:
|
|
raise HTTPException(404, f"Call '{call_id}' not found.")
|
|
|
|
call_ids = list(doc.get("call_ids") or [])
|
|
if call_id not in call_ids:
|
|
call_ids.append(call_id)
|
|
await fstore.doc_update("incidents", incident_id, {
|
|
"call_ids": call_ids,
|
|
"updated_at": datetime.now(timezone.utc).isoformat(),
|
|
# A manually attached call changes what the incident is about.
|
|
"summary_stale": True,
|
|
})
|
|
|
|
# incident_ids is the canonical link — it is what the correlator writes and
|
|
# what the frontend queries with array-contains. This route only ever set
|
|
# the legacy scalar incident_id, so a manually attached call stayed
|
|
# invisible on the incident's own page.
|
|
incident_ids = list(call.get("incident_ids") or ([call["incident_id"]] if call.get("incident_id") else []))
|
|
if incident_id not in incident_ids:
|
|
incident_ids.append(incident_id)
|
|
await fstore.doc_update("calls", call_id, {
|
|
"incident_ids": incident_ids,
|
|
"incident_id": incident_id,
|
|
"corr_path": "manual",
|
|
})
|
|
return {"ok": True, "incident_ids": incident_ids}
|
|
|
|
|
|
@router.delete("/{incident_id}/calls/{call_id}")
|
|
async def unlink_call_from_incident(incident_id: str, call_id: str, _: dict = Depends(require_admin_token)):
|
|
"""
|
|
Detach a call from an incident — the other half of manual attribution.
|
|
|
|
An incident left with no calls is resolved rather than deleted, matching
|
|
what calls.py's transcript correction does when it empties one.
|
|
"""
|
|
doc = await fstore.doc_get("incidents", incident_id)
|
|
if not doc:
|
|
raise HTTPException(404, f"Incident '{incident_id}' not found.")
|
|
|
|
remaining = [c for c in (doc.get("call_ids") or []) if c != call_id]
|
|
updates: dict = {
|
|
"call_ids": remaining,
|
|
"updated_at": datetime.now(timezone.utc).isoformat(),
|
|
"summary_stale": True,
|
|
}
|
|
if not remaining:
|
|
updates["status"] = "resolved"
|
|
updates["resolved_at"] = datetime.now(timezone.utc).isoformat()
|
|
updates["resolved_via"] = "emptied_by_admin"
|
|
await fstore.doc_update("incidents", incident_id, updates)
|
|
|
|
call = await fstore.doc_get("calls", call_id)
|
|
if call:
|
|
incident_ids = [
|
|
i for i in (call.get("incident_ids") or ([call["incident_id"]] if call.get("incident_id") else []))
|
|
if i != incident_id
|
|
]
|
|
await fstore.doc_update("calls", call_id, {
|
|
"incident_ids": incident_ids,
|
|
"incident_id": incident_ids[0] if incident_ids else None,
|
|
})
|
|
return {"ok": True, "incident_emptied": not remaining}
|