diff --git a/drb-c2-core/app/routers/admin.py b/drb-c2-core/app/routers/admin.py index 11de641..21df096 100644 --- a/drb-c2-core/app/routers/admin.py +++ b/drb-c2-core/app/routers/admin.py @@ -98,13 +98,29 @@ async def debug_correlation( ai_systems = await _get_ai_enabled_system_ids(global_flags) # ── Fetch recent incidents (AI-enabled systems only) ────────────────────── - all_incidents = await fstore.collection_list("incidents") - all_incidents.sort(key=lambda i: i.get("updated_at", ""), reverse=True) + # Read a bounded, already-sorted window rather than the whole collection. + # This route used to pull every incident ever created and sort in Python, + # which stopped returning at all once the collection grew — Firestore kills + # an unbounded scan with a 503 and the request just hangs. Ordering on the + # single field updated_at needs no composite index. + # + # The AI-system filter runs in Python (it's a membership test against a set + # the flags decide), so the window has to be wider than `limit` or filtering + # could empty it. 10x with a floor of 200 covers a debug view; if a fetch + # still comes back short, incidents_window_exhausted says so in the payload + # rather than quietly looking like "no incidents". + window = max(limit * 10, 200) + all_incidents = await fstore.collection_where( + "incidents", [], + order_by=[("updated_at", "DESCENDING")], + limit_to=window, + ) ai_incidents = [ i for i in all_incidents if any(sid in ai_systems for sid in (i.get("system_ids") or [])) ] incidents = ai_incidents[:limit] + incidents_window_exhausted = len(all_incidents) >= window and len(ai_incidents) < limit # ── Fetch all linked call docs in parallel ──────────────────────────────── all_call_ids: list[str] = [] @@ -130,9 +146,17 @@ async def debug_correlation( # Use a single-field range query to avoid requiring a composite Firestore index; # filter status and system in Python. cutoff = datetime.now(timezone.utc) - timedelta(hours=orphan_hours) - recent_calls = await fstore.collection_where("calls", [ - ("ended_at", ">=", cutoff), - ]) + # Bounded for the same reason as the incident read above. The range and the + # sort are both on ended_at, which is what keeps this a single-field query + # needing no composite index. + _ORPHAN_SCAN_CAP = 3000 + recent_calls = await fstore.collection_where( + "calls", + [("ended_at", ">=", cutoff)], + order_by=[("ended_at", "DESCENDING")], + limit_to=_ORPHAN_SCAN_CAP, + ) + orphan_scan_truncated = len(recent_calls) >= _ORPHAN_SCAN_CAP orphans = [ _call_summary(c) for c in recent_calls if c.get("status") == "ended" @@ -162,6 +186,11 @@ async def debug_correlation( return { "generated_at": datetime.now(timezone.utc).isoformat(), + # Both reads are capped, so say plainly when a cap was hit — otherwise a + # truncated window is indistinguishable from a quiet night. + "incidents_window_exhausted": incidents_window_exhausted, + "orphan_scan_truncated": orphan_scan_truncated, + "orphan_scan_cap": _ORPHAN_SCAN_CAP, "incident_count": len(incident_records), "orphaned_call_count": len(orphans), "orphans_by_talkgroup": sorted(orphans_by_tg.values(), key=lambda x: x["count"], reverse=True),