correlator: shrink the same-talkgroup escape hatch from 2h to a few minutes (#115) #126

Merged
logan merged 3 commits from fix/115-escape-hatch-window into main 2026-09-12 04:47:50 -04:00
2 changed files with 103 additions and 4 deletions
Showing only changes of commit 400b74b519 - Show all commits
+50 -4
View File
@@ -112,20 +112,66 @@ async def upload_call_audio(
def _recent_incident_on_same_talkgroup(ctx: dict) -> bool:
"""
True when one of the already-loaded recent incidents is running on this
call's own system + talkgroup. Covers the "unit dispatched on the dispatch
channel, thin acknowledgement 10-30s later" case: the ack carries no
substance of its own but plainly belongs to the job just opened.
call's own system + talkgroup AND was active within the last
`settings.tg_dispatch_thin_idle_minutes` minutes. Covers the "unit
dispatched on the dispatch channel, thin acknowledgement 10-30s later"
case: the ack carries no substance of its own but plainly belongs to the
job just opened.
That window intentionally reuses `tg_dispatch_thin_idle_minutes` (5 min)
rather than inventing a new constant — it's the same recency bound the
fast/thin path already uses for this exact "dispatch, thin ack" scenario
(see its tuning note above in config.py), so both places agree on what
"just happened on this channel" means.
This used to be a plain "does any recent incident exist on this
talkgroup" check against a 2-hour window (`correlation_window_hours`).
Measured live in production (server-26#115, CORRELATION_REVIEW_0912.md,
window #3): on a busy dispatch channel producing 3-13 incidents per 2h,
that condition is satisfied almost unconditionally, so the surrounding
LLM-orphan gate never fired on exactly the channels it exists to
protect (0/24 target-shaped calls gated in a 4h window). The docstring's
own intent was always "10-30 seconds", not "hours" — a few minutes is
the right shape.
Reads ctx["recent"] — the same window-filtered candidate list the rules
engine already loaded — so this adds no Firestore read.
Known limitation (server-26#115): ctx["recent"] is derived from
`all_active` in `_build_context` — incidents with `status=="active"`
for the call's org, with over-capacity incidents already dropped by
`_drop_capped` — not a full scan of the `incidents` collection. A
same-talkgroup incident that has already auto-resolved (no longer
"active") or hit `incident_max_calls`/`incident_max_duration_minutes`
will NOT appear here even though it is chronologically recent. This is
the confirmed explanation for 2/24 gate misses in the window #3
measurement where a naive full-collection timestamp scan found no
same-talkgroup candidate either once status/capacity are accounted for
— i.e. the check here was already correct for those two calls; some
other `_call_is_substanceless` condition (severity/substance/type) must
have been true instead. A proper fix for the truncation case (an
incident that WAS same-talkgroup-recent by clock time but is invisible
here because it resolved or capped) needs a dedicated Firestore query
that is not status/capacity filtered — a new read, out of scope for
this pass.
# TODO(server-26#115): add a talkgroup-scoped incident lookup (any
# status, no capacity filter) if this escape hatch ever needs to see
# resolved/capped incidents rather than just the active candidate pool.
"""
from app.internal.incident_correlator import _idle_gate_minutes
tg_id = ctx.get("talkgroup_id")
system_id = ctx.get("system_id")
if tg_id is None or not system_id:
return False
tg_str = str(tg_id)
now = ctx.get("now") or datetime.now(timezone.utc)
for inc in ctx.get("recent") or []:
if system_id in (inc.get("system_ids") or []) and tg_str in (inc.get("talkgroup_ids") or []):
if system_id not in (inc.get("system_ids") or []):
continue
if tg_str not in (inc.get("talkgroup_ids") or []):
continue
if _idle_gate_minutes(inc, now) <= settings.tg_dispatch_thin_idle_minutes:
return True
return False
+53
View File
@@ -156,10 +156,12 @@ async def test_recent_incident_on_same_talkgroup_is_not_gated():
ctx = {
"system_id": "sys-1",
"talkgroup_id": 9048,
"now": NOW,
"recent": [{
"incident_id": "inc-live",
"system_ids": ["sys-1"],
"talkgroup_ids": ["9048"],
"updated_at": (NOW - timedelta(minutes=1)).isoformat(),
}],
}
m_apply, m_tiebreak = await _run_consensus(
@@ -168,6 +170,57 @@ async def test_recent_incident_on_same_talkgroup_is_not_gated():
m_tiebreak.assert_called_once()
# server-26#115 window #3 (CORRELATION_REVIEW_0912.md): the escape hatch used
# to treat ANY same-talkgroup incident inside the 2h correlation_window_hours
# as "recent", which on a busy dispatch channel (3-13 incidents/2h) was
# satisfied almost unconditionally — the gate fired 0/24 times against its own
# target shape. It now only counts an incident as recent within
# settings.tg_dispatch_thin_idle_minutes (5 min) — the same bound the
# fast/thin path uses for the "dispatch, thin ack" case this escape hatch is
# actually for.
async def test_recent_same_tg_incident_inside_new_short_window_still_escapes_gate():
ctx = {
"system_id": "sys-1",
"talkgroup_id": 9048,
"now": NOW,
"recent": [{
"incident_id": "inc-live",
"system_ids": ["sys-1"],
"talkgroup_ids": ["9048"],
# 3 min ago — inside tg_dispatch_thin_idle_minutes (5).
"updated_at": (NOW - timedelta(minutes=3)).isoformat(),
}],
}
m_apply, m_tiebreak = await _run_consensus(
_preview("new", {}, ctx=ctx), _llm("orphan"),
)
m_tiebreak.assert_called_once()
async def test_recent_same_tg_incident_older_than_short_window_now_gates():
# Regression test for the fix: this incident is well outside the new
# 5-minute window but still inside the OLD 2-hour correlation_window_hours
# lookback — before the fix this escaped the gate (tiebreak called);
# after the fix it no longer counts as "recent", so the gate fires.
ctx = {
"system_id": "sys-1",
"talkgroup_id": 9048,
"now": NOW,
"recent": [{
"incident_id": "inc-stale",
"system_ids": ["sys-1"],
"talkgroup_ids": ["9048"],
"updated_at": (NOW - timedelta(minutes=45)).isoformat(),
}],
}
m_apply, m_tiebreak = await _run_consensus(
_preview("new", {}, ctx=ctx), _llm("orphan"),
)
m_tiebreak.assert_not_called()
assert m_apply.call_args[0][0]["decision"]["action"] == "orphan"
async def test_recent_incident_on_a_different_talkgroup_still_gates():
ctx = {
"system_id": "sys-1",