correlator: shrink the same-talkgroup escape hatch from 2h to a few minutes (#115) #126
@@ -112,20 +112,66 @@ async def upload_call_audio(
|
||||
def _recent_incident_on_same_talkgroup(ctx: dict) -> bool:
|
||||
"""
|
||||
True when one of the already-loaded recent incidents is running on this
|
||||
call's own system + talkgroup. Covers the "unit dispatched on the dispatch
|
||||
channel, thin acknowledgement 10-30s later" case: the ack carries no
|
||||
substance of its own but plainly belongs to the job just opened.
|
||||
call's own system + talkgroup AND was active within the last
|
||||
`settings.tg_dispatch_thin_idle_minutes` minutes. Covers the "unit
|
||||
dispatched on the dispatch channel, thin acknowledgement 10-30s later"
|
||||
case: the ack carries no substance of its own but plainly belongs to the
|
||||
job just opened.
|
||||
|
||||
That window intentionally reuses `tg_dispatch_thin_idle_minutes` (5 min)
|
||||
rather than inventing a new constant — it's the same recency bound the
|
||||
fast/thin path already uses for this exact "dispatch, thin ack" scenario
|
||||
(see its tuning note above in config.py), so both places agree on what
|
||||
"just happened on this channel" means.
|
||||
|
||||
This used to be a plain "does any recent incident exist on this
|
||||
talkgroup" check against a 2-hour window (`correlation_window_hours`).
|
||||
Measured live in production (server-26#115, CORRELATION_REVIEW_0912.md,
|
||||
window #3): on a busy dispatch channel producing 3-13 incidents per 2h,
|
||||
that condition is satisfied almost unconditionally, so the surrounding
|
||||
LLM-orphan gate never fired on exactly the channels it exists to
|
||||
protect (0/24 target-shaped calls gated in a 4h window). The docstring's
|
||||
own intent was always "10-30 seconds", not "hours" — a few minutes is
|
||||
the right shape.
|
||||
|
||||
Reads ctx["recent"] — the same window-filtered candidate list the rules
|
||||
engine already loaded — so this adds no Firestore read.
|
||||
|
||||
Known limitation (server-26#115): ctx["recent"] is derived from
|
||||
`all_active` in `_build_context` — incidents with `status=="active"`
|
||||
for the call's org, with over-capacity incidents already dropped by
|
||||
`_drop_capped` — not a full scan of the `incidents` collection. A
|
||||
same-talkgroup incident that has already auto-resolved (no longer
|
||||
"active") or hit `incident_max_calls`/`incident_max_duration_minutes`
|
||||
will NOT appear here even though it is chronologically recent. This is
|
||||
the confirmed explanation for 2/24 gate misses in the window #3
|
||||
measurement where a naive full-collection timestamp scan found no
|
||||
same-talkgroup candidate either once status/capacity are accounted for
|
||||
— i.e. the check here was already correct for those two calls; some
|
||||
other `_call_is_substanceless` condition (severity/substance/type) must
|
||||
have been true instead. A proper fix for the truncation case (an
|
||||
incident that WAS same-talkgroup-recent by clock time but is invisible
|
||||
here because it resolved or capped) needs a dedicated Firestore query
|
||||
that is not status/capacity filtered — a new read, out of scope for
|
||||
this pass.
|
||||
# TODO(server-26#115): add a talkgroup-scoped incident lookup (any
|
||||
# status, no capacity filter) if this escape hatch ever needs to see
|
||||
# resolved/capped incidents rather than just the active candidate pool.
|
||||
"""
|
||||
from app.internal.incident_correlator import _idle_gate_minutes
|
||||
|
||||
tg_id = ctx.get("talkgroup_id")
|
||||
system_id = ctx.get("system_id")
|
||||
if tg_id is None or not system_id:
|
||||
return False
|
||||
tg_str = str(tg_id)
|
||||
now = ctx.get("now") or datetime.now(timezone.utc)
|
||||
for inc in ctx.get("recent") or []:
|
||||
if system_id in (inc.get("system_ids") or []) and tg_str in (inc.get("talkgroup_ids") or []):
|
||||
if system_id not in (inc.get("system_ids") or []):
|
||||
continue
|
||||
if tg_str not in (inc.get("talkgroup_ids") or []):
|
||||
continue
|
||||
if _idle_gate_minutes(inc, now) <= settings.tg_dispatch_thin_idle_minutes:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@@ -156,10 +156,12 @@ async def test_recent_incident_on_same_talkgroup_is_not_gated():
|
||||
ctx = {
|
||||
"system_id": "sys-1",
|
||||
"talkgroup_id": 9048,
|
||||
"now": NOW,
|
||||
"recent": [{
|
||||
"incident_id": "inc-live",
|
||||
"system_ids": ["sys-1"],
|
||||
"talkgroup_ids": ["9048"],
|
||||
"updated_at": (NOW - timedelta(minutes=1)).isoformat(),
|
||||
}],
|
||||
}
|
||||
m_apply, m_tiebreak = await _run_consensus(
|
||||
@@ -168,6 +170,57 @@ async def test_recent_incident_on_same_talkgroup_is_not_gated():
|
||||
m_tiebreak.assert_called_once()
|
||||
|
||||
|
||||
# server-26#115 window #3 (CORRELATION_REVIEW_0912.md): the escape hatch used
|
||||
# to treat ANY same-talkgroup incident inside the 2h correlation_window_hours
|
||||
# as "recent", which on a busy dispatch channel (3-13 incidents/2h) was
|
||||
# satisfied almost unconditionally — the gate fired 0/24 times against its own
|
||||
# target shape. It now only counts an incident as recent within
|
||||
# settings.tg_dispatch_thin_idle_minutes (5 min) — the same bound the
|
||||
# fast/thin path uses for the "dispatch, thin ack" case this escape hatch is
|
||||
# actually for.
|
||||
|
||||
async def test_recent_same_tg_incident_inside_new_short_window_still_escapes_gate():
|
||||
ctx = {
|
||||
"system_id": "sys-1",
|
||||
"talkgroup_id": 9048,
|
||||
"now": NOW,
|
||||
"recent": [{
|
||||
"incident_id": "inc-live",
|
||||
"system_ids": ["sys-1"],
|
||||
"talkgroup_ids": ["9048"],
|
||||
# 3 min ago — inside tg_dispatch_thin_idle_minutes (5).
|
||||
"updated_at": (NOW - timedelta(minutes=3)).isoformat(),
|
||||
}],
|
||||
}
|
||||
m_apply, m_tiebreak = await _run_consensus(
|
||||
_preview("new", {}, ctx=ctx), _llm("orphan"),
|
||||
)
|
||||
m_tiebreak.assert_called_once()
|
||||
|
||||
|
||||
async def test_recent_same_tg_incident_older_than_short_window_now_gates():
|
||||
# Regression test for the fix: this incident is well outside the new
|
||||
# 5-minute window but still inside the OLD 2-hour correlation_window_hours
|
||||
# lookback — before the fix this escaped the gate (tiebreak called);
|
||||
# after the fix it no longer counts as "recent", so the gate fires.
|
||||
ctx = {
|
||||
"system_id": "sys-1",
|
||||
"talkgroup_id": 9048,
|
||||
"now": NOW,
|
||||
"recent": [{
|
||||
"incident_id": "inc-stale",
|
||||
"system_ids": ["sys-1"],
|
||||
"talkgroup_ids": ["9048"],
|
||||
"updated_at": (NOW - timedelta(minutes=45)).isoformat(),
|
||||
}],
|
||||
}
|
||||
m_apply, m_tiebreak = await _run_consensus(
|
||||
_preview("new", {}, ctx=ctx), _llm("orphan"),
|
||||
)
|
||||
m_tiebreak.assert_not_called()
|
||||
assert m_apply.call_args[0][0]["decision"]["action"] == "orphan"
|
||||
|
||||
|
||||
async def test_recent_incident_on_a_different_talkgroup_still_gates():
|
||||
ctx = {
|
||||
"system_id": "sys-1",
|
||||
|
||||
Reference in New Issue
Block a user