diff --git a/drb-c2-core/app/internal/chatter_classifier.py b/drb-c2-core/app/internal/chatter_classifier.py new file mode 100644 index 0000000..81e244d --- /dev/null +++ b/drb-c2-core/app/internal/chatter_classifier.py @@ -0,0 +1,135 @@ +""" +Upstream dispatch-vs-chatter classifier — SHADOW MODE (server-26#115 follow-up). + +Three live measurement windows (CORRELATION_REVIEW_0907.md, _0907b.md, _0912.md) +and two consensus-layer fixes (#125, #126) all converged on the same conclusion: +the actual non-event-promotion problem lives upstream of correlation entirely. +Radio housekeeping — unit check-ins, roll call, bare 10-4/10-8/98 acknowledgements +— has no incident content for `intelligence.extract_scenes` to find, but nothing +stops it from being sent to the scene-extraction LLM and coming out the other end +as a thin "scene" for the correlator to then judge. See CORRELATION_REVIEW_0912.md +("Reminder: the real fix is still unscoped") and issue #115. + +This module is that classifier. It is a PURE function of the transcript text — +no Firestore, no LLM call, no side effects — so it is cheap to run on every +transcript and cheap to test against real dumps offline. + +SHADOW MODE ONLY. As of this module's introduction, nothing skips scene +extraction based on this verdict. `intelligence.extract_scenes` calls +`classify_chatter` purely to record the verdict on the call doc +(`chatter_classifier_verdict` / `chatter_classifier_reason`) so it becomes +observable in the next `/admin` correlation-debug dump, exactly like +`corr_gate_veto` (server-26#115 / PR #126). See the TODO at that call site for +what has to be true before this flips live. + +Precision over recall, deliberately. A false positive here — flagging a REAL +event as chatter — would, once live, silently mean that event never gets a +scene, never gets tags/location/severity, and never has a chance to become an +incident. That is a much bigger, harder-to-notice failure than a false +negative (a housekeeping call that still goes through the existing expensive +pipeline and gets judged "not an incident" the same way it is today). When a +transcript doesn't clearly match one of the shapes below, this returns +(False, None) and the existing pipeline runs exactly as it does today. + +Patterns are drawn from hand-labeled examples in CORRELATION_REVIEW_0907b.md +and CORRELATION_REVIEW_0912.md, cross-referenced against the real transcripts +in corr_dump_9-7_0437am.json / corr_dump_9-7_pm.json / corr_dump_9-12.json — +not invented regexes. See the backtest script referenced in the PR for the +per-dump catch rate and false-positive count. +""" +import re +from typing import Optional + +# Police/law-enforcement phonetic alphabet words (APCO + NATO). Deliberately +# duplicated from intelligence.py's `_PHONETIC_ALPHA_WORDS` rather than +# imported — intelligence.py imports this module (to write the shadow-mode +# verdict onto the call doc), so importing back would be circular. Keep the +# two sets in sync if either changes; they're small and rarely touched. +_PHONETIC_ALPHA_WORDS = frozenset({ + # APCO (law enforcement) + "adam", "baker", "charles", "david", "edward", "frank", "george", "henry", + "ida", "john", "king", "lincoln", "mary", "nora", "ocean", "paul", "queen", + "robert", "sam", "tom", "union", "victor", "william", "x-ray", "young", "zebra", + # NATO + "alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", + "india", "juliet", "kilo", "lima", "mike", "november", "oscar", "papa", + "quebec", "romeo", "sierra", "tango", "uniform", "whiskey", "yankee", "zulu", +}) + +_TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9\-]*") + +# Bare radio-procedure words that carry zero incident content by themselves. +# Deliberately small and literal — this is not a general stopword list, it's +# the exact vocabulary observed in hand-labeled chatter transcripts. Words +# that are ambiguous outside a pure-procedure context (e.g. "location", +# "call", "phone", "number", "go") are left OUT on purpose: including them +# risks reducing a real, substantive transcript down to nothing. +_FILLER_WORDS = frozenset({ + "to", "this", "is", "the", "a", "and", "for", "you", "can", "i", "in", + "on", "of", "that", "just", "from", "out", "ok", "okay", "at", "be", + "show", "me", "mark", "marked", "charge", "standby", "stand", "by", + "clear", "available", "affirm", "affirmative", "negative", "copy", + "copies", "received", "roger", +}) + +# Agency/procedural designators — who's being addressed, not what happened. +_RADIO_DESIGNATORS = frozenset({ + "central", "dispatch", "headquarters", "hq", "post", "unit", "sergeant", + "sgt", "metro", "mta", "division", "county", +}) + +_ROLL_CALL_RE = re.compile(r"\broll\s*call\b") + + +def _tokenize(transcript: str) -> list[str]: + return _TOKEN_RE.findall(transcript.lower()) + + +def _is_filler_token(token: str) -> bool: + # Any token starting with a digit is a unit ID, 10-code, badge/post + # number, or call-number fragment ("10-4", "6-8", "72-holland", + # "11-victor", "98", "114") — procedural, not incident content. This is + # deliberately broad: a real event transcript that happens to include a + # digit-led token (an address number, a case number) still has other, + # non-digit descriptive words left over, so this alone never reduces a + # real transcript to nothing. See the backtest for confirmation. + if token[0].isdigit(): + return True + return ( + token in _FILLER_WORDS + or token in _RADIO_DESIGNATORS + or token in _PHONETIC_ALPHA_WORDS + ) + + +def classify_chatter(transcript: Optional[str]) -> tuple[bool, Optional[str]]: + """ + Pure classification of a transcript as non-event radio housekeeping. + + Returns (is_chatter, reason): + (True, "roll_call") — contains a roll-call announcement + (True, "bare_acknowledgement") — every token is a callsign/10-code/ + procedural filler word; nothing else + (False, None) — not confidently chatter; let the + existing pipeline run as today + + Takes only the transcript. Other call metadata (talkgroup, severity, tags) + doesn't exist yet at the point this needs to run — this classifier is + upstream of the scene-extraction call that produces those fields — so it + deliberately doesn't take them as input. + """ + if not transcript or not transcript.strip(): + return False, None + + lowered = transcript.lower() + if _ROLL_CALL_RE.search(lowered): + return True, "roll_call" + + tokens = _tokenize(transcript) + if not tokens: + return False, None + + if any(not _is_filler_token(t) for t in tokens): + return False, None + + return True, "bare_acknowledgement" diff --git a/drb-c2-core/app/internal/intelligence.py b/drb-c2-core/app/internal/intelligence.py index d62a0e1..49e6578 100644 --- a/drb-c2-core/app/internal/intelligence.py +++ b/drb-c2-core/app/internal/intelligence.py @@ -16,6 +16,7 @@ from typing import Optional from app.internal.logger import logger from app.internal import firestore as fstore from app.internal import area_context +from app.internal.chatter_classifier import classify_chatter # Location validity is defined once, by the module that owns the incident's # location/pin invariant. incident_correlator does not import this module, so # this is not a cycle. @@ -199,6 +200,28 @@ async def extract_scenes( pass return [] + # server-26#127 — SHADOW MODE ONLY. Computes whether this transcript looks + # like non-event radio housekeeping (roll call, bare 10-4/10-8/98 + # acknowledgements, unit check-ins) and records the verdict on the call + # doc, but does NOT skip extraction anywhere below — every path runs + # exactly as it did before this landed. Deliberately ahead of the ≤5-word + # skip: most bare acknowledgements ARE ≤5 words, and the first pass of + # this feature put the classifier after that return, so it never saw the + # bulk of its own target population — a review backtest against three + # live dumps found 82% of what it would have flagged already exits above + # as transcript_too_short, meaning a shadow-mode window would have shown + # roughly a fifth of the real catch rate. Computing it once, here, and + # folding the result into whichever skip/continue path runs below fixes + # that without adding a second Firestore write. + # TODO(server-26#127): flip this from shadow to live (skip extraction and + # write skip_reason="non_event_chatter" instead of just recording the + # verdict) once a live shadow-mode window confirms 0 false positives on + # real production traffic — pay particular attention to whole-transcript + # vs contains-anywhere matching for "roll call" and to digit-hyphen street + # addresses (e.g. "72-Holland"), both flagged as classifier risks that the + # dump backtest could not surface on its own. + chatter_is_chatter, chatter_reason = classify_chatter(transcript) + # Transcripts with ≤5 words carry no extractable intelligence — GPT hallucinates # units and tags from thin context (e.g. "Main Lot", "10-4", "David"). if len(transcript.split()) <= 5: @@ -213,11 +236,21 @@ async def extract_scenes( await fstore.doc_set("calls", call_id, { "skip_reason": "transcript_too_short", "severity": "routine", + "chatter_classifier_verdict": chatter_is_chatter, + "chatter_classifier_reason": chatter_reason, }) except Exception: pass return [] + try: + await fstore.doc_set("calls", call_id, { + "chatter_classifier_verdict": chatter_is_chatter, + "chatter_classifier_reason": chatter_reason, + }) + except Exception: + pass + raw_scenes: list[dict] = await asyncio.to_thread( _sync_extract, transcript, talkgroup_name, talkgroup_id, system_id, segments, vocabulary, ten_codes, diff --git a/drb-c2-core/app/routers/admin.py b/drb-c2-core/app/routers/admin.py index 815b6ee..bebfec8 100644 --- a/drb-c2-core/app/routers/admin.py +++ b/drb-c2-core/app/routers/admin.py @@ -141,6 +141,13 @@ async def debug_correlation( # written here specifically so a live measurement window can read # the reason instead of reconstructing it by hand from the dump. "corr_gate_veto": call.get("corr_gate_veto"), + # server-26#127 — shadow-mode upstream chatter classifier verdict. + # Written by intelligence.extract_scenes on every transcript that + # reaches real scene extraction (not on garbage/too-short skips). + # Nothing skips extraction on this yet — it's here purely so a + # live measurement window can read the false-positive rate. + "chatter_classifier_verdict": call.get("chatter_classifier_verdict"), + "chatter_classifier_reason": call.get("chatter_classifier_reason"), } # ── Determine which systems have AI active ──────────────────────────────── @@ -302,6 +309,17 @@ async def debug_correlation( # server-26#115 — this IS the number the escape-hatch fix exists to # produce: why each llm=orphan/rules=new call escaped the gate. "corr_gate_veto": _tally(c.get("corr_gate_veto") for c in linked), + # server-26#127 — shadow-mode chatter classifier. The target + # population is non-events, which land as orphans or single-call + # incidents, NOT as a slice of every linked call -- tally `orphans` + # too or this undercounts the exact thing the feature measures. + "chatter_classifier_flagged": sum( + 1 for c in (linked + orphans) if c.get("chatter_classifier_verdict") + ), + "chatter_classifier_reason": _tally( + c.get("chatter_classifier_reason") for c in (linked + orphans) + if c.get("chatter_classifier_verdict") + ), # STT coverage: correlation quality is capped by this, so it belongs in # the same view rather than a separate investigation. "linked_calls_with_transcript": with_transcript, diff --git a/drb-c2-core/tests/test_chatter_classifier.py b/drb-c2-core/tests/test_chatter_classifier.py new file mode 100644 index 0000000..174c72b --- /dev/null +++ b/drb-c2-core/tests/test_chatter_classifier.py @@ -0,0 +1,144 @@ +""" +server-26#127 — upstream dispatch-vs-chatter classifier, shadow mode. + +Fixtures are real transcripts, not invented ones: pulled from +`corr_dump_9-7_0437am.json`, `corr_dump_9-7_pm.json`, `corr_dump_9-12.json` +and the hand-labeled examples in `CORRELATION_REVIEW_0907b.md` / +`CORRELATION_REVIEW_0912.md`. The "must classify False" set specifically +includes every transcript those review docs flagged as dangerous to drop — +a false positive here is a real event silently losing its scene once this +classifier ever goes live, which is a much worse failure than a missed +chatter call staying in the existing (already-working) pipeline. +""" +import pytest + +from app.internal.chatter_classifier import classify_chatter + + +# ───────────────────────────────────────────────────────────────────────────── +# Must classify as chatter +# ───────────────────────────────────────────────────────────────────────────── + +CHATTER_EXAMPLES = [ + # Bare acknowledgements / unit check-ins (CORRELATION_REVIEW_0907b.md) + ("114 Paul.\n114 Paul, Metro Central.\n10-4.", "bare_acknowledgement"), + ("Affirmative, in charge of 10-8. 10-8, 10-4.", "bare_acknowledgement"), + ("6-8, you can show me 98. 10-4.", "bare_acknowledgement"), + ("10-4, 10-4 Central, 98. 10-4, 98.", "bare_acknowledgement"), + ("7 for Post 1 and 2, 98. Affirm.", "bare_acknowledgement"), + ("11-Victor to Central. 11-Victor. 72-Holland, 1-5. Central.", "bare_acknowledgement"), + # Roll call (CORRELATION_REVIEW_0907b.md / _0912.md) + ("Post 4, Ossining. And to volunteer patrol, stand by for roll call.", "roll_call"), + ("Headquarters to all cars, stand by for roll call.", "roll_call"), + ("All Troop NYC Patrols, stand by for roll call.", "roll_call"), + ( + "Car 100, roll call.\nHenry 1.\nHenry 1.\nSam 1.\nSam 1.\n45 Baker.\n" + "45 Baker.\n11 Adam.\nAdam.\n11 Baker.\nBaker.\nStaff 1.\n1.\nStaff 2.", + "roll_call", + ), + ( + "Headquarters, all cars on a roll call. Baker 1? Baker 1. Henry 1? " + "Henry 1. Sam 2? Sam 2. 11 Adam? 11. 11 Baker? 11 Baker.", + "roll_call", + ), + ("Because all cars came out for roll call.", "roll_call"), + ("10-1. KL Cars, that concludes roll call, time is 3-31.", "roll_call"), + # Minimal single-word / bare-code transmissions (orphan pool, all 3 dumps) + ("10-4.", "bare_acknowledgement"), + ("Roger.", "bare_acknowledgement"), + ("Clear.", "bare_acknowledgement"), + ("Affirmative.", "bare_acknowledgement"), + ("Received.", "bare_acknowledgement"), + ("10-8, clear. 10-4.", "bare_acknowledgement"), + ("Post 4, 10-8. 10-4.", "bare_acknowledgement"), + ("Central to 6 Henry.", "bare_acknowledgement"), +] + + +@pytest.mark.parametrize("transcript,expected_reason", CHATTER_EXAMPLES) +def test_classifies_chatter(transcript, expected_reason): + is_chatter, reason = classify_chatter(transcript) + assert is_chatter is True + assert reason == expected_reason + + +# ───────────────────────────────────────────────────────────────────────────── +# Must NOT classify as chatter — real events, including every transcript the +# review docs specifically named as dangerous to drop. +# ───────────────────────────────────────────────────────────────────────────── + +REAL_EVENT_EXAMPLES = [ + # The major "extinguishing fire" call (severity=major, tags=[extinguishing-fire]) + ("Dispatch, this is 7-4, extinguishing fire.", "extinguishing_fire"), + # Geocoded 911-hangup call (has location_coords) + ( + "7, Charlie. Charlie, check and advise, we've got a call for service " + "coming over, it's going to be a 9-1-1 hangout, no voice contact. " + "Looks like it was an automated message saying it's the Doral Hat Company.", + "geocoded_911_hangup", + ), + # Pursuit updates (severity=major, tags include pursuit / low-speed-pursuit) + ("I'm aware of that one. It's a low-speed pursuit. It's refusing to pull over.", "low_speed_pursuit"), + ( + "1. Headquarters to 5-charlie. I'm going to say the last thing to anyone.\n" + "2. Info, Sgt. Repeat.\n" + "3. The SP is on a pursuit southbound on I-684. It's approaching the airport.\n" + "4. Okay, thank you.\n5. 23-59.", + "pursuit_i684", + ), + # "6 Alpha ... Pelham Station" subject check (CORRELATION_REVIEW_0907b.md's + # own "genuinely distinct events" list) — looks like a bare check-in but + # dispatches a unit to a specific location. + ( + "6 Alpha, this is Central. 7 Alpha here.\n" + "6 Alpha, can you show me on scene at Pelham Station? Stand by.", + "pelham_station_subject_check", + ), + # Property-retrieval call (tags=[property-retrieval]) + ( + "Property was retrieved with a 911. Can I get a phone number? 10-4. " + "Phone number is 214792. 214792.", + "property_retrieval", + ), + # Subject check south of Maronex Station (tags=[subject-check]) + ( + "Proceed. Show me on a subject south of Maronex Station. Can I get a " + "15 check by New York client ID?", + "maronex_subject_check", + ), + # Trespassing at milepost 13.7 (tags=[trespassing]) + ( + "Can you just 10-5 that job? You came over real muffled.\n" + "10-4, there's going to be a trespass on the tracks.\n" + "Train 8755 reports two juveniles, one male, one female, both wearing " + "white shirts, track three side, at milepost 13.7.", + "trespass_milepost_13_7", + ), + # MVA (severity=moderate, tags=[traffic-accident]) + ("1. Train patrol 9.\n2. MVA 4, how close is it?\n3. 10-4.", "mva"), +] + + +@pytest.mark.parametrize("transcript,label", REAL_EVENT_EXAMPLES, ids=[l for _, l in REAL_EVENT_EXAMPLES]) +def test_does_not_classify_real_events_as_chatter(transcript, label): + is_chatter, reason = classify_chatter(transcript) + assert is_chatter is False, f"{label}: false positive, reason={reason!r}" + assert reason is None + + +# ───────────────────────────────────────────────────────────────────────────── +# Edge cases +# ───────────────────────────────────────────────────────────────────────────── + +def test_empty_transcript_not_chatter(): + assert classify_chatter("") == (False, None) + assert classify_chatter(None) == (False, None) + assert classify_chatter(" ") == (False, None) + + +def test_unrecognized_content_defaults_to_not_chatter(): + # Anything containing real descriptive words the classifier doesn't + # recognize must fall through to (False, None), not guess. + is_chatter, reason = classify_chatter("Shots fired, officer down, requesting backup immediately.") + assert is_chatter is False + assert reason is None