Merge pull request 'intelligence: shadow-mode upstream dispatch-vs-chatter classifier (#127)' (#128) from feat/115-chatter-classifier-shadow-mode into main

drb-c2-core/app/internal/chatter_classifier.py

@@ -0,0 +1,135 @@
+"""
+Upstream dispatch-vs-chatter classifier — SHADOW MODE (server-26#115 follow-up).
+
+Three live measurement windows (CORRELATION_REVIEW_0907.md, _0907b.md, _0912.md)
+and two consensus-layer fixes (#125, #126) all converged on the same conclusion:
+the actual non-event-promotion problem lives upstream of correlation entirely.
+Radio housekeeping — unit check-ins, roll call, bare 10-4/10-8/98 acknowledgements
+— has no incident content for `intelligence.extract_scenes` to find, but nothing
+stops it from being sent to the scene-extraction LLM and coming out the other end
+as a thin "scene" for the correlator to then judge. See CORRELATION_REVIEW_0912.md
+("Reminder: the real fix is still unscoped") and issue #115.
+
+This module is that classifier. It is a PURE function of the transcript text —
+no Firestore, no LLM call, no side effects — so it is cheap to run on every
+transcript and cheap to test against real dumps offline.
+
+SHADOW MODE ONLY. As of this module's introduction, nothing skips scene
+extraction based on this verdict. `intelligence.extract_scenes` calls
+`classify_chatter` purely to record the verdict on the call doc
+(`chatter_classifier_verdict` / `chatter_classifier_reason`) so it becomes
+observable in the next `/admin` correlation-debug dump, exactly like
+`corr_gate_veto` (server-26#115 / PR #126). See the TODO at that call site for
+what has to be true before this flips live.
+
+Precision over recall, deliberately. A false positive here — flagging a REAL
+event as chatter — would, once live, silently mean that event never gets a
+scene, never gets tags/location/severity, and never has a chance to become an
+incident. That is a much bigger, harder-to-notice failure than a false
+negative (a housekeeping call that still goes through the existing expensive
+pipeline and gets judged "not an incident" the same way it is today). When a
+transcript doesn't clearly match one of the shapes below, this returns
+(False, None) and the existing pipeline runs exactly as it does today.
+
+Patterns are drawn from hand-labeled examples in CORRELATION_REVIEW_0907b.md
+and CORRELATION_REVIEW_0912.md, cross-referenced against the real transcripts
+in corr_dump_9-7_0437am.json / corr_dump_9-7_pm.json / corr_dump_9-12.json —
+not invented regexes. See the backtest script referenced in the PR for the
+per-dump catch rate and false-positive count.
+"""
+import re
+from typing import Optional
+
+# Police/law-enforcement phonetic alphabet words (APCO + NATO). Deliberately
+# duplicated from intelligence.py's `_PHONETIC_ALPHA_WORDS` rather than
+# imported — intelligence.py imports this module (to write the shadow-mode
+# verdict onto the call doc), so importing back would be circular. Keep the
+# two sets in sync if either changes; they're small and rarely touched.
+_PHONETIC_ALPHA_WORDS = frozenset({
+    # APCO (law enforcement)
+    "adam", "baker", "charles", "david", "edward", "frank", "george", "henry",
+    "ida", "john", "king", "lincoln", "mary", "nora", "ocean", "paul", "queen",
+    "robert", "sam", "tom", "union", "victor", "william", "x-ray", "young", "zebra",
+    # NATO
+    "alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel",
+    "india", "juliet", "kilo", "lima", "mike", "november", "oscar", "papa",
+    "quebec", "romeo", "sierra", "tango", "uniform", "whiskey", "yankee", "zulu",
+})
+
+_TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9\-]*")
+
+# Bare radio-procedure words that carry zero incident content by themselves.
+# Deliberately small and literal — this is not a general stopword list, it's
+# the exact vocabulary observed in hand-labeled chatter transcripts. Words
+# that are ambiguous outside a pure-procedure context (e.g. "location",
+# "call", "phone", "number", "go") are left OUT on purpose: including them
+# risks reducing a real, substantive transcript down to nothing.
+_FILLER_WORDS = frozenset({
+    "to", "this", "is", "the", "a", "and", "for", "you", "can", "i", "in",
+    "on", "of", "that", "just", "from", "out", "ok", "okay", "at", "be",
+    "show", "me", "mark", "marked", "charge", "standby", "stand", "by",
+    "clear", "available", "affirm", "affirmative", "negative", "copy",
+    "copies", "received", "roger",
+})
+
+# Agency/procedural designators — who's being addressed, not what happened.
+_RADIO_DESIGNATORS = frozenset({
+    "central", "dispatch", "headquarters", "hq", "post", "unit", "sergeant",
+    "sgt", "metro", "mta", "division", "county",
+})
+
+_ROLL_CALL_RE = re.compile(r"\broll\s*call\b")
+
+
+def _tokenize(transcript: str) -> list[str]:
+    return _TOKEN_RE.findall(transcript.lower())
+
+
+def _is_filler_token(token: str) -> bool:
+    # Any token starting with a digit is a unit ID, 10-code, badge/post
+    # number, or call-number fragment ("10-4", "6-8", "72-holland",
+    # "11-victor", "98", "114") — procedural, not incident content. This is
+    # deliberately broad: a real event transcript that happens to include a
+    # digit-led token (an address number, a case number) still has other,
+    # non-digit descriptive words left over, so this alone never reduces a
+    # real transcript to nothing. See the backtest for confirmation.
+    if token[0].isdigit():
+        return True
+    return (
+        token in _FILLER_WORDS
+        or token in _RADIO_DESIGNATORS
+        or token in _PHONETIC_ALPHA_WORDS
+    )
+
+
+def classify_chatter(transcript: Optional[str]) -> tuple[bool, Optional[str]]:
+    """
+    Pure classification of a transcript as non-event radio housekeeping.
+
+    Returns (is_chatter, reason):
+      (True,  "roll_call")            — contains a roll-call announcement
+      (True,  "bare_acknowledgement") — every token is a callsign/10-code/
+                                         procedural filler word; nothing else
+      (False, None)                   — not confidently chatter; let the
+                                         existing pipeline run as today
+
+    Takes only the transcript. Other call metadata (talkgroup, severity, tags)
+    doesn't exist yet at the point this needs to run — this classifier is
+    upstream of the scene-extraction call that produces those fields — so it
+    deliberately doesn't take them as input.
+    """
+    if not transcript or not transcript.strip():
+        return False, None
+
+    lowered = transcript.lower()
+    if _ROLL_CALL_RE.search(lowered):
+        return True, "roll_call"
+
+    tokens = _tokenize(transcript)
+    if not tokens:
+        return False, None
+
+    if any(not _is_filler_token(t) for t in tokens):
+        return False, None
+
+    return True, "bare_acknowledgement"

drb-c2-core/app/internal/intelligence.py

@@ -16,6 +16,7 @@ from typing import Optional
 from app.internal.logger import logger
 from app.internal import firestore as fstore
 from app.internal import area_context
+from app.internal.chatter_classifier import classify_chatter
 # Location validity is defined once, by the module that owns the incident's
 # location/pin invariant.  incident_correlator does not import this module, so
 # this is not a cycle.
@@ -199,6 +200,28 @@ async def extract_scenes(
             pass
         return []
 
+    # server-26#127 — SHADOW MODE ONLY. Computes whether this transcript looks
+    # like non-event radio housekeeping (roll call, bare 10-4/10-8/98
+    # acknowledgements, unit check-ins) and records the verdict on the call
+    # doc, but does NOT skip extraction anywhere below — every path runs
+    # exactly as it did before this landed. Deliberately ahead of the ≤5-word
+    # skip: most bare acknowledgements ARE ≤5 words, and the first pass of
+    # this feature put the classifier after that return, so it never saw the
+    # bulk of its own target population — a review backtest against three
+    # live dumps found 82% of what it would have flagged already exits above
+    # as transcript_too_short, meaning a shadow-mode window would have shown
+    # roughly a fifth of the real catch rate. Computing it once, here, and
+    # folding the result into whichever skip/continue path runs below fixes
+    # that without adding a second Firestore write.
+    # TODO(server-26#127): flip this from shadow to live (skip extraction and
+    # write skip_reason="non_event_chatter" instead of just recording the
+    # verdict) once a live shadow-mode window confirms 0 false positives on
+    # real production traffic — pay particular attention to whole-transcript
+    # vs contains-anywhere matching for "roll call" and to digit-hyphen street
+    # addresses (e.g. "72-Holland"), both flagged as classifier risks that the
+    # dump backtest could not surface on its own.
+    chatter_is_chatter, chatter_reason = classify_chatter(transcript)
+
     # Transcripts with ≤5 words carry no extractable intelligence — GPT hallucinates
     # units and tags from thin context (e.g. "Main Lot", "10-4", "David").
     if len(transcript.split()) <= 5:
@@ -213,11 +236,21 @@ async def extract_scenes(
             await fstore.doc_set("calls", call_id, {
                 "skip_reason": "transcript_too_short",
                 "severity":    "routine",
+                "chatter_classifier_verdict": chatter_is_chatter,
+                "chatter_classifier_reason":  chatter_reason,
             })
         except Exception:
             pass
         return []
 
+    try:
+        await fstore.doc_set("calls", call_id, {
+            "chatter_classifier_verdict": chatter_is_chatter,
+            "chatter_classifier_reason":  chatter_reason,
+        })
+    except Exception:
+        pass
+
     raw_scenes: list[dict] = await asyncio.to_thread(
         _sync_extract,
         transcript, talkgroup_name, talkgroup_id, system_id, segments, vocabulary, ten_codes,

drb-c2-core/app/routers/admin.py

@@ -141,6 +141,13 @@ async def debug_correlation(
             # written here specifically so a live measurement window can read
             # the reason instead of reconstructing it by hand from the dump.
             "corr_gate_veto":        call.get("corr_gate_veto"),
+            # server-26#127 — shadow-mode upstream chatter classifier verdict.
+            # Written by intelligence.extract_scenes on every transcript that
+            # reaches real scene extraction (not on garbage/too-short skips).
+            # Nothing skips extraction on this yet — it's here purely so a
+            # live measurement window can read the false-positive rate.
+            "chatter_classifier_verdict": call.get("chatter_classifier_verdict"),
+            "chatter_classifier_reason":  call.get("chatter_classifier_reason"),
         }
 
     # ── Determine which systems have AI active ────────────────────────────────
@@ -302,6 +309,17 @@ async def debug_correlation(
         # server-26#115 — this IS the number the escape-hatch fix exists to
         # produce: why each llm=orphan/rules=new call escaped the gate.
         "corr_gate_veto":         _tally(c.get("corr_gate_veto") for c in linked),
+        # server-26#127 — shadow-mode chatter classifier. The target
+        # population is non-events, which land as orphans or single-call
+        # incidents, NOT as a slice of every linked call -- tally `orphans`
+        # too or this undercounts the exact thing the feature measures.
+        "chatter_classifier_flagged": sum(
+            1 for c in (linked + orphans) if c.get("chatter_classifier_verdict")
+        ),
+        "chatter_classifier_reason": _tally(
+            c.get("chatter_classifier_reason") for c in (linked + orphans)
+            if c.get("chatter_classifier_verdict")
+        ),
         # STT coverage: correlation quality is capped by this, so it belongs in
         # the same view rather than a separate investigation.
         "linked_calls_with_transcript": with_transcript,

drb-c2-core/tests/test_chatter_classifier.py

@@ -0,0 +1,144 @@
+"""
+server-26#127 — upstream dispatch-vs-chatter classifier, shadow mode.
+
+Fixtures are real transcripts, not invented ones: pulled from
+`corr_dump_9-7_0437am.json`, `corr_dump_9-7_pm.json`, `corr_dump_9-12.json`
+and the hand-labeled examples in `CORRELATION_REVIEW_0907b.md` /
+`CORRELATION_REVIEW_0912.md`. The "must classify False" set specifically
+includes every transcript those review docs flagged as dangerous to drop —
+a false positive here is a real event silently losing its scene once this
+classifier ever goes live, which is a much worse failure than a missed
+chatter call staying in the existing (already-working) pipeline.
+"""
+import pytest
+
+from app.internal.chatter_classifier import classify_chatter
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Must classify as chatter
+# ─────────────────────────────────────────────────────────────────────────────
+
+CHATTER_EXAMPLES = [
+    # Bare acknowledgements / unit check-ins (CORRELATION_REVIEW_0907b.md)
+    ("114 Paul.\n114 Paul, Metro Central.\n10-4.", "bare_acknowledgement"),
+    ("Affirmative, in charge of 10-8. 10-8, 10-4.", "bare_acknowledgement"),
+    ("6-8, you can show me 98. 10-4.", "bare_acknowledgement"),
+    ("10-4, 10-4 Central, 98. 10-4, 98.", "bare_acknowledgement"),
+    ("7 for Post 1 and 2, 98. Affirm.", "bare_acknowledgement"),
+    ("11-Victor to Central. 11-Victor. 72-Holland, 1-5. Central.", "bare_acknowledgement"),
+    # Roll call (CORRELATION_REVIEW_0907b.md / _0912.md)
+    ("Post 4, Ossining. And to volunteer patrol, stand by for roll call.", "roll_call"),
+    ("Headquarters to all cars, stand by for roll call.", "roll_call"),
+    ("All Troop NYC Patrols, stand by for roll call.", "roll_call"),
+    (
+        "Car 100, roll call.\nHenry 1.\nHenry 1.\nSam 1.\nSam 1.\n45 Baker.\n"
+        "45 Baker.\n11 Adam.\nAdam.\n11 Baker.\nBaker.\nStaff 1.\n1.\nStaff 2.",
+        "roll_call",
+    ),
+    (
+        "Headquarters, all cars on a roll call. Baker 1? Baker 1. Henry 1? "
+        "Henry 1. Sam 2? Sam 2. 11 Adam? 11. 11 Baker? 11 Baker.",
+        "roll_call",
+    ),
+    ("Because all cars came out for roll call.", "roll_call"),
+    ("10-1. KL Cars, that concludes roll call, time is 3-31.", "roll_call"),
+    # Minimal single-word / bare-code transmissions (orphan pool, all 3 dumps)
+    ("10-4.", "bare_acknowledgement"),
+    ("Roger.", "bare_acknowledgement"),
+    ("Clear.", "bare_acknowledgement"),
+    ("Affirmative.", "bare_acknowledgement"),
+    ("Received.", "bare_acknowledgement"),
+    ("10-8, clear. 10-4.", "bare_acknowledgement"),
+    ("Post 4, 10-8. 10-4.", "bare_acknowledgement"),
+    ("Central to 6 Henry.", "bare_acknowledgement"),
+]
+
+
+@pytest.mark.parametrize("transcript,expected_reason", CHATTER_EXAMPLES)
+def test_classifies_chatter(transcript, expected_reason):
+    is_chatter, reason = classify_chatter(transcript)
+    assert is_chatter is True
+    assert reason == expected_reason
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Must NOT classify as chatter — real events, including every transcript the
+# review docs specifically named as dangerous to drop.
+# ─────────────────────────────────────────────────────────────────────────────
+
+REAL_EVENT_EXAMPLES = [
+    # The major "extinguishing fire" call (severity=major, tags=[extinguishing-fire])
+    ("Dispatch, this is 7-4, extinguishing fire.", "extinguishing_fire"),
+    # Geocoded 911-hangup call (has location_coords)
+    (
+        "7, Charlie. Charlie, check and advise, we've got a call for service "
+        "coming over, it's going to be a 9-1-1 hangout, no voice contact. "
+        "Looks like it was an automated message saying it's the Doral Hat Company.",
+        "geocoded_911_hangup",
+    ),
+    # Pursuit updates (severity=major, tags include pursuit / low-speed-pursuit)
+    ("I'm aware of that one. It's a low-speed pursuit. It's refusing to pull over.", "low_speed_pursuit"),
+    (
+        "1. Headquarters to 5-charlie. I'm going to say the last thing to anyone.\n"
+        "2. Info, Sgt. Repeat.\n"
+        "3. The SP is on a pursuit southbound on I-684. It's approaching the airport.\n"
+        "4. Okay, thank you.\n5. 23-59.",
+        "pursuit_i684",
+    ),
+    # "6 Alpha ... Pelham Station" subject check (CORRELATION_REVIEW_0907b.md's
+    # own "genuinely distinct events" list) — looks like a bare check-in but
+    # dispatches a unit to a specific location.
+    (
+        "6 Alpha, this is Central. 7 Alpha here.\n"
+        "6 Alpha, can you show me on scene at Pelham Station? Stand by.",
+        "pelham_station_subject_check",
+    ),
+    # Property-retrieval call (tags=[property-retrieval])
+    (
+        "Property was retrieved with a 911. Can I get a phone number? 10-4. "
+        "Phone number is 214792. 214792.",
+        "property_retrieval",
+    ),
+    # Subject check south of Maronex Station (tags=[subject-check])
+    (
+        "Proceed. Show me on a subject south of Maronex Station. Can I get a "
+        "15 check by New York client ID?",
+        "maronex_subject_check",
+    ),
+    # Trespassing at milepost 13.7 (tags=[trespassing])
+    (
+        "Can you just 10-5 that job? You came over real muffled.\n"
+        "10-4, there's going to be a trespass on the tracks.\n"
+        "Train 8755 reports two juveniles, one male, one female, both wearing "
+        "white shirts, track three side, at milepost 13.7.",
+        "trespass_milepost_13_7",
+    ),
+    # MVA (severity=moderate, tags=[traffic-accident])
+    ("1. Train patrol 9.\n2. MVA 4, how close is it?\n3. 10-4.", "mva"),
+]
+
+
+@pytest.mark.parametrize("transcript,label", REAL_EVENT_EXAMPLES, ids=[l for _, l in REAL_EVENT_EXAMPLES])
+def test_does_not_classify_real_events_as_chatter(transcript, label):
+    is_chatter, reason = classify_chatter(transcript)
+    assert is_chatter is False, f"{label}: false positive, reason={reason!r}"
+    assert reason is None
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Edge cases
+# ─────────────────────────────────────────────────────────────────────────────
+
+def test_empty_transcript_not_chatter():
+    assert classify_chatter("") == (False, None)
+    assert classify_chatter(None) == (False, None)
+    assert classify_chatter("   ") == (False, None)
+
+
+def test_unrecognized_content_defaults_to_not_chatter():
+    # Anything containing real descriptive words the classifier doesn't
+    # recognize must fall through to (False, None), not guess.
+    is_chatter, reason = classify_chatter("Shots fired, officer down, requesting backup immediately.")
+    assert is_chatter is False
+    assert reason is None