intelligence: shadow-mode upstream dispatch-vs-chatter classifier (#127) #128
@@ -200,6 +200,28 @@ async def extract_scenes(
|
||||
pass
|
||||
return []
|
||||
|
||||
# server-26#127 — SHADOW MODE ONLY. Computes whether this transcript looks
|
||||
# like non-event radio housekeeping (roll call, bare 10-4/10-8/98
|
||||
# acknowledgements, unit check-ins) and records the verdict on the call
|
||||
# doc, but does NOT skip extraction anywhere below — every path runs
|
||||
# exactly as it did before this landed. Deliberately ahead of the ≤5-word
|
||||
# skip: most bare acknowledgements ARE ≤5 words, and the first pass of
|
||||
# this feature put the classifier after that return, so it never saw the
|
||||
# bulk of its own target population — a review backtest against three
|
||||
# live dumps found 82% of what it would have flagged already exits above
|
||||
# as transcript_too_short, meaning a shadow-mode window would have shown
|
||||
# roughly a fifth of the real catch rate. Computing it once, here, and
|
||||
# folding the result into whichever skip/continue path runs below fixes
|
||||
# that without adding a second Firestore write.
|
||||
# TODO(server-26#127): flip this from shadow to live (skip extraction and
|
||||
# write skip_reason="non_event_chatter" instead of just recording the
|
||||
# verdict) once a live shadow-mode window confirms 0 false positives on
|
||||
# real production traffic — pay particular attention to whole-transcript
|
||||
# vs contains-anywhere matching for "roll call" and to digit-hyphen street
|
||||
# addresses (e.g. "72-Holland"), both flagged as classifier risks that the
|
||||
# dump backtest could not surface on its own.
|
||||
chatter_is_chatter, chatter_reason = classify_chatter(transcript)
|
||||
|
||||
# Transcripts with ≤5 words carry no extractable intelligence — GPT hallucinates
|
||||
# units and tags from thin context (e.g. "Main Lot", "10-4", "David").
|
||||
if len(transcript.split()) <= 5:
|
||||
@@ -214,23 +236,13 @@ async def extract_scenes(
|
||||
await fstore.doc_set("calls", call_id, {
|
||||
"skip_reason": "transcript_too_short",
|
||||
"severity": "routine",
|
||||
"chatter_classifier_verdict": chatter_is_chatter,
|
||||
"chatter_classifier_reason": chatter_reason,
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
# server-26#127 — SHADOW MODE ONLY. Computes whether this transcript looks
|
||||
# like non-event radio housekeeping (roll call, bare 10-4/10-8/98
|
||||
# acknowledgements, unit check-ins) and records the verdict on the call
|
||||
# doc, but does NOT skip extraction — the scene-extraction call below runs
|
||||
# exactly as it does today regardless of what this says. This is step one
|
||||
# of getting live production data on the classifier's false-positive rate
|
||||
# before trusting it with anything real; see CORRELATION_REVIEW_0912.md.
|
||||
# TODO(server-26#127): flip this from shadow to live (skip extraction and
|
||||
# write skip_reason="non_event_chatter" instead of just recording the
|
||||
# verdict) once a live shadow-mode window confirms 0 false positives on
|
||||
# real production traffic.
|
||||
chatter_is_chatter, chatter_reason = classify_chatter(transcript)
|
||||
try:
|
||||
await fstore.doc_set("calls", call_id, {
|
||||
"chatter_classifier_verdict": chatter_is_chatter,
|
||||
|
||||
@@ -309,6 +309,17 @@ async def debug_correlation(
|
||||
# server-26#115 — this IS the number the escape-hatch fix exists to
|
||||
# produce: why each llm=orphan/rules=new call escaped the gate.
|
||||
"corr_gate_veto": _tally(c.get("corr_gate_veto") for c in linked),
|
||||
# server-26#127 — shadow-mode chatter classifier. The target
|
||||
# population is non-events, which land as orphans or single-call
|
||||
# incidents, NOT as a slice of every linked call -- tally `orphans`
|
||||
# too or this undercounts the exact thing the feature measures.
|
||||
"chatter_classifier_flagged": sum(
|
||||
1 for c in (linked + orphans) if c.get("chatter_classifier_verdict")
|
||||
),
|
||||
"chatter_classifier_reason": _tally(
|
||||
c.get("chatter_classifier_reason") for c in (linked + orphans)
|
||||
if c.get("chatter_classifier_verdict")
|
||||
),
|
||||
# STT coverage: correlation quality is capped by this, so it belongs in
|
||||
# the same view rather than a separate investigation.
|
||||
"linked_calls_with_transcript": with_transcript,
|
||||
|
||||
Reference in New Issue
Block a user