diff --git a/drb-c2-core/app/internal/transcript_correction.py b/drb-c2-core/app/internal/transcript_correction.py index b5d0809..0359aa6 100644 --- a/drb-c2-core/app/internal/transcript_correction.py +++ b/drb-c2-core/app/internal/transcript_correction.py @@ -28,10 +28,23 @@ counties may have one talkgroup covering a single municipality, and that municipality's streets must not be buried under a county-wide list. A single-municipality system is the degenerate case: populate the system level and every talkgroup inherits it. + +THE PROMPT'S OWN RULES ARE NOT ENFORCED (server-26#162). "Do NOT expand +ten-codes" and "NEVER add information" are instructions to the model, not +checks on its output — `correct()` used to accept `raw["corrected"]` verbatim. +Caught live: the same call came back with "10-7" rewritten to "10-13" in one +place and "10-4" in another, and "7" expanded into "ShotSpotter" — a real code +swapped for a different real code reads exactly as confident and trustworthy +as a correct one, which is worse than leaving the raw mishearing in place. The +model isn't graded on this at write time; `_code_tokens()` is a +verify-what-you-can-cheaply-check backstop, not a fix to the model's judgment: +it only catches a code-shaped token changing, not a wrong word substituted for +another equally plausible word. """ import asyncio import json +import re from typing import Any, Optional from app.config import settings @@ -96,6 +109,19 @@ def _dedupe(items: list[str]) -> list[str]: return out +# Ten-codes ("10-4"), unit/signal shorthand ("4-2"), and the digit-group +# fragments radio traffic reads out loud ("7-2-1" of a case number) all share +# this shape. The guard below does not need to know which of those a given +# token is — it only needs the SET of them to survive a "correction" +# unchanged, in order. A model rewriting "10-7" as "10-13" is not the kind of +# mishearing this pass exists to fix (server-26#162). +_CODE_TOKEN_RE = re.compile(r"\b\d{1,3}(?:-\d{1,3})+\b") + + +def _code_tokens(text: str) -> list[str]: + return _CODE_TOKEN_RE.findall(text or "") + + def _talkgroup_entry(system_doc: dict, talkgroup_id: Optional[int]) -> dict: """The config.talkgroups[] entry for this talkgroup, or {}.""" if talkgroup_id is None: @@ -317,6 +343,31 @@ async def correct( if verified_segments: corrected_segments = verified_segments + # server-26#162: a code-shaped token ("10-7", "4-2", a case-number + # fragment like "7-2-1") changing at all — not just going missing, any + # change — means the model touched something this pass has no business + # touching. Reject that half of the correction outright rather than trust + # a rewrite that already broke its own instructions once. Checked against + # the ORIGINAL text/segment, not each other, so a joined-text correction + # and a segment correction are judged independently, same as everywhere + # else in this function. + if corrected is not None and _code_tokens(corrected) != _code_tokens(text): + logger.warning( + f"Transcript correction for call {call_id} changed code-shaped " + f"tokens ({_code_tokens(text)} -> {_code_tokens(corrected)}) — " + f"discarding the joined correction" + ) + corrected = None + if corrected_segments is not None: + for seg, orig in zip(corrected_segments, segments or []): + if _code_tokens(seg["text"]) != _code_tokens(orig.get("text", "")): + logger.warning( + f"Transcript correction for call {call_id} changed " + f"code-shaped tokens in a segment — discarding segment corrections" + ) + corrected_segments = None + break + if corrected or corrected_segments or not_speech: changed = raw.get("changed") or [] logger.info( diff --git a/drb-c2-core/tests/test_transcript_correction.py b/drb-c2-core/tests/test_transcript_correction.py index 40eda3b..060328e 100644 --- a/drb-c2-core/tests/test_transcript_correction.py +++ b/drb-c2-core/tests/test_transcript_correction.py @@ -211,6 +211,60 @@ async def test_model_failure_leaves_the_transcript_alone(): assert await tc.correct("c1", "x y z w", SEGS, system_id="sys-1") == (None, None, False) +# ── Code-token guard (server-26#162) ──────────────────────────────────────── +# Caught live: the same call came back with "10-7" rewritten to "10-13" in one +# place and "10-4" in another. A real code swapped for a different real code +# reads exactly as trustworthy as a correct one — worse than leaving the raw +# mishearing in place, since nothing downstream can tell it happened. + +@pytest.mark.asyncio +async def test_changed_ten_code_is_discarded(): + payload = {"corrected": "10-13, we're back in town."} + with _system(), _gemini(payload): + text, _, _ = await tc.correct("c1", "10-7, we're back in town.", None, system_id="sys-1") + assert text is None + + +@pytest.mark.asyncio +async def test_invented_code_token_is_discarded(): + """Nothing code-shaped in the original — the model added one from nothing.""" + payload = {"corrected": "ShotSpotter, 10-4, group of 3 shooting outside."} + with _system(), _gemini(payload): + text, _, _ = await tc.correct("c1", "Seven, group of 3 shooting outside.", None, system_id="sys-1") + assert text is None + + +@pytest.mark.asyncio +async def test_legitimate_place_correction_with_unchanged_codes_still_applies(): + """The guard must not collateral-damage a correction that never touches + a code token — Home/Forest for Holmes/4th-and-Rowe is exactly the kind of + fix this pass exists to make.""" + payload = {"corrected": "10-13 coming over on Home Street and Forest Ave, 4-2."} + with _system(), _gemini(payload): + text, _, _ = await tc.correct( + "c1", "10-13 coming over on Holmes Street and 4th and Rowe, 4-2.", + None, system_id="sys-1", + ) + assert text == "10-13 coming over on Home Street and Forest Ave, 4-2." + + +@pytest.mark.asyncio +async def test_segment_code_change_discards_segments_only(): + """A code change in one segment discards the whole segments array (same + all-or-nothing rule as a length mismatch), but the independently-checked + joined correction still stands if it kept its own codes intact. The + joined `text`/`corrected` pair here is deliberately code-free — this test + isolates the segment-level guard, not the joined-text one.""" + payload = { + "corrected": "Show it out to Ossining, back to Route 9.", + "segments": ["Headquarters, 10-13.", "Show it out to Ossining.", "360 north, back to Route 9."], + } + with _system(), _gemini(payload): + text, segs, _ = await tc.correct("c1", "x y z w", SEGS, system_id="sys-1", talkgroup_id=9048) + assert segs is None + assert text == "Show it out to Ossining, back to Route 9." + + @pytest.mark.asyncio async def test_reference_data_reaches_the_prompt(): seen = {}