Logan CusanoandClaude Opus 5
58efdbd6eb
Correction existed, but as a line in intelligence.py's EXTRACTION_PROMPT --
which put it in the wrong place twice over. The same model call that extracted
units, location and severity emitted the correction afterwards, so extraction
reasoned over text already known to be wrong; and it sat behind
correlation_enabled, so during a cost-controlled STT-only window nothing was
ever corrected at all. That is the normal state during development.
internal/transcript_correction.py is now its own pass, between the degenerate
filter and the Firestore write. It receives an already-produced transcript plus
a reference list, so unlike a Whisper prompt it has no series to extend -- the
distinction that keeps vocabulary out of the recogniser's prompt, where an
enumerated ten-code list once made it hallucinate ten-code runs.
Reference data is merged from the talkgroup and the system, TALKGROUP FIRST. A
system spanning several counties can have a talkgroup covering one
municipality, and that municipality's streets must not be buried under a
county-wide list. A single-municipality system is the degenerate case: populate
the system level and every talkgroup inherits it. Area context is now SET --
municipality, county, roads, landmarks, on both scopes -- rather than guessed
from talkgroup names, which is what vocabulary_learner did and which is close
to useless across multiple counties.
Segments are corrected too, not just the joined text. extract_scenes builds its
prompt from numbered segments whenever there is more than one, so a correction
that only fixed the transcript would have been discarded on exactly the
multi-transmission calls carrying the most content. Alignment is enforced: an
array of the wrong length or type is dropped whole, because scenes map back to
transmissions by index and a shifted array would misattribute audio silently.
Whisper is also retried once on degenerate output. Call e49ea32c produced a
56-word ten-code counting run on one attempt and ordinary speech on the next --
same clip, same temperature=0 -- so a hallucination is a coin-flip, and
discarding on the first bad roll threw away a recoverable transcript.
Two things found on the way:
PUT /systems/{id} wiped ten_codes on every save. The systems form sends only
{name, type, config}, and model_dump() wrote every omitted field as its default
over the top. Now exclude_unset. area_context would have been the next victim,
which is why it gets its own route alongside ten-codes rather than a field on
that payload.
Closes server-26#36.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
212 lines
8.2 KiB
Python
212 lines
8.2 KiB
Python
"""
|
|
Unit tests for the transcript correction pass (server-26#36).
|
|
|
|
Two properties carry real risk and are pinned hardest here:
|
|
|
|
* SCOPE RESOLUTION — talkgroup reference data must rank ABOVE system data.
|
|
A system spanning several counties can have a talkgroup covering one
|
|
municipality, and burying that municipality's streets under a county-wide
|
|
list is the failure this whole feature exists to avoid.
|
|
|
|
* SEGMENT ALIGNMENT — scene extraction maps scenes to transmissions by index
|
|
(segment_indices), so a corrected array of the wrong length would silently
|
|
attribute the wrong audio to a scene. Anything but an exact 1:1 match must
|
|
be discarded whole.
|
|
"""
|
|
import pytest
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
from app.internal import transcript_correction as tc
|
|
|
|
SYSTEM = {
|
|
"vocabulary": ["Croton-Harmon", "Metro-North"],
|
|
"ten_codes": {"10-4": "acknowledged", "10-13": "officer needs assistance"},
|
|
"area_context": {"county": "Westchester", "roads": ["Route 9", "Saw Mill Parkway"]},
|
|
"config": {
|
|
"talkgroups": [
|
|
{
|
|
"id": 9048,
|
|
"name": "Ossining - Police Dispatch",
|
|
"vocabulary": ["Snowden Avenue", "Croton-Harmon"],
|
|
"area_context": {"municipality": "Ossining", "landmarks": ["Sing Sing"]},
|
|
},
|
|
{"id": 9600, "name": "Harrison - Police/EMS Dispatch"},
|
|
{"id": 9563, "ten_codes": {"10-4": "on scene"}},
|
|
]
|
|
},
|
|
}
|
|
|
|
|
|
def _system(doc=SYSTEM):
|
|
return patch.object(tc.fstore, "doc_get_cached", AsyncMock(return_value=doc))
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _api_key():
|
|
"""
|
|
The dev venv has no GEMINI_API_KEY, and correct() returns early without one
|
|
— which would make every assertion below pass for the wrong reason.
|
|
"""
|
|
with patch.object(tc.settings, "gemini_api_key", "test-key"):
|
|
yield
|
|
|
|
|
|
# ── Scope resolution ────────────────────────────────────────────────────────
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_talkgroup_vocabulary_ranks_above_system():
|
|
with _system():
|
|
ctx = await tc.resolve_context("sys-1", 9048)
|
|
assert ctx["vocabulary"][0] == "Snowden Avenue", "talkgroup terms must come first"
|
|
assert "Metro-North" in ctx["vocabulary"], "system terms are still inherited"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_duplicate_terms_are_not_repeated():
|
|
"""Croton-Harmon is on both scopes; it should appear once, at talkgroup rank."""
|
|
with _system():
|
|
ctx = await tc.resolve_context("sys-1", 9048)
|
|
assert [t.lower() for t in ctx["vocabulary"]].count("croton-harmon") == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_talkgroup_area_precedes_system_area():
|
|
with _system():
|
|
ctx = await tc.resolve_context("sys-1", 9048)
|
|
joined = "\n".join(ctx["area_lines"])
|
|
assert joined.index("Ossining") < joined.index("Westchester")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_talkgroup_without_own_data_inherits_system():
|
|
with _system():
|
|
ctx = await tc.resolve_context("sys-1", 9600)
|
|
assert ctx["vocabulary"] == ["Croton-Harmon", "Metro-North"]
|
|
assert any("Westchester" in line for line in ctx["area_lines"])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_talkgroup_ten_code_overrides_system_meaning():
|
|
with _system():
|
|
ctx = await tc.resolve_context("sys-1", 9563)
|
|
assert ctx["ten_codes"]["10-4"] == "on scene"
|
|
assert ctx["ten_codes"]["10-13"] == "officer needs assistance"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("system_id, tgid", [(None, 9048), ("sys-1", None)])
|
|
async def test_missing_scope_is_not_an_error(system_id, tgid):
|
|
with _system():
|
|
ctx = await tc.resolve_context(system_id, tgid)
|
|
assert isinstance(ctx["vocabulary"], list)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unconfigured_system_yields_empty_context():
|
|
with _system(doc=None):
|
|
ctx = await tc.resolve_context("sys-1", 9048)
|
|
assert ctx == {"vocabulary": [], "ten_codes": {}, "area_lines": []}
|
|
|
|
|
|
# ── Correction behaviour ────────────────────────────────────────────────────
|
|
|
|
def _gemini(payload):
|
|
return patch.object(tc, "_sync_gemini", lambda model, prompt: payload)
|
|
|
|
|
|
SEGS = [{"start": 0.0, "end": 1.0, "text": "Headquarters, 11-9."},
|
|
{"start": 1.0, "end": 2.0, "text": "Shout out to Optum."},
|
|
{"start": 2.0, "end": 3.0, "text": "360 north, back to Rose."}]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_short_transcript_is_never_sent():
|
|
"""9 of 29 calls in the sample window were <=3 words. Nothing to correct."""
|
|
with patch.object(tc, "_sync_gemini") as m:
|
|
out = await tc.correct("c1", "10-4.", None, system_id="sys-1")
|
|
assert out == (None, None, False)
|
|
m.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_disabled_by_setting():
|
|
with patch.object(tc.settings, "transcript_correction_enabled", False), \
|
|
patch.object(tc, "_sync_gemini") as m:
|
|
assert await tc.correct("c1", "a b c d e", None) == (None, None, False)
|
|
m.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_segments_corrected_when_lengths_match():
|
|
payload = {"corrected": "Headquarters, 11-9. Show it out to Ossining. 360 north, back to Route 9.",
|
|
"segments": ["Headquarters, 11-9.", "Show it out to Ossining.", "360 north, back to Route 9."]}
|
|
with _system(), _gemini(payload):
|
|
text, segs, not_speech = await tc.correct("c1", "x y z w", SEGS, system_id="sys-1", talkgroup_id=9048)
|
|
assert not_speech is False
|
|
assert segs is not None and len(segs) == 3
|
|
assert segs[1]["text"] == "Show it out to Ossining."
|
|
assert segs[1]["start"] == 1.0, "timing must survive correction untouched"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_wrong_segment_count_is_discarded_whole():
|
|
"""A short array would silently misattribute audio to the wrong scene."""
|
|
payload = {"corrected": "fine", "segments": ["only", "two"]}
|
|
with _system(), _gemini(payload):
|
|
text, segs, _ = await tc.correct("c1", "x y z w", SEGS, system_id="sys-1")
|
|
assert segs is None
|
|
assert text == "fine", "the joined correction still stands"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_non_string_segment_entries_are_discarded():
|
|
payload = {"corrected": None, "segments": ["ok", 42, "ok"]}
|
|
with _system(), _gemini(payload):
|
|
_, segs, _ = await tc.correct("c1", "x y z w", SEGS, system_id="sys-1")
|
|
assert segs is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unchanged_segments_report_no_correction():
|
|
payload = {"corrected": None, "segments": [s["text"] for s in SEGS]}
|
|
with _system(), _gemini(payload):
|
|
text, segs, _ = await tc.correct("c1", "x y z w", SEGS, system_id="sys-1")
|
|
assert (text, segs) == (None, None)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_echoed_transcript_counts_as_no_change():
|
|
with _system(), _gemini({"corrected": " x y z w "}):
|
|
text, _, _ = await tc.correct("c1", "x y z w", None, system_id="sys-1")
|
|
assert text is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_not_speech_is_surfaced():
|
|
with _system(), _gemini({"corrected": None, "not_speech": True}):
|
|
_, _, not_speech = await tc.correct("c1", "10-11. 10-12. 10-13. 10-14.", None, system_id="sys-1")
|
|
assert not_speech is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_model_failure_leaves_the_transcript_alone():
|
|
"""Correction is an improvement, never a dependency."""
|
|
def boom(model, prompt):
|
|
raise RuntimeError("gemini exploded")
|
|
with _system(), patch.object(tc, "_sync_gemini", boom):
|
|
assert await tc.correct("c1", "x y z w", SEGS, system_id="sys-1") == (None, None, False)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_reference_data_reaches_the_prompt():
|
|
seen = {}
|
|
def capture(model, prompt):
|
|
seen["prompt"] = prompt
|
|
return {"corrected": None}
|
|
with _system(), patch.object(tc, "_sync_gemini", capture):
|
|
await tc.correct("c1", "x y z w", None, system_id="sys-1",
|
|
talkgroup_id=9048, talkgroup_name="Ossining - Police Dispatch")
|
|
p = seen["prompt"]
|
|
assert "Snowden Avenue" in p and "Sing Sing" in p and "Ossining - Police Dispatch" in p
|
|
assert "10-13=officer needs assistance" in p
|