Logan CusanoandClaude Opus 5
964343c819
#36 — the correction pass shipped in 58efdbd was right, its reference-data
shape was not. One shape now, at both scopes, every field nullable:
area_context: { municipality?, county?, state?,
center?, radius_km?, resolved_from?, resolved_at?,
local_knowledge?: [{term, meaning}] }
`state` closes the ambiguity that made "Ossining" a national guess.
`local_knowledge` replaces roads[]/landmarks[], which could not hold
intersections, schools or nicknames and carried no meanings — `11-X-ray` is
useless alone, `11-X-ray — MTA PD patrol unit` is what a corrector can act on.
Pre-#36 roads[]/landmarks[] are read forward as bare terms so nothing an
operator already entered is lost.
Nullability is the mechanism: which scope gets filled is the operator's
declaration of how homogeneous the system is. One town — fill it once at system
level. Statewide — leave it blank and fill each talkgroup.
The backend owns the derived anchor. PUT /systems/{id} merges config.talkgroups[]
against what is stored instead of writing the client's blob verbatim, which
would have erased the anchor and the pending queue — the same defect as the
ten_codes wipe.
#37 — Maps as a verifier, not as prompt stuffing. The corrector emits its
location nouns; each is geocoded against the talkgroup's anchor, and on a miss
we look for a sound-alike that does resolve there, correct to it, and propose
{term, meaning} to that talkgroup. Cost scales with location nouns, not calls.
No anchor means SKIP. An area too wide to discriminate stores no anchor at all,
because a statewide radius would confirm anything inside it — verification that
passes everything is worse than none, since it reads as a check in the data.
Also re-anchors _geocode_location, which rejected results >40km from the NODE
(server-26#6). An antenna is not a jurisdiction; distance-from-node was always
a stand-in for the anchor and is now only the fallback.
The induction loop proposes at talkgroup level and never promotes. Blast
radius: a wrong term on a channel misleads that channel, the same term
system-wide misleads one 400km away on a statewide system.
38 new tests; 240 pass. Frontend typechecks clean.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
228 lines
8.7 KiB
Python
228 lines
8.7 KiB
Python
"""
|
|
Unit tests for the transcript correction pass (server-26#36).
|
|
|
|
Two properties carry real risk and are pinned hardest here:
|
|
|
|
* SCOPE RESOLUTION — talkgroup reference data must rank ABOVE system data.
|
|
A system spanning several counties can have a talkgroup covering one
|
|
municipality, and burying that municipality's streets under a county-wide
|
|
list is the failure this whole feature exists to avoid.
|
|
|
|
* SEGMENT ALIGNMENT — scene extraction maps scenes to transmissions by index
|
|
(segment_indices), so a corrected array of the wrong length would silently
|
|
attribute the wrong audio to a scene. Anything but an exact 1:1 match must
|
|
be discarded whole.
|
|
"""
|
|
import pytest
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
from app.internal import transcript_correction as tc
|
|
|
|
SYSTEM = {
|
|
"vocabulary": ["Croton-Harmon", "Metro-North"],
|
|
"ten_codes": {"10-4": "acknowledged", "10-13": "officer needs assistance"},
|
|
"area_context": {
|
|
"county": "Westchester",
|
|
"state": "New York",
|
|
"local_knowledge": [
|
|
{"term": "Route 9", "meaning": "north-south state highway"},
|
|
{"term": "Saw Mill Parkway"},
|
|
],
|
|
},
|
|
"config": {
|
|
"talkgroups": [
|
|
{
|
|
"id": 9048,
|
|
"name": "Ossining - Police Dispatch",
|
|
"vocabulary": ["Snowden Avenue", "Croton-Harmon"],
|
|
"area_context": {
|
|
"municipality": "Ossining",
|
|
"local_knowledge": [{"term": "Sing Sing", "meaning": "state prison"}],
|
|
},
|
|
},
|
|
{"id": 9600, "name": "Harrison - Police/EMS Dispatch"},
|
|
{"id": 9563, "ten_codes": {"10-4": "on scene"}},
|
|
]
|
|
},
|
|
}
|
|
|
|
|
|
def _system(doc=SYSTEM):
|
|
return patch.object(tc.fstore, "doc_get_cached", AsyncMock(return_value=doc))
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _api_key():
|
|
"""
|
|
The dev venv has no GEMINI_API_KEY, and correct() returns early without one
|
|
— which would make every assertion below pass for the wrong reason.
|
|
"""
|
|
with patch.object(tc.settings, "gemini_api_key", "test-key"):
|
|
yield
|
|
|
|
|
|
# ── Scope resolution ────────────────────────────────────────────────────────
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_talkgroup_vocabulary_ranks_above_system():
|
|
with _system():
|
|
ctx = await tc.resolve_context("sys-1", 9048)
|
|
assert ctx["vocabulary"][0] == "Snowden Avenue", "talkgroup terms must come first"
|
|
assert "Metro-North" in ctx["vocabulary"], "system terms are still inherited"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_duplicate_terms_are_not_repeated():
|
|
"""Croton-Harmon is on both scopes; it should appear once, at talkgroup rank."""
|
|
with _system():
|
|
ctx = await tc.resolve_context("sys-1", 9048)
|
|
assert [t.lower() for t in ctx["vocabulary"]].count("croton-harmon") == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_talkgroup_area_precedes_system_area():
|
|
with _system():
|
|
ctx = await tc.resolve_context("sys-1", 9048)
|
|
joined = "\n".join(ctx["area_lines"])
|
|
assert joined.index("Ossining") < joined.index("Westchester")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_talkgroup_without_own_data_inherits_system():
|
|
with _system():
|
|
ctx = await tc.resolve_context("sys-1", 9600)
|
|
assert ctx["vocabulary"] == ["Croton-Harmon", "Metro-North"]
|
|
assert any("Westchester" in line for line in ctx["area_lines"])
|
|
assert ctx["area"].get("municipality") is None, "inherits, invents nothing"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_talkgroup_ten_code_overrides_system_meaning():
|
|
with _system():
|
|
ctx = await tc.resolve_context("sys-1", 9563)
|
|
assert ctx["ten_codes"]["10-4"] == "on scene"
|
|
assert ctx["ten_codes"]["10-13"] == "officer needs assistance"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("system_id, tgid", [(None, 9048), ("sys-1", None)])
|
|
async def test_missing_scope_is_not_an_error(system_id, tgid):
|
|
with _system():
|
|
ctx = await tc.resolve_context(system_id, tgid)
|
|
assert isinstance(ctx["vocabulary"], list)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unconfigured_system_yields_empty_context():
|
|
with _system(doc=None):
|
|
ctx = await tc.resolve_context("sys-1", 9048)
|
|
assert ctx == {
|
|
"vocabulary": [], "ten_codes": {}, "area_lines": [],
|
|
"area": {}, "system_area": {}, "tg_area": {},
|
|
}
|
|
|
|
|
|
# ── Correction behaviour ────────────────────────────────────────────────────
|
|
|
|
def _gemini(payload):
|
|
return patch.object(tc, "_sync_gemini", lambda model, prompt: payload)
|
|
|
|
|
|
SEGS = [{"start": 0.0, "end": 1.0, "text": "Headquarters, 11-9."},
|
|
{"start": 1.0, "end": 2.0, "text": "Shout out to Optum."},
|
|
{"start": 2.0, "end": 3.0, "text": "360 north, back to Rose."}]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_short_transcript_is_never_sent():
|
|
"""9 of 29 calls in the sample window were <=3 words. Nothing to correct."""
|
|
with patch.object(tc, "_sync_gemini") as m:
|
|
out = await tc.correct("c1", "10-4.", None, system_id="sys-1")
|
|
assert out == (None, None, False)
|
|
m.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_disabled_by_setting():
|
|
with patch.object(tc.settings, "transcript_correction_enabled", False), \
|
|
patch.object(tc, "_sync_gemini") as m:
|
|
assert await tc.correct("c1", "a b c d e", None) == (None, None, False)
|
|
m.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_segments_corrected_when_lengths_match():
|
|
payload = {"corrected": "Headquarters, 11-9. Show it out to Ossining. 360 north, back to Route 9.",
|
|
"segments": ["Headquarters, 11-9.", "Show it out to Ossining.", "360 north, back to Route 9."]}
|
|
with _system(), _gemini(payload):
|
|
text, segs, not_speech = await tc.correct("c1", "x y z w", SEGS, system_id="sys-1", talkgroup_id=9048)
|
|
assert not_speech is False
|
|
assert segs is not None and len(segs) == 3
|
|
assert segs[1]["text"] == "Show it out to Ossining."
|
|
assert segs[1]["start"] == 1.0, "timing must survive correction untouched"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_wrong_segment_count_is_discarded_whole():
|
|
"""A short array would silently misattribute audio to the wrong scene."""
|
|
payload = {"corrected": "fine", "segments": ["only", "two"]}
|
|
with _system(), _gemini(payload):
|
|
text, segs, _ = await tc.correct("c1", "x y z w", SEGS, system_id="sys-1")
|
|
assert segs is None
|
|
assert text == "fine", "the joined correction still stands"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_non_string_segment_entries_are_discarded():
|
|
payload = {"corrected": None, "segments": ["ok", 42, "ok"]}
|
|
with _system(), _gemini(payload):
|
|
_, segs, _ = await tc.correct("c1", "x y z w", SEGS, system_id="sys-1")
|
|
assert segs is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unchanged_segments_report_no_correction():
|
|
payload = {"corrected": None, "segments": [s["text"] for s in SEGS]}
|
|
with _system(), _gemini(payload):
|
|
text, segs, _ = await tc.correct("c1", "x y z w", SEGS, system_id="sys-1")
|
|
assert (text, segs) == (None, None)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_echoed_transcript_counts_as_no_change():
|
|
with _system(), _gemini({"corrected": " x y z w "}):
|
|
text, _, _ = await tc.correct("c1", "x y z w", None, system_id="sys-1")
|
|
assert text is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_not_speech_is_surfaced():
|
|
with _system(), _gemini({"corrected": None, "not_speech": True}):
|
|
_, _, not_speech = await tc.correct("c1", "10-11. 10-12. 10-13. 10-14.", None, system_id="sys-1")
|
|
assert not_speech is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_model_failure_leaves_the_transcript_alone():
|
|
"""Correction is an improvement, never a dependency."""
|
|
def boom(model, prompt):
|
|
raise RuntimeError("gemini exploded")
|
|
with _system(), patch.object(tc, "_sync_gemini", boom):
|
|
assert await tc.correct("c1", "x y z w", SEGS, system_id="sys-1") == (None, None, False)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_reference_data_reaches_the_prompt():
|
|
seen = {}
|
|
def capture(model, prompt):
|
|
seen["prompt"] = prompt
|
|
return {"corrected": None}
|
|
with _system(), patch.object(tc, "_sync_gemini", capture):
|
|
await tc.correct("c1", "x y z w", None, system_id="sys-1",
|
|
talkgroup_id=9048, talkgroup_name="Ossining - Police Dispatch")
|
|
p = seen["prompt"]
|
|
assert "Snowden Avenue" in p and "Ossining - Police Dispatch" in p
|
|
assert "Sing Sing — state prison" in p, "a term without its meaning is half the information"
|
|
assert "Ossining, Westchester, New York" in p, "state must reach the prompt (server-26#36)"
|
|
assert "10-13=officer needs assistance" in p
|