Merge feat/297-cheaper-ai into feat/299-correlation-simplify
Resolve upload.py (no-scene fallback uses the new _correlate with #298's non_event_chatter reason) and chatter_classifier docstring (gate built, off by default). Fold intelligence._llm_bool into incident_correlator.parse_llm_bool (#88). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Logan Cusano
Claude Opus 5.5
@@ -96,3 +96,13 @@ def _no_real_discord(monkeypatch):
|
||||
monkeypatch.setattr(discord_rest, "_transport", httpx.MockTransport(_offline))
|
||||
yield
|
||||
discord_rest.reset()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fresh_geocode_cache():
|
||||
"""The geocode cache (server-26#298) is process-wide; a test that stubs
|
||||
Maps must never be answered by a previous test's stub."""
|
||||
from app.internal import intelligence
|
||||
intelligence.clear_geocode_cache()
|
||||
yield
|
||||
intelligence.clear_geocode_cache()
|
||||
|
||||
@@ -219,15 +219,21 @@ async def test_transcript_correction_is_skipped_when_its_flag_is_off():
|
||||
patch.object(transcription, "fstore") as fs, \
|
||||
patch.object(transcription, "ai_health") as health, \
|
||||
patch.object(transcription, "transcript_correction") as tc, \
|
||||
patch("asyncio.to_thread", AsyncMock(return_value=("units respond", [], False))):
|
||||
patch.object(transcription.settings, "openai_api_key", "k"), \
|
||||
patch.object(transcription.settings, "groq_api_key", None), \
|
||||
patch.object(transcription, "_download", lambda uri: "/nonexistent.mp3"), \
|
||||
patch.object(transcription, "_sync_transcribe_file", lambda *a, **k: {
|
||||
"text": "units respond to the fire on Main Street", "segments": [],
|
||||
"degenerate": False, "duration": 3.0, "confidence": None}):
|
||||
fs.doc_set = AsyncMock()
|
||||
health.report_healthy = AsyncMock()
|
||||
health.report_failure = AsyncMock()
|
||||
tc.correct = AsyncMock()
|
||||
await transcription.transcribe_call(
|
||||
text, _ = await transcription.transcribe_call(
|
||||
"call-1", "gs://bucket/call-1.mp3", "PD Dispatch", system_id="sys-1"
|
||||
)
|
||||
|
||||
assert text == "units respond to the fire on Main Street", "the transcript must still be produced"
|
||||
tc.correct.assert_not_awaited()
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
"""
|
||||
server-26#300 — confidence gate before any push alert.
|
||||
|
||||
CrimeRadar pushed "shots fired at an elementary school" after STT heard "fire
|
||||
alarm" as "firearm". A single uncorroborated transcript hit on a
|
||||
high-severity term must be held, not pushed; a corroborated incident releases
|
||||
it; low-confidence matches never push.
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from app.internal import alerter
|
||||
from tests.fake_firestore import FakeFirestore
|
||||
|
||||
|
||||
def _ev(**over):
|
||||
base = {"incident_ids": [], "max_calls": 0, "severity": None,
|
||||
"stt_confidence": None, "not_speech": False}
|
||||
base.update(over)
|
||||
return base
|
||||
|
||||
|
||||
# ── the pure decision ───────────────────────────────────────────────────────
|
||||
|
||||
def test_talkgroup_only_rule_always_pushes():
|
||||
v = alerter._assess(["talkgroup:9048"], [], _ev(stt_confidence=0.1))
|
||||
assert v["push"] and v["confidence"] == 1.0
|
||||
|
||||
|
||||
def test_transcript_only_firearm_single_call_is_held():
|
||||
v = alerter._assess(["firearm"], [], _ev())
|
||||
assert not v["push"] and v["hold_reason"] == "needs_corroboration"
|
||||
assert v["high_severity"] is True
|
||||
|
||||
|
||||
def test_tagged_shots_fired_still_needs_corroboration():
|
||||
v = alerter._assess(["shots-fired"], ["shots-fired"], _ev(max_calls=1))
|
||||
assert not v["push"] and v["confidence"] == 0.8
|
||||
|
||||
|
||||
def test_corroborated_high_severity_pushes():
|
||||
v = alerter._assess(["shots-fired"], ["shots-fired"], _ev(max_calls=2))
|
||||
assert v["push"]
|
||||
|
||||
|
||||
def test_major_incident_makes_any_content_match_high_severity():
|
||||
v = alerter._assess(["school"], [], _ev(severity="major"))
|
||||
assert v["high_severity"] and not v["push"]
|
||||
|
||||
|
||||
def test_ordinary_tag_match_pushes():
|
||||
v = alerter._assess(["mva"], ["mva"], _ev(max_calls=1))
|
||||
assert v["push"] and not v["high_severity"]
|
||||
|
||||
|
||||
def test_low_stt_confidence_holds_even_ordinary_matches():
|
||||
v = alerter._assess(["mva"], ["mva"], _ev(stt_confidence=0.1))
|
||||
assert not v["push"] and v["hold_reason"] == "low_confidence"
|
||||
assert v["confidence"] == round(0.8 * 0.1 / 0.3, 3)
|
||||
|
||||
|
||||
def test_typical_radio_logprob_does_not_hold_a_transcript_match():
|
||||
"""exp(avg_logprob) of ~0.4 is normal on vocoded audio (review of #300)."""
|
||||
v = alerter._assess(["mva"], [], _ev(stt_confidence=0.4))
|
||||
assert v["push"]
|
||||
|
||||
|
||||
def test_not_speech_never_pushes_content():
|
||||
v = alerter._assess(["mva"], ["mva"], _ev(not_speech=True))
|
||||
assert not v["push"]
|
||||
|
||||
|
||||
# ── end to end through Firestore ────────────────────────────────────────────
|
||||
|
||||
RULE = {"rule_id": "r1", "name": "Weapons", "enabled": True, "org_id": "org-1",
|
||||
"keywords": ["firearm", "shots-fired"], "discord_webhook": "https://discord.example/wh"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fs(monkeypatch):
|
||||
fake = FakeFirestore.install(monkeypatch)
|
||||
fake.put("alert_rules", "r1", dict(RULE))
|
||||
return fake
|
||||
|
||||
|
||||
def _posts():
|
||||
posted = []
|
||||
|
||||
class _Client:
|
||||
async def __aenter__(self): return self
|
||||
async def __aexit__(self, *a): return False
|
||||
|
||||
async def post(self, url, json=None, **k):
|
||||
posted.append(json)
|
||||
|
||||
return posted, patch("httpx.AsyncClient", lambda *a, **k: _Client())
|
||||
|
||||
|
||||
async def _dispatch(call_id, tags, transcript):
|
||||
await alerter.check_and_dispatch(call_id=call_id, node_id="n1", talkgroup_id=1,
|
||||
talkgroup_name="PD Dispatch", tags=tags, transcript=transcript)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crimeradar_case_is_held_then_released_by_corroboration(fs):
|
||||
fs.put("calls", "c1", {"org_id": "org-1", "incident_ids": ["inc-1"]})
|
||||
fs.put("incidents", "inc-1", {"call_ids": ["c1"], "severity": "moderate"})
|
||||
posted, http = _posts()
|
||||
with http:
|
||||
await _dispatch("c1", [], "report of a firearm at the elementary school")
|
||||
assert posted == [], "a single uncorroborated firearm hit must not push"
|
||||
assert fs.data.get("alert_events", {}) == {}
|
||||
(hold,) = fs.data["alert_holds"].values()
|
||||
assert hold["status"] == "held" and hold["hold_reason"] == "needs_corroboration"
|
||||
assert hold["incident_id"] == "inc-1"
|
||||
|
||||
# A second call joins the incident; it does not itself mention a firearm.
|
||||
fs.put("calls", "c2", {"org_id": "org-1", "incident_ids": ["inc-1"]})
|
||||
fs.put("incidents", "inc-1", {"call_ids": ["c1", "c2"], "severity": "moderate"})
|
||||
with http:
|
||||
await _dispatch("c2", [], "units on scene, stand by")
|
||||
assert len(posted) == 1 and "Weapons" in posted[0]["content"]
|
||||
(event,) = fs.data["alert_events"].values()
|
||||
assert event["released_from_hold"] is True and event["corroborated_by_call_id"] == "c2"
|
||||
assert fs.get("alert_holds", hold["alert_id"])["status"] == "released"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_low_confidence_hold_is_not_released_by_corroboration(fs):
|
||||
fs.put("calls", "c1", {"org_id": "org-1", "incident_ids": ["inc-1"], "stt_confidence": 0.1})
|
||||
fs.put("incidents", "inc-1", {"call_ids": ["c1"]})
|
||||
posted, http = _posts()
|
||||
with http:
|
||||
await _dispatch("c1", [], "firearm")
|
||||
(hold,) = fs.data["alert_holds"].values()
|
||||
assert hold["hold_reason"] == "low_confidence"
|
||||
|
||||
fs.put("calls", "c2", {"org_id": "org-1", "incident_ids": ["inc-1"]})
|
||||
fs.put("incidents", "inc-1", {"call_ids": ["c1", "c2"]})
|
||||
with http:
|
||||
await _dispatch("c2", [], "units on scene")
|
||||
assert posted == []
|
||||
assert fs.get("alert_holds", hold["alert_id"])["status"] == "held"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rule_pushed_now_supersedes_its_hold_instead_of_alerting_twice(fs):
|
||||
fs.put("calls", "c1", {"org_id": "org-1", "incident_ids": ["inc-1"]})
|
||||
fs.put("incidents", "inc-1", {"call_ids": ["c1"]})
|
||||
posted, http = _posts()
|
||||
with http:
|
||||
await _dispatch("c1", ["shots-fired"], "shots fired")
|
||||
fs.put("calls", "c2", {"org_id": "org-1", "incident_ids": ["inc-1"]})
|
||||
fs.put("incidents", "inc-1", {"call_ids": ["c1", "c2"]})
|
||||
with http:
|
||||
await _dispatch("c2", ["shots-fired"], "more shots fired")
|
||||
assert len(posted) == 1
|
||||
(hold,) = fs.data["alert_holds"].values()
|
||||
assert hold["status"] == "superseded"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_held_alerts_are_listed_for_admins(fs):
|
||||
from app.routers import alerts as alerts_router
|
||||
fs.put("alert_holds", "h1", {"alert_id": "h1", "status": "held", "triggered_at": "2026-10-27T01:00:00"})
|
||||
fs.put("alert_holds", "h2", {"alert_id": "h2", "status": "released", "triggered_at": "2026-10-27T02:00:00"})
|
||||
out = await alerts_router.list_held_alerts(status="held", org_id=None, _={})
|
||||
assert [h["alert_id"] for h in out] == ["h1"]
|
||||
@@ -0,0 +1,194 @@
|
||||
"""
|
||||
server-26#298 — the chatter pre-filter gates every paid call.
|
||||
|
||||
Radio housekeeping (roll call, a bare "Post 4, 10-8, 10-4") used to go through
|
||||
transcript correction, scene extraction, an embedding per scene and a geocode,
|
||||
only to come back as a thin routine scene. Now it pays for none of them; the
|
||||
free rule paths (clearance parse, self-initiated/plate backstop) still run.
|
||||
"""
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import settings
|
||||
from app.internal import intelligence, transcription
|
||||
from app.internal.chatter_classifier import gate_chatter
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _gate_on():
|
||||
"""The gate ships OFF (shadow) until a live window clears it (#127);
|
||||
these tests exercise it switched on."""
|
||||
with patch.object(settings, "chatter_gate_enabled", True):
|
||||
yield
|
||||
|
||||
|
||||
class _NoPaidCalls:
|
||||
"""Fail the test if any paid extraction/embedding/geocode call happens."""
|
||||
|
||||
def __enter__(self):
|
||||
def _boom(*a, **k):
|
||||
raise AssertionError("a paid AI call was made for chatter")
|
||||
|
||||
async def _aboom(*a, **k):
|
||||
raise AssertionError("a paid geocode was made for chatter")
|
||||
|
||||
self.ps = [
|
||||
patch.object(intelligence, "_sync_extract", _boom),
|
||||
patch.object(intelligence, "_sync_embed", _boom),
|
||||
patch.object(intelligence, "_geocode_lookup", _aboom),
|
||||
]
|
||||
for p in self.ps:
|
||||
p.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
for p in self.ps:
|
||||
p.stop()
|
||||
|
||||
|
||||
async def _extract(transcript, talkgroup_name="PD Dispatch"):
|
||||
saved: dict = {}
|
||||
|
||||
async def _doc_set(coll, doc_id, data, merge=True):
|
||||
saved.update(data)
|
||||
|
||||
with patch.object(intelligence.fstore, "doc_get_cached", AsyncMock(return_value=None)), \
|
||||
patch.object(intelligence.fstore, "doc_set", _doc_set):
|
||||
scenes = await intelligence.extract_scenes("call-1", transcript, talkgroup_name)
|
||||
return scenes, saved
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bare_acknowledgement_over_five_words_skips_extraction():
|
||||
with _NoPaidCalls():
|
||||
scenes, saved = await _extract("10-4, 10-4 Central, 98. 10-4, 98.")
|
||||
assert scenes == []
|
||||
assert saved["skip_reason"] == "non_event_chatter"
|
||||
assert saved["severity"] == "routine"
|
||||
assert saved["chatter_classifier_verdict"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_whole_roll_call_skips_extraction():
|
||||
roll = ("Car 100, roll call. Henry 1. Henry 1. Sam 1. Sam 1. 45 Baker. 45 Baker. "
|
||||
"11 Adam. Adam. 11 Baker. Baker. Staff 1. 1. Staff 2.")
|
||||
with _NoPaidCalls():
|
||||
scenes, saved = await _extract(roll)
|
||||
assert scenes == [] and saved["skip_reason"] == "non_event_chatter"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_roll_call_mentioned_on_the_way_to_a_real_job_still_extracts():
|
||||
"""The #127 review's false-positive shape must still reach the model."""
|
||||
text = "Delayed for roll call, then respond to a structure fire at 12 Main Street"
|
||||
assert gate_chatter(text) == (False, None)
|
||||
with patch.object(intelligence, "_sync_extract", return_value=[]) as ext:
|
||||
await _extract(text)
|
||||
ext.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plate_read_chatter_still_becomes_a_stop_for_free():
|
||||
text = "Lincoln Charlie Robert 7-4-0-7, Lincoln Charlie Robert 7-4-0-7."
|
||||
with _NoPaidCalls():
|
||||
scenes, saved = await _extract(text)
|
||||
assert len(scenes) == 1 and scenes[0]["tags"] == ["traffic-stop"]
|
||||
assert saved["severity"] == "minor"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gate_off_restores_shadow_mode():
|
||||
with patch.object(settings, "chatter_gate_enabled", False), \
|
||||
patch.object(intelligence, "_sync_extract", return_value=[]) as ext:
|
||||
_, saved = await _extract("10-4, 10-4 Central, 98. 10-4, 98.")
|
||||
ext.assert_called_once()
|
||||
assert saved["chatter_classifier_verdict"] is True, "the shadow verdict is still recorded"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_real_event_is_not_gated():
|
||||
with patch.object(intelligence, "_sync_extract", return_value=[]) as ext:
|
||||
await _extract("Shots fired, officer down, requesting backup immediately at the school.")
|
||||
ext.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcription_skips_correction_for_chatter():
|
||||
flags = {"transcript_correction_enabled": True}
|
||||
with patch.object(settings, "openai_api_key", "k"), \
|
||||
patch.object(settings, "groq_api_key", None), \
|
||||
patch.object(transcription, "_download", lambda uri: "/nonexistent.flac"), \
|
||||
patch.object(transcription, "_sync_transcribe_file", lambda *a, **k: {
|
||||
"text": "Post 4, 10-8. 10-4. Copy, 98.", "segments": [],
|
||||
"degenerate": False, "duration": 2.0, "confidence": None}), \
|
||||
patch.object(transcription.fstore, "doc_set", AsyncMock()), \
|
||||
patch.object(transcription, "ai_health") as health, \
|
||||
patch.object(transcription.transcript_correction, "correct", AsyncMock()) as correct, \
|
||||
patch("app.internal.feature_flags.resolve_flags",
|
||||
AsyncMock(return_value=(flags, lambda k: flags.get(k, True)))):
|
||||
health.report_healthy = AsyncMock()
|
||||
text, _ = await transcription.transcribe_call("call-1", "gs://b/c.flac", "PD Dispatch")
|
||||
assert text == "Post 4, 10-8. 10-4. Copy, 98."
|
||||
correct.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chatter_still_reaches_the_free_thin_correlation_path():
|
||||
"""non_event_chatter joins transcript_too_short in upload.py's no-scene
|
||||
fallback (rules-only for a thin call — no LLM)."""
|
||||
from app.routers import upload
|
||||
|
||||
with patch.object(upload, "fstore") as fs, \
|
||||
patch.object(upload, "_correlate", AsyncMock(return_value=(None, None))) as corr:
|
||||
fs.doc_get = AsyncMock(return_value={"skip_reason": "non_event_chatter", "severity": "routine"})
|
||||
fs.doc_set = AsyncMock()
|
||||
await upload._extract_and_correlate(
|
||||
call_id="c1", node_id="n1", system_id="s1", talkgroup_id=1,
|
||||
talkgroup_name="PD Dispatch", transcript="10-4 Central, 98.", scenes=[],
|
||||
)
|
||||
corr.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_scene_fallback_passes_the_call_doc_severity():
|
||||
"""server-26#104: a reprocessed call whose doc says 'major' must not be
|
||||
re-correlated as 'routine' (thin, no fit check)."""
|
||||
from app.routers import upload
|
||||
|
||||
with patch.object(upload, "fstore") as fs, \
|
||||
patch.object(upload, "_correlate", AsyncMock(return_value=(None, None))) as corr:
|
||||
fs.doc_get = AsyncMock(return_value={"severity": "major"})
|
||||
fs.doc_set = AsyncMock()
|
||||
await upload._extract_and_correlate(
|
||||
call_id="c1", node_id="n1", system_id="s1", talkgroup_id=1,
|
||||
talkgroup_name="PD Dispatch", transcript="whatever", scenes=[],
|
||||
)
|
||||
assert corr.await_args.kwargs["severity"] == "major"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gate_off_still_corrects_chatter():
|
||||
"""With the gate off, transcription must not skip correction either."""
|
||||
flags = {"transcript_correction_enabled": True}
|
||||
with patch.object(settings, "chatter_gate_enabled", False), \
|
||||
patch.object(settings, "openai_api_key", "k"), \
|
||||
patch.object(settings, "groq_api_key", None), \
|
||||
patch.object(transcription, "_download", lambda uri: "/nonexistent.flac"), \
|
||||
patch.object(transcription, "_sync_transcribe_file", lambda *a, **k: {
|
||||
"text": "Post 4, 10-8. 10-4. Copy, 98.", "segments": [],
|
||||
"degenerate": False, "duration": 2.0, "confidence": None}), \
|
||||
patch.object(transcription.fstore, "doc_set", AsyncMock()), \
|
||||
patch.object(transcription, "ai_health") as health, \
|
||||
patch.object(transcription.transcript_correction, "correct",
|
||||
AsyncMock(return_value=(None, None, False))) as correct, \
|
||||
patch("app.internal.feature_flags.resolve_flags",
|
||||
AsyncMock(return_value=(flags, lambda k: flags.get(k, True)))):
|
||||
health.report_healthy = AsyncMock()
|
||||
await transcription.transcribe_call("call-1", "gs://b/c.flac", "PD Dispatch")
|
||||
correct.assert_awaited_once()
|
||||
|
||||
|
||||
def test_gate_ships_off():
|
||||
from app.config import Settings
|
||||
assert Settings.model_fields["chatter_gate_enabled"].default is False
|
||||
@@ -0,0 +1,162 @@
|
||||
"""
|
||||
server-26#298 — geocode result cache in front of the Google Geocoding API,
|
||||
and server-26#160 — place verification gets an anchor from the talkgroup's
|
||||
own name when nobody configured one.
|
||||
"""
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import app.internal.intelligence as intel
|
||||
from app.config import settings
|
||||
from app.internal import place_verifier as pv
|
||||
|
||||
ANCHOR = {"lat": 41.16, "lng": -73.86, "radius_km": 10.0}
|
||||
|
||||
|
||||
class _Counter:
|
||||
def __init__(self, payload=None, exc=None):
|
||||
self.calls = 0
|
||||
self.payload = payload
|
||||
self.exc = exc
|
||||
|
||||
def patch(self):
|
||||
outer = self
|
||||
|
||||
class _Resp:
|
||||
def raise_for_status(self):
|
||||
if outer.exc:
|
||||
raise outer.exc
|
||||
|
||||
def json(self):
|
||||
return outer.payload
|
||||
|
||||
class _Client:
|
||||
async def __aenter__(self): return self
|
||||
async def __aexit__(self, *a): return False
|
||||
|
||||
async def get(self, *a, **k):
|
||||
outer.calls += 1
|
||||
return _Resp()
|
||||
|
||||
return patch("httpx.AsyncClient", lambda *a, **k: _Client())
|
||||
|
||||
|
||||
def _ok(lat=41.161, lng=-73.861):
|
||||
return {"status": "OK", "results": [{"geometry": {
|
||||
"location": {"lat": lat, "lng": lng}, "location_type": "ROOFTOP"}}]}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _key():
|
||||
with patch.object(settings, "google_maps_api_key", "test-key"):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_repeat_address_is_one_request():
|
||||
c = _Counter(_ok())
|
||||
with c.patch():
|
||||
a = await intel._geocode_location("12 Main Street, Ossining", anchor=ANCHOR)
|
||||
b = await intel._geocode_location("12 main street, ossining ", anchor=ANCHOR)
|
||||
assert a == b == {"lat": 41.161, "lng": -73.861, "location_type": "ROOFTOP"}
|
||||
assert c.calls == 1
|
||||
assert intel.geocode_cache_stats == {"hits": 1, "misses": 1}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cached_answer_is_still_judged_against_the_caller_anchor():
|
||||
c = _Counter(_ok())
|
||||
with c.patch():
|
||||
assert await intel._geocode_location("12 Main Street", anchor=ANCHOR)
|
||||
far = {"lat": 0.0, "lng": 0.0, "radius_km": 10.0}
|
||||
# Different bounds -> different key, so a separate request — never a
|
||||
# cached answer leaking past a different area's distance check.
|
||||
assert await intel._geocode_location("12 Main Street", anchor=far) is None
|
||||
assert c.calls == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zero_results_is_cached_but_errors_are_not():
|
||||
c = _Counter({"status": "ZERO_RESULTS", "results": []})
|
||||
with c.patch():
|
||||
await intel._geocode_location("Cool Parts, Illinois", anchor=ANCHOR)
|
||||
await intel._geocode_location("Cool Parts, Illinois", anchor=ANCHOR)
|
||||
assert c.calls == 1
|
||||
|
||||
denied = _Counter({"status": "REQUEST_DENIED"})
|
||||
with denied.patch():
|
||||
await intel._geocode_location("Snowden Avenue", anchor=ANCHOR)
|
||||
await intel._geocode_location("Snowden Avenue", anchor=ANCHOR)
|
||||
assert denied.calls == 2, "a key/quota failure is not an answer about the address"
|
||||
|
||||
boom = _Counter(_ok(), exc=RuntimeError("503"))
|
||||
with boom.patch():
|
||||
await intel._geocode_location("Croton Point", anchor=ANCHOR)
|
||||
await intel._geocode_location("Croton Point", anchor=ANCHOR)
|
||||
assert boom.calls == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ttl_expiry_and_size_bound():
|
||||
c = _Counter(_ok())
|
||||
with c.patch(), patch.object(settings, "geocode_cache_ttl_seconds", 0):
|
||||
await intel._geocode_location("12 Main Street", anchor=ANCHOR)
|
||||
await intel._geocode_location("12 Main Street", anchor=ANCHOR)
|
||||
assert c.calls == 2, "TTL 0 disables the cache"
|
||||
|
||||
c = _Counter(_ok())
|
||||
with c.patch(), patch.object(settings, "geocode_cache_max_entries", 2):
|
||||
for street in ("A Street", "B Street", "C Street"):
|
||||
await intel._geocode_location(street, anchor=ANCHOR)
|
||||
assert len(intel._geocode_cache) == 2
|
||||
|
||||
|
||||
# ── server-26#160 ───────────────────────────────────────────────────────────
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unconfigured_channel_gets_an_anchor_from_its_talkgroup_name():
|
||||
resolved = {"center": {"lat": 40.71, "lng": -74.0}, "radius_km": 25.0}
|
||||
seen = {}
|
||||
|
||||
async def _geo(query, anchor):
|
||||
seen["query"], seen["anchor"] = query, anchor
|
||||
return {"lat": 40.7, "lng": -74.0}
|
||||
|
||||
with patch.object(pv.settings, "place_verification_enabled", True), \
|
||||
patch.object(intel.fstore, "doc_get_cached", AsyncMock(return_value={"lat": 41.2, "lon": -73.8})), \
|
||||
patch.object(intel, "_get_node_state", AsyncMock(return_value="New York")), \
|
||||
patch.object(pv.area_context, "resolve_anchor", AsyncMock(return_value=resolved)) as res, \
|
||||
patch.object(pv, "_geocode_in_anchor", _geo):
|
||||
await pv.verify("c1", "Respond to Lexington Avenue", None, ["Lexington Avenue"],
|
||||
{}, {}, talkgroup_name="New York City PD", node_id="node-1")
|
||||
res.assert_awaited_once_with({"municipality": "New York City", "state": "New York"})
|
||||
assert seen["anchor"] == {"lat": 40.71, "lng": -74.0, "radius_km": 25.0}
|
||||
assert seen["query"] == "Lexington Avenue, New York City, New York"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_state_means_no_derived_anchor():
|
||||
"""'Springfield PD' alone could anchor anywhere — skip, never guess."""
|
||||
with patch.object(pv.settings, "place_verification_enabled", True), \
|
||||
patch.object(intel.fstore, "doc_get_cached", AsyncMock(return_value={"lat": 41.2, "lon": -73.8})), \
|
||||
patch.object(intel, "_get_node_state", AsyncMock(return_value="")), \
|
||||
patch.object(pv, "_geocode_in_anchor") as geo:
|
||||
out = await pv.verify("c1", "Respond to Main Street", None, ["Main Street"],
|
||||
{}, {}, talkgroup_name="Springfield PD", node_id="node-1")
|
||||
assert out == (None, None)
|
||||
geo.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_configured_place_without_anchor_still_skips():
|
||||
"""A place that is set but too wide (no anchor stored) keeps the
|
||||
load-bearing skip; the talkgroup-name fallback is only for unset areas."""
|
||||
with patch.object(pv.settings, "place_verification_enabled", True), \
|
||||
patch.object(pv, "_talkgroup_anchor", AsyncMock()) as derived, \
|
||||
patch.object(pv, "_geocode_in_anchor") as geo:
|
||||
out = await pv.verify("c1", "Main Street", None, ["Main Street"],
|
||||
{"state": "Colorado"}, {}, talkgroup_name="Denver PD", node_id="n")
|
||||
assert out == (None, None)
|
||||
derived.assert_not_awaited()
|
||||
geo.assert_not_called()
|
||||
@@ -0,0 +1,14 @@
|
||||
"""server-26#88 — resolved/reassignment are parsed, not bool()-coerced."""
|
||||
import pytest
|
||||
|
||||
from app.internal.incident_correlator import parse_llm_bool as _llm_bool
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value,expected", [
|
||||
(True, True), (False, False),
|
||||
("true", True), ("TRUE", True), (" Yes ", True), ("yes", True),
|
||||
("false", False), ("False", False), ("no", False), ("NO", False),
|
||||
("", False), ("maybe", False), (None, False), (1, False), (0, False), ([], False),
|
||||
])
|
||||
def test_llm_bool(value, expected):
|
||||
assert _llm_bool(value) is expected
|
||||
@@ -0,0 +1,279 @@
|
||||
"""
|
||||
server-26#297 — config-driven STT provider with OpenAI as the fallback.
|
||||
|
||||
The default must stay OpenAI until a Groq key exists (so prod does not break
|
||||
before the key is provisioned), a failing primary must fall back to whisper-1
|
||||
without losing the call, and the vocabulary prompt must never carry a
|
||||
ten-code series (the hallucination documented in transcription.py).
|
||||
"""
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.internal import transcription as tr
|
||||
from app.config import settings
|
||||
|
||||
|
||||
def _settings(**over):
|
||||
base = dict(stt_provider="auto", groq_api_key=None, openai_api_key="sk-openai",
|
||||
stt_api_key=None, stt_base_url=None, stt_fallback_to_openai=True,
|
||||
stt_model="whisper-1", stt_compat_model="whisper-large-v3-turbo")
|
||||
base.update(over)
|
||||
return [patch.object(settings, k, v) for k, v in base.items()]
|
||||
|
||||
|
||||
class _patched:
|
||||
def __init__(self, **over):
|
||||
self.ps = _settings(**over)
|
||||
|
||||
def __enter__(self):
|
||||
for p in self.ps:
|
||||
p.start()
|
||||
|
||||
def __exit__(self, *a):
|
||||
for p in self.ps:
|
||||
p.stop()
|
||||
|
||||
|
||||
# ── provider resolution ─────────────────────────────────────────────────────
|
||||
|
||||
def test_default_without_groq_key_is_openai_only():
|
||||
with _patched():
|
||||
targets = tr.stt_targets()
|
||||
assert [(t.provider, t.model, t.tier) for t in targets] == [("openai", "whisper-1", "transcription")]
|
||||
assert targets[0].vocab_prompt is False, "the OpenAI path keeps today's fixed prompt"
|
||||
|
||||
|
||||
def test_groq_key_makes_groq_primary_with_openai_fallback():
|
||||
with _patched(groq_api_key="gsk"):
|
||||
targets = tr.stt_targets()
|
||||
assert [(t.provider, t.model) for t in targets] == [
|
||||
("groq", "whisper-large-v3-turbo"), ("openai", "whisper-1")]
|
||||
assert targets[0].base_url == "https://api.groq.com/openai/v1"
|
||||
assert targets[0].api_key == "gsk"
|
||||
assert targets[1].tier == "transcription_fallback"
|
||||
|
||||
|
||||
def test_fallback_can_be_switched_off():
|
||||
with _patched(groq_api_key="gsk", stt_fallback_to_openai=False):
|
||||
assert [t.provider for t in tr.stt_targets()] == ["groq"]
|
||||
|
||||
|
||||
def test_no_openai_key_means_no_fallback():
|
||||
with _patched(groq_api_key="gsk", openai_api_key=None):
|
||||
assert [t.provider for t in tr.stt_targets()] == ["groq"]
|
||||
|
||||
|
||||
def test_compat_server_from_base_url():
|
||||
with _patched(stt_base_url="http://whisper.local/v1", stt_compat_model="large-v3"):
|
||||
targets = tr.stt_targets()
|
||||
assert targets[0].provider == "compat" and targets[0].base_url == "http://whisper.local/v1"
|
||||
assert targets[0].model == "large-v3"
|
||||
|
||||
|
||||
def test_explicit_groq_without_key_degrades_to_openai():
|
||||
with _patched(stt_provider="groq"):
|
||||
assert [t.provider for t in tr.stt_targets()] == ["openai"]
|
||||
|
||||
|
||||
def test_nothing_configured_is_empty():
|
||||
with _patched(openai_api_key=None):
|
||||
assert tr.stt_targets() == []
|
||||
|
||||
|
||||
# ── fallback in transcribe_call ─────────────────────────────────────────────
|
||||
|
||||
def _ok(text="Engine 5 responding to a structure fire on Main Street"):
|
||||
return {"text": text, "segments": [{"start": 0.0, "end": 3.0, "text": text}],
|
||||
"degenerate": False, "duration": 3.0, "confidence": 0.81}
|
||||
|
||||
|
||||
async def _transcribe(sync_impl, **over):
|
||||
saved: dict = {}
|
||||
|
||||
async def _doc_set(coll, doc_id, data, merge=True):
|
||||
saved.update(data)
|
||||
|
||||
flags = {"transcript_correction_enabled": False}
|
||||
with _patched(**over), \
|
||||
patch.object(tr, "_download", lambda uri: "/nonexistent.flac"), \
|
||||
patch.object(tr, "_sync_transcribe_file", sync_impl), \
|
||||
patch.object(tr.transcript_correction, "resolve_context", AsyncMock(return_value={})), \
|
||||
patch.object(tr.fstore, "doc_set", _doc_set), \
|
||||
patch.object(tr, "ai_health") as health, \
|
||||
patch("app.internal.feature_flags.resolve_flags",
|
||||
AsyncMock(return_value=(flags, lambda k: flags.get(k, True)))):
|
||||
health.report_healthy = AsyncMock()
|
||||
health.report_degraded = AsyncMock()
|
||||
health.classify = lambda text: "transient"
|
||||
result = await tr.transcribe_call("call-1", "gs://b/call-1.flac", "PD Dispatch", system_id="sys-1")
|
||||
return result, saved, health
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_primary_failure_falls_back_to_openai():
|
||||
seen = []
|
||||
|
||||
def sync(path, target, prompt, glossary=None):
|
||||
seen.append(target.provider)
|
||||
if target.provider == "groq":
|
||||
raise RuntimeError("503 Service Unavailable")
|
||||
return _ok()
|
||||
|
||||
(text, segments), saved, health = await _transcribe(sync, groq_api_key="gsk")
|
||||
assert seen == ["groq", "openai"]
|
||||
assert text.startswith("Engine 5")
|
||||
assert saved["stt_provider"] == "openai" and saved["stt_model"] == "whisper-1"
|
||||
# The primary stays red; only the fallback tier is marked healthy.
|
||||
assert health.report_degraded.await_args.args[:3] == ("transcription", "groq", "whisper-large-v3-turbo")
|
||||
health.report_healthy.assert_awaited_once_with("transcription_fallback")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_primary_success_records_provider_and_cost_inputs():
|
||||
(text, _), saved, health = await _transcribe(lambda *a, **k: _ok(), groq_api_key="gsk")
|
||||
assert saved["stt_provider"] == "groq"
|
||||
assert saved["stt_model"] == "whisper-large-v3-turbo"
|
||||
assert saved["stt_audio_seconds"] == 3.0
|
||||
assert saved["stt_confidence"] == 0.81
|
||||
health.report_healthy.assert_awaited_once_with("transcription")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_every_target_failing_returns_nothing():
|
||||
def sync(*a, **k):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
(text, segments), saved, health = await _transcribe(sync, groq_api_key="gsk")
|
||||
assert (text, segments) == (None, [])
|
||||
assert health.report_degraded.await_count == 2
|
||||
health.report_healthy.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_config_never_calls_groq():
|
||||
seen = []
|
||||
|
||||
def sync(path, target, prompt, glossary=None):
|
||||
seen.append((target.provider, target.base_url))
|
||||
return _ok()
|
||||
|
||||
await _transcribe(sync)
|
||||
assert seen == [("openai", None)]
|
||||
|
||||
|
||||
# ── prompt ──────────────────────────────────────────────────────────────────
|
||||
|
||||
CONTEXT = {
|
||||
"area": {"municipality": "Ossining", "county": "Westchester", "state": "New York",
|
||||
"local_knowledge": [{"term": "Snowden Avenue"}, {"term": "Sing Sing", "meaning": "prison"}]},
|
||||
"vocabulary": ["10-13", "5-David", "Croton Point", "Snowden Avenue"],
|
||||
"ten_codes": {"10-4": "ack", "10-13": "officer needs assistance"},
|
||||
}
|
||||
|
||||
|
||||
def test_no_context_is_the_legacy_prompt():
|
||||
prompt, glossary = tr.build_stt_prompt("PD Dispatch", None)
|
||||
assert prompt == "Talkgroup: PD Dispatch. " + tr._WHISPER_PROMPT
|
||||
assert glossary == []
|
||||
|
||||
|
||||
def test_vocabulary_prompt_carries_places_but_never_ten_codes():
|
||||
prompt, glossary = tr.build_stt_prompt("PD Dispatch", CONTEXT)
|
||||
assert "Ossining, Westchester, New York" in prompt
|
||||
assert glossary == ["Snowden Avenue", "Sing Sing", "5-David", "Croton Point"]
|
||||
assert "10-13" not in prompt, "a code-shaped vocabulary term must stay out"
|
||||
assert "officer needs assistance" not in prompt
|
||||
assert prompt.endswith(tr._WHISPER_PROMPT)
|
||||
|
||||
|
||||
def test_prompt_stays_inside_the_token_budget():
|
||||
big = {"area": {}, "vocabulary": [f"Placename{i} Boulevard" for i in range(200)]}
|
||||
with patch.object(settings, "stt_prompt_max_tokens", 200):
|
||||
prompt, glossary = tr.build_stt_prompt("PD Dispatch", big)
|
||||
assert tr._estimate_tokens(prompt) <= 200
|
||||
assert 0 < len(glossary) < 200
|
||||
|
||||
|
||||
def test_token_estimate_is_conservative_for_radio_text():
|
||||
# GPT-2 tokenises this at ~25 tokens; the estimate must not be lower.
|
||||
assert tr._estimate_tokens("Engine 5 responding to a structure fire on Main Street, 10-4.") >= 20
|
||||
|
||||
|
||||
# ── degenerate output the vocabulary prompt could add ───────────────────────
|
||||
|
||||
def test_prompt_echo_is_discarded():
|
||||
glossary = ["Snowden Avenue", "Sing Sing", "Croton Point", "Main Street"]
|
||||
assert tr._is_degenerate("Snowden Avenue, Sing Sing, Croton Point, Main Street.", [], glossary)
|
||||
assert tr._is_degenerate("Local names: Snowden Avenue, Sing Sing.", [], glossary)
|
||||
|
||||
|
||||
def test_one_real_street_in_a_sentence_is_not_an_echo():
|
||||
glossary = ["Snowden Avenue", "Sing Sing", "Croton Point"]
|
||||
real = "Car 4 respond to Snowden Avenue for a report of a disabled vehicle."
|
||||
assert not tr._is_degenerate(real, [], glossary)
|
||||
|
||||
|
||||
def test_compression_ratio_catches_an_alternating_loop():
|
||||
loop = "Dispatch, copy. Unit 4, go ahead. " * 12
|
||||
assert not tr._phrase_loop(loop), "the old checks miss this shape"
|
||||
assert tr._is_degenerate(loop, [])
|
||||
|
||||
|
||||
def test_real_long_traffic_passes_the_compression_check():
|
||||
real = (
|
||||
"Headquarters to 5 David, respond to 12 Snowden Avenue for a report of a "
|
||||
"two-car accident with injuries, caller states one vehicle is overturned. "
|
||||
"5 David copies, en route from Route 9. Be advised EMS has been notified "
|
||||
"and Ossining Fire is responding as well. 5 David on scene, two vehicles, "
|
||||
"one patient complaining of neck pain, requesting a tow for one."
|
||||
)
|
||||
assert len(real) >= tr._MIN_CHARS_FOR_COMPRESSION
|
||||
assert not tr._is_degenerate(real, [])
|
||||
|
||||
|
||||
def test_sweep_hold_back_counts_the_fallback_attempt():
|
||||
from app.internal import recorrelation_sweep
|
||||
with _patched():
|
||||
single = recorrelation_sweep.pipeline_worst_case_seconds()
|
||||
with _patched(groq_api_key="gsk"):
|
||||
double = recorrelation_sweep.pipeline_worst_case_seconds()
|
||||
stt_one = (2 if settings.stt_retry_on_degenerate else 1) * (1 + settings.openai_max_retries) \
|
||||
* settings.stt_timeout_seconds
|
||||
assert double - single == stt_one
|
||||
assert double / 60 < settings.recorrelation_scan_minutes, "the sweep must still be able to recover the call"
|
||||
|
||||
|
||||
def test_location_readback_is_not_an_echo():
|
||||
"""Review of #297: intersection/address readbacks are where pins come from."""
|
||||
glossary = ["Route 9", "Croton Avenue", "North Highland Place", "Main Street", "Ossining"]
|
||||
for real in ("Route 9 and Croton Avenue, North Highland Place.",
|
||||
"Main Street, Croton Avenue, Route 9, Ossining."):
|
||||
assert not tr._is_degenerate(real, [], glossary), real
|
||||
|
||||
|
||||
def test_long_busy_call_is_judged_per_window():
|
||||
seg_text = "Engine 5 responding, Ladder 2 responding, Medic 1 en route to Main Street. "
|
||||
segments = [{"start": float(i * 5), "end": float(i * 5 + 4), "text": seg_text} for i in range(60)]
|
||||
text = " ".join(s["text"] for s in segments)
|
||||
# One repeated real update for 5 minutes still loops within every window,
|
||||
# so this is degenerate — but a varied call with one looping window out of three is not.
|
||||
assert tr._is_degenerate(text, segments)
|
||||
real = [
|
||||
"Headquarters to 5 David, respond to 12 Snowden Avenue for a two-car accident.",
|
||||
"5 David copies, en route from Route 9, ETA four minutes.",
|
||||
"Be advised EMS has been notified and Ossining Fire is responding as well.",
|
||||
"5 David on scene, two vehicles, one patient complaining of neck pain.",
|
||||
"Requesting a tow for the blue sedan, it is blocking the northbound lane.",
|
||||
"Copy, tow is being contacted, any further units needed at this time?",
|
||||
"Negative, traffic is backing up toward Croton Point, send one for control.",
|
||||
"6 Henry, can you clear the larceny and assist 5 David on Snowden?",
|
||||
"6 Henry clearing, show me en route, coming down from North Highland.",
|
||||
"Medic 1 on location, patient is being packaged for transport to Phelps.",
|
||||
"Fire is standing by for fluids, no entrapment, both drivers out of the cars.",
|
||||
"Received, all units be advised the roadway is now closed southbound.",
|
||||
]
|
||||
loop = "Dispatch, copy. Unit 4, go ahead. " * 12
|
||||
varied = [{"start": float(i * 5), "end": float(i * 5 + 4), "text": t} for i, t in enumerate(real)]
|
||||
varied.append({"start": 70.0, "end": 98.0, "text": loop}) # its own window
|
||||
assert not tr._is_degenerate(" ".join(s["text"] for s in varied), varied)
|
||||
@@ -260,7 +260,10 @@ async def test_segment_code_change_discards_segments_only():
|
||||
"segments": ["Headquarters, 10-13.", "Show it out to Ossining.", "360 north, back to Route 9."],
|
||||
}
|
||||
with _system(), _gemini(payload):
|
||||
text, segs, _ = await tc.correct("c1", "x y z w", SEGS, system_id="sys-1", talkgroup_id=9048)
|
||||
# A realistic joined original (code-free, like the correction): a
|
||||
# placeholder like "x y z w" now trips the server-26#162 growth guard.
|
||||
text, segs, _ = await tc.correct("c1", "Shout out to Optum, back to Rose.", SEGS,
|
||||
system_id="sys-1", talkgroup_id=9048)
|
||||
assert segs is None
|
||||
assert text == "Show it out to Ossining, back to Route 9."
|
||||
|
||||
@@ -279,3 +282,30 @@ async def test_reference_data_reaches_the_prompt():
|
||||
assert "Sing Sing — state prison" in p, "a term without its meaning is half the information"
|
||||
assert "Ossining, Westchester, New York" in p, "state must reach the prompt (server-26#36)"
|
||||
assert "10-13=officer needs assistance" in p
|
||||
|
||||
|
||||
# ── server-26#162, the rest: bare numbers and invented words ────────────────
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_bare_number_turned_into_a_word_is_rejected():
|
||||
"""The live case: a bare "7" came back as "ShotSpotter"."""
|
||||
payload = {"corrected": "Car ShotSpotter on Home Street and Forest Ave."}
|
||||
with _system(), _gemini(payload):
|
||||
text, _, _ = await tc.correct("c1", "Car 7 on Holmes Street and 4th and Rowe.", None, system_id="sys-1")
|
||||
assert text is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_place_fix_that_keeps_its_numbers_still_passes():
|
||||
payload = {"corrected": "Car 7 on Home Street and Forest Ave."}
|
||||
with _system(), _gemini(payload):
|
||||
text, _, _ = await tc.correct("c1", "Car 7 on Holmes Street and 4th and Rowe.", None, system_id="sys-1")
|
||||
assert text == "Car 7 on Home Street and Forest Ave."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_correction_that_writes_new_sentences_is_rejected():
|
||||
payload = {"corrected": "Car 7 on Main Street. Suspect is armed and fled northbound in a red sedan."}
|
||||
with _system(), _gemini(payload):
|
||||
text, _, _ = await tc.correct("c1", "Car 7 on Mane Street.", None, system_id="sys-1")
|
||||
assert text is None
|
||||
|
||||
Reference in New Issue
Block a user