Merge feat/297-cheaper-ai into feat/299-correlation-simplify

Resolve upload.py (no-scene fallback uses the new _correlate with #298's
non_event_chatter reason) and chatter_classifier docstring (gate built, off by
default). Fold intelligence._llm_bool into incident_correlator.parse_llm_bool (#88).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Logan Cusano
2026-10-27 20:31:54 -04:00
co-authored by Claude Opus 5.5
22 changed files with 2046 additions and 231 deletions
+18
View File
@@ -34,6 +34,24 @@ def openai_client(timeout: Optional[float] = None):
)
def stt_client(api_key: Optional[str], base_url: Optional[str] = None):
"""
Synchronous client for an OpenAI-compatible transcription endpoint
(server-26#297) -- OpenAI itself, Groq, or a self-hosted server. Same
timeout/retry limits as every other client here, with STT's own timeout.
"""
from openai import OpenAI
kwargs = {}
if base_url:
kwargs["base_url"] = base_url
return OpenAI(
api_key=api_key,
timeout=settings.stt_timeout_seconds,
max_retries=settings.openai_max_retries,
**kwargs,
)
def async_openai_client(timeout: Optional[float] = None):
"""Async OpenAI client, same limits as openai_client()."""
from openai import AsyncOpenAI
+208 -1
View File
@@ -2,9 +2,19 @@
Alert dispatch engine.
Loads enabled alert rules from Firestore and checks each one against the call's
talkgroup ID, tags, and transcript. On a match:
talkgroup ID, tags, and transcript. On a match that passes the confidence
gate (server-26#300, `_assess`):
1. Creates an AlertEvent document in Firestore.
2. Optionally POSTs a Discord webhook message if the rule has one configured.
A match that fails the gate is written to `alert_holds` instead -- never
pushed, readable by admins (GET /alerts/held) -- and a hold that was waiting
only on corroboration is released when a later call brings its incident to
alert_corroboration_min_calls calls.
WHY A GATE: CrimeRadar pushed "shots fired at an elementary school" after STT
heard "fire alarm" as "firearm". A keyword hit in raw transcript text is the
weakest evidence this pipeline has, and a push alert is the one surface that
cannot be taken back.
Raw transcript text is withheld from both by default -- see _snippet_allowed
and server-26#85.
@@ -54,6 +64,8 @@ async def check_and_dispatch(
# Loop-invariant: every rule here belongs to the same org, so the opt-in is
# resolved once rather than per match.
snippet_allowed = await _snippet_allowed(org_id)
evidence = await _evidence(call_doc or {})
pushed_rules: set[str] = set()
for rule in rules:
matched_keywords = _match_rule(rule, talkgroup_id, tags, transcript)
@@ -63,6 +75,12 @@ async def check_and_dispatch(
alert_id = str(uuid.uuid4())
snippet = _snippet(transcript) if snippet_allowed else None
now = datetime.now(timezone.utc).isoformat()
verdict = _assess(matched_keywords, tags, evidence)
if not verdict["push"]:
await _hold(alert_id, rule, org_id, call_id, node_id, talkgroup_id,
talkgroup_name, matched_keywords, snippet, now, verdict, evidence)
continue
pushed_rules.add(rule.get("rule_id", ""))
event = {
"alert_id": alert_id,
"org_id": org_id,
@@ -76,6 +94,9 @@ async def check_and_dispatch(
"transcript_snippet": snippet,
"triggered_at": now,
"acknowledged": False,
"confidence": verdict["confidence"],
"high_severity": verdict["high_severity"],
"corroborating_calls": evidence["max_calls"],
}
try:
@@ -92,6 +113,192 @@ async def check_and_dispatch(
if webhook_url:
await _post_webhook(webhook_url, rule.get("name", ""), talkgroup_name, matched_keywords, snippet)
await _release_corroborated(rules, evidence, call_id, pushed_rules)
# ── Confidence gate (server-26#300) ─────────────────────────────────────────
# How a rule matched, as evidence. Talkgroup activity is metadata, not
# recognised speech, so it cannot be misheard.
_W_TALKGROUP = 1.0
_W_TAG = 0.8 # the extraction model put it in tags
_W_TRANSCRIPT = 0.6 # substring of raw STT text only (above the 0.5 default floor)
def _norm_term(term: str) -> str:
return (term or "").strip().lower().replace("-", " ")
def _is_high_severity_term(term: str) -> bool:
wanted = {_norm_term(t) for t in settings.alert_high_severity_terms}
return _norm_term(term) in wanted
async def _evidence(call_doc: dict) -> dict:
"""What the call doc and its incidents say about how much to trust it."""
incident_ids = list(call_doc.get("incident_ids") or [])
max_calls = 0
max_severity = call_doc.get("severity")
severity_rank = {"routine": 0, "minor": 1, "moderate": 2, "major": 3}
for iid in incident_ids:
try:
inc = await fstore.doc_get("incidents", iid) or {}
except Exception as e:
logger.warning(f"Alerter: could not read incident {iid}: {e}")
continue
max_calls = max(max_calls, len(inc.get("call_ids") or []))
if severity_rank.get(inc.get("severity"), -1) > severity_rank.get(max_severity, -1):
max_severity = inc.get("severity")
stt = call_doc.get("stt_confidence")
return {
"incident_ids": incident_ids,
"max_calls": max_calls,
"severity": max_severity,
"stt_confidence": stt if isinstance(stt, (int, float)) else None,
"not_speech": bool(call_doc.get("transcript_not_speech")),
}
def _assess(matched: list[str], tags: list[str], evidence: dict) -> dict:
"""
Push or hold, and why. Pure, so the gate is testable without Firestore.
confidence = match weight x STT factor. The STT factor is 1.0 unless
Whisper's own mean token probability is under alert_stt_reference, and
1.0 when there is none (a json-only model) rather than guessing.
"""
content = [m for m in matched if not m.startswith("talkgroup:")]
if not content:
weight = _W_TALKGROUP
elif any(m in (tags or []) for m in content):
weight = _W_TAG
else:
weight = _W_TRANSCRIPT
stt = evidence.get("stt_confidence")
stt_factor = 1.0
if content and stt is not None and settings.alert_stt_reference > 0:
stt_factor = min(1.0, max(0.0, stt / settings.alert_stt_reference))
if content and evidence.get("not_speech"):
stt_factor = 0.0 # the corrector called this recogniser noise
confidence = round(weight * stt_factor, 3)
high = bool(content) and (
any(_is_high_severity_term(m) for m in content)
or any(_is_high_severity_term(t) for t in (tags or []))
or evidence.get("severity") == "major"
)
corroborated = evidence.get("max_calls", 0) >= settings.alert_corroboration_min_calls
if confidence < settings.alert_min_confidence:
return {"push": False, "confidence": confidence, "high_severity": high,
"hold_reason": "low_confidence"}
if high and not corroborated and confidence < settings.alert_high_severity_min_confidence:
return {"push": False, "confidence": confidence, "high_severity": high,
"hold_reason": "needs_corroboration"}
return {"push": True, "confidence": confidence, "high_severity": high, "hold_reason": None}
async def _hold(alert_id, rule, org_id, call_id, node_id, talkgroup_id, talkgroup_name,
matched_keywords, snippet, now, verdict, evidence) -> None:
"""Record a gated match where admins can see it, and push nothing."""
hold = {
"alert_id": alert_id,
"org_id": org_id,
"rule_id": rule.get("rule_id", ""),
"rule_name": rule.get("name", ""),
"call_id": call_id,
"node_id": node_id,
"talkgroup_id": talkgroup_id,
"talkgroup_name": talkgroup_name or "",
"matched_keywords": matched_keywords,
"transcript_snippet": snippet,
"triggered_at": now,
"status": "held",
"hold_reason": verdict["hold_reason"],
"confidence": verdict["confidence"],
"high_severity": verdict["high_severity"],
# One incident to wait on: the first the call linked to. A call that
# linked to none can never be corroborated, so it stays held.
"incident_id": (evidence["incident_ids"] or [None])[0],
"corroborating_calls": evidence["max_calls"],
}
try:
await fstore.doc_set("alert_holds", alert_id, hold, merge=False)
except Exception as e:
logger.warning(f"Alerter: could not save held alert: {e}")
return
logger.info(
f"Alert HELD ({verdict['hold_reason']}, confidence={verdict['confidence']}, "
f"high_severity={verdict['high_severity']}, calls={evidence['max_calls']}): "
f"rule='{rule.get('name')}' call={call_id} keywords={matched_keywords}"
)
async def _release_corroborated(rules: list[dict], evidence: dict, call_id: str,
pushed_rules: set[str]) -> None:
"""
Push holds that were waiting only on corroboration once their incident
has enough calls. A low-confidence hold is never released this way: a
second call on the incident says the event is real, not that the first
call was heard right. A rule this call already pushed for supersedes its
hold instead of alerting twice.
"""
if evidence["max_calls"] < settings.alert_corroboration_min_calls:
return
by_id = {r.get("rule_id", ""): r for r in rules}
for iid in evidence["incident_ids"]:
try:
holds = await fstore.collection_list("alert_holds", incident_id=iid, status="held")
except Exception as e:
logger.warning(f"Alerter: could not read holds for incident {iid}: {e}")
continue
for hold in holds:
if hold.get("hold_reason") != "needs_corroboration":
continue
rule = by_id.get(hold.get("rule_id", ""))
now = datetime.now(timezone.utc).isoformat()
if rule is None or hold.get("rule_id", "") in pushed_rules:
status = "superseded" if rule is not None else "rule_gone"
await _set_hold_status(hold, status, call_id, now)
continue
event = {k: hold.get(k) for k in (
"alert_id", "org_id", "rule_id", "rule_name", "call_id", "node_id",
"talkgroup_id", "talkgroup_name", "matched_keywords",
"transcript_snippet", "confidence", "high_severity",
)}
event.update({
"triggered_at": now,
"acknowledged": False,
"released_from_hold": True,
"corroborated_by_call_id": call_id,
"corroborating_calls": evidence["max_calls"],
})
try:
await fstore.doc_set("alert_events", hold["alert_id"], event, merge=False)
except Exception as e:
logger.warning(f"Alerter: could not release held alert {hold.get('alert_id')}: {e}")
continue
await _set_hold_status(hold, "released", call_id, now)
pushed_rules.add(hold.get("rule_id", ""))
logger.info(
f"Alert released from hold: rule='{hold.get('rule_name')}' incident={iid} "
f"corroborated by call {call_id} ({evidence['max_calls']} calls)"
)
webhook_url = rule.get("discord_webhook")
if webhook_url:
await _post_webhook(webhook_url, rule.get("name", ""), hold.get("talkgroup_name"),
hold.get("matched_keywords") or [], hold.get("transcript_snippet"))
async def _set_hold_status(hold: dict, status: str, call_id: str, now: str) -> None:
try:
await fstore.doc_update("alert_holds", hold["alert_id"], {
"status": status, "resolved_at": now, "resolved_by_call_id": call_id,
})
except Exception as e:
logger.warning(f"Alerter: could not update hold {hold.get('alert_id')}: {e}")
async def _snippet_allowed(org_id: Optional[str]) -> bool:
"""
+45 -7
View File
@@ -14,13 +14,18 @@ This module is that classifier. It is a PURE function of the transcript text —
no Firestore, no LLM call, no side effects — so it is cheap to run on every
transcript and cheap to test against real dumps offline.
SHADOW MODE ONLY. As of this module's introduction, nothing skips scene
extraction based on this verdict. `intelligence.extract_scenes` calls
`classify_chatter` purely to record the verdict on the call doc
(`chatter_classifier_verdict` / `chatter_classifier_reason`) so it becomes
observable in the next `/admin` correlation-debug dump (as `corr_gate_veto`
was, before server-26#299 removed that gate). See the TODO at that call site for
what has to be true before this flips live.
A COST GATE SINCE server-26#298, through `gate_chatter`, not
`classify_chatter` — built but OFF by default (`chatter_gate_enabled`) until a
shadow window shows gated calls don't over-merge. `classify_chatter` is still
the shadow verdict recorded on every call doc (`chatter_classifier_verdict` /
`chatter_classifier_reason`), so the `/admin` tallies keep measuring the same
thing. The gate is stricter on the one shape the #127 review flagged as a live
false-positive risk: `roll_call` matches "roll call" anywhere, so "Delayed for
roll call, then respond to a structure fire at 12 Main Street" is a roll_call
verdict. The gate only skips a roll call whose remaining words are callsigns
and roll-call phrasing. When enabled, gated transcripts skip transcript
correction and scene extraction (and with it embedding and geocoding); the free
rule paths (clearance parse, self-initiated/plate backstop) still run.
Precision over recall, deliberately. A false positive here — flagging a REAL
event as chatter — would, once live, silently mean that event never gets a
@@ -133,3 +138,36 @@ def classify_chatter(transcript: Optional[str]) -> tuple[bool, Optional[str]]:
return False, None
return True, "bare_acknowledgement"
# Words a roll call is made of, beyond the filler/designator/phonetic sets.
_ROLL_CALL_WORDS = frozenset({
"roll", "call", "all", "cars", "car", "patrol", "patrols", "troop", "staff",
"concludes", "time", "came", "because", "volunteer", "nyc", "kl", "henry",
"here", "present", "s", "m", "it",
})
# A roll call may name a place or two ("Post 4, Ossining"); more than this many
# leftover words means something else is being said in the same transmission.
_ROLL_CALL_MAX_LEFTOVER = 2
def gate_chatter(transcript: Optional[str]) -> tuple[bool, Optional[str]]:
"""
Whether to skip paid AI for this transcript (server-26#298).
`classify_chatter`'s verdict, with roll_call narrowed: the transcript must
be roll-call content end to end, not a roll call mentioned on the way to
something real. Precision over recall still — a false positive here means
a real event never gets a scene.
"""
is_chatter, reason = classify_chatter(transcript)
if not is_chatter:
return False, None
if reason == "roll_call":
leftover = [
t for t in _tokenize(transcript or "")
if not _is_filler_token(t) and t not in _ROLL_CALL_WORDS
]
if len(leftover) > _ROLL_CALL_MAX_LEFTOVER:
return False, None
return True, reason
+169 -81
View File
@@ -1,7 +1,8 @@
"""
GPT-4o-mini intelligence extraction from call transcripts.
Intelligence extraction from call transcripts.
Sends the transcript to GPT-4o-mini with a structured prompt that detects
Sends the transcript to the extraction model (settings.extraction_model,
gpt-4o-mini by default) with a structured prompt that detects
whether the recording contains one or multiple distinct scenes (back-to-back
dispatch conversations on a busy channel). Returns a list of scene dicts —
one per detected incident. Most calls produce a single scene.
@@ -12,12 +13,14 @@ import asyncio
import json
import math
import re
from collections import OrderedDict
from typing import Optional
from app.internal.logger import logger
from app.internal import firestore as fstore
from app.internal import ai_health
from app.internal import area_context
from app.internal.chatter_classifier import classify_chatter
from app.internal import incident_correlator
from app.internal.chatter_classifier import classify_chatter, gate_chatter
# Location validity is defined once, by the module that owns the incident's
# location/pin invariant. incident_correlator does not import this module, so
# this is not a cycle.
@@ -218,11 +221,12 @@ async def extract_scenes(
pass
return []
# server-26#127 — SHADOW MODE ONLY. Computes whether this transcript looks
# server-26#127 — the shadow verdict. Computes whether this transcript looks
# like non-event radio housekeeping (roll call, bare 10-4/10-8/98
# acknowledgements, unit check-ins) and records the verdict on the call
# doc, but does NOT skip extraction anywhere below — every path runs
# exactly as it did before this landed. Deliberately ahead of the ≤5-word
# acknowledgements, unit check-ins) and records it on the call doc. The
# live cost gate (server-26#298, gate_chatter below) is a stricter view of
# the same classifier; this verdict is recorded unchanged so the /admin
# tallies keep measuring the same thing. Deliberately ahead of the ≤5-word
# skip: most bare acknowledgements ARE ≤5 words, and the first pass of
# this feature put the classifier after that return, so it never saw the
# bulk of its own target population — a review backtest against three
@@ -231,13 +235,9 @@ async def extract_scenes(
# roughly a fifth of the real catch rate. Computing it once, here, and
# folding the result into whichever skip/continue path runs below fixes
# that without adding a second Firestore write.
# TODO(server-26#127): flip this from shadow to live (skip extraction and
# write skip_reason="non_event_chatter" instead of just recording the
# verdict) once a live shadow-mode window confirms 0 false positives on
# real production traffic — pay particular attention to whole-transcript
# vs contains-anywhere matching for "roll call" and to digit-hyphen street
# addresses (e.g. "72-Holland"), both flagged as classifier risks that the
# dump backtest could not surface on its own.
# Residual risk of the live gate, unchanged from the #127 review: a
# digit-hyphen street address ("72-Holland") alone in a transmission reads
# as a unit ID. CHATTER_GATE_ENABLED=false restores shadow mode.
chatter_is_chatter, chatter_reason = classify_chatter(transcript)
# Transcripts with ≤5 words carry no extractable intelligence — GPT hallucinates
@@ -277,6 +277,40 @@ async def extract_scenes(
"tags": tags, "incident_type": typ, "severity": sev}]
return []
# server-26#298 — the gate is live. Radio housekeeping has no incident
# content for the extraction model to find, so it no longer pays for an
# extraction, an embedding per scene and a geocode to come back as a thin
# routine scene. gate_chatter is classify_chatter with roll_call narrowed
# to whole-transcript roll calls (the #127 review's false-positive shape).
# The free rule paths still run, exactly as on the <=5-word path above.
from app.config import settings
gated, gate_reason = gate_chatter(transcript) if settings.chatter_gate_enabled else (False, None)
if gated:
logger.info(
f"Intelligence: call {call_id} — non-event chatter ({gate_reason}), skipping extraction"
)
cleared_unit = _short_clearance_unit(transcript)
tags, typ, sev = _self_initiated_backstop(transcript, [], None, "routine", talkgroup_name)
updates = {
"skip_reason": "non_event_chatter",
"severity": sev if tags else "routine",
"chatter_classifier_verdict": chatter_is_chatter,
"chatter_classifier_reason": chatter_reason,
}
if cleared_unit:
updates["units"] = [cleared_unit]
updates["cleared_units"] = [cleared_unit]
try:
await fstore.doc_set("calls", call_id, updates)
except Exception:
pass
if cleared_unit:
return [_clearance_scene(transcript, cleared_unit)]
if tags:
return [{**_clearance_scene(transcript, ""), "units": [], "cleared_units": [],
"tags": tags, "incident_type": typ, "severity": sev}]
return []
try:
await fstore.doc_set("calls", call_id, {
"chatter_classifier_verdict": chatter_is_chatter,
@@ -294,13 +328,14 @@ async def extract_scenes(
except Exception as e:
text = str(e)
kind = ai_health.classify(text)
logger.warning(f"GPT-4o-mini extraction failed for call {call_id}: {text}")
from app.config import settings as _s
logger.warning(f"Extraction ({_s.extraction_model}) failed for call {call_id}: {text}")
await ai_health.report_degraded(
"extraction", "openai", "gpt-4o-mini",
"extraction", "openai", _s.extraction_model,
{"billing": "the OpenAI account is out of credit",
"dead_model": "model is unavailable"}.get(kind, f"extraction failed: {text[:200]}"),
{"billing": "Top up OpenAI billing",
"dead_model": "Update the extraction model in intelligence.py"}.get(kind, "Usually transient"),
"dead_model": "Update EXTRACTION_MODEL"}.get(kind, "Usually transient"),
permanent=kind != "transient",
)
return []
@@ -364,8 +399,10 @@ async def extract_scenes(
severity: str = scene.get("severity") or "routine"
if severity == "unknown":
severity = "routine"
resolved: bool = bool(scene.get("resolved", False))
reassignment: bool = bool(scene.get("reassignment", False))
# server-26#88: bool("false") is True, which let a model answering
# with a JSON string resolve an incident it merely linked into.
resolved: bool = incident_correlator.parse_llm_bool(scene.get("resolved"))
reassignment: bool = incident_correlator.parse_llm_bool(scene.get("reassignment"))
transcript_corrected: Optional[str]= scene.get("transcript_corrected") or None
segment_indices: Optional[list] = scene.get("segment_indices")
@@ -729,7 +766,6 @@ async def _geocode_location(
touch the anchor path at all — an anchor's own radius is always authoritative
when one has been resolved.
"""
import httpx
from app.config import settings
if not settings.google_maps_api_key:
@@ -755,70 +791,122 @@ async def _geocode_location(
f"{ref_lat - delta},{ref_lon - delta}"
f"|{ref_lat + delta},{ref_lon + delta}"
)
params = {
"address": location_str,
"bounds": bounds,
"region": "us",
"key": settings.google_maps_api_key,
}
try:
async with httpx.AsyncClient(timeout=5.0) as client:
r = await client.get(
"https://maps.googleapis.com/maps/api/geocode/json",
params=params,
)
r.raise_for_status()
data = r.json()
if data.get("status") != "OK" or not data.get("results"):
return None
result = data["results"][0]
location_type = result.get("geometry", {}).get("location_type", "")
# Reject only APPROXIMATE — a region/city boundary centroid, which is
# what an ungeocodable string degrades to and is genuinely useless.
#
# ROOFTOP-only was too strict and emptied the map: dispatch names
# places the way people speak, and Google returns GEOMETRIC_CENTER for
# exactly those forms — intersections ("Lake Street and Veterans
# Memorial Drive") and named POIs ("Brewster Station"). Both are
# precise enough to plot and to proximity-match; requiring a street
# address threw away nearly every real dispatch location, leaving only
# numbered addresses geocoded.
if location_type not in ("ROOFTOP", "RANGE_INTERPOLATED", "GEOMETRIC_CENTER"):
logger.info(
f"Geocoding rejected '{location_str}' — imprecise result "
f"(location_type={location_type!r}), returning None"
)
return None
loc = result["geometry"]["location"]
lat, lng = float(loc["lat"]), float(loc["lng"])
dist_km = _geo_dist_km(ref_lat, ref_lon, lat, lng)
if dist_km > max_km:
# server-26#159: the node-distance bound is a proxy for "is
# this plausible" that only makes sense when the node's own
# position is our best guess at the area — never when the
# query already names a different region on its own terms.
if not (ref_label == "node" and trust_named_region):
logger.warning(
f"Geocoding rejected '{location_str}' → ({lat:.4f}, {lng:.4f}) "
f"— {dist_km:.1f}km from {ref_label} exceeds {max_km:.1f}km"
)
return None
logger.info(
f"Geocoding '{location_str}' → ({lat:.4f}, {lng:.4f}) is "
f"{dist_km:.1f}km from the receiving node, past {max_km:.1f}km — "
f"accepted anyway: the query names its own region, not the node's"
)
coords = {"lat": lat, "lng": lng, "location_type": location_type}
result = await _geocode_lookup(location_str, bounds)
if result is None:
return None
location_type = result.get("geometry", {}).get("location_type", "")
# Reject only APPROXIMATE — a region/city boundary centroid, which is
# what an ungeocodable string degrades to and is genuinely useless.
#
# ROOFTOP-only was too strict and emptied the map: dispatch names
# places the way people speak, and Google returns GEOMETRIC_CENTER for
# exactly those forms — intersections ("Lake Street and Veterans
# Memorial Drive") and named POIs ("Brewster Station"). Both are
# precise enough to plot and to proximity-match; requiring a street
# address threw away nearly every real dispatch location, leaving only
# numbered addresses geocoded.
if location_type not in ("ROOFTOP", "RANGE_INTERPOLATED", "GEOMETRIC_CENTER"):
logger.info(
f"Geocoded '{location_str}' → {coords} "
f"({dist_km:.1f}km from {ref_label}) [{location_type}]"
f"Geocoding rejected '{location_str}' — imprecise result "
f"(location_type={location_type!r}), returning None"
)
return coords
return None
loc = result["geometry"]["location"]
lat, lng = float(loc["lat"]), float(loc["lng"])
dist_km = _geo_dist_km(ref_lat, ref_lon, lat, lng)
if dist_km > max_km:
# server-26#159: the node-distance bound is a proxy for "is
# this plausible" that only makes sense when the node's own
# position is our best guess at the area — never when the
# query already names a different region on its own terms.
if not (ref_label == "node" and trust_named_region):
logger.warning(
f"Geocoding rejected '{location_str}' → ({lat:.4f}, {lng:.4f}) "
f"— {dist_km:.1f}km from {ref_label} exceeds {max_km:.1f}km"
)
return None
logger.info(
f"Geocoding '{location_str}' → ({lat:.4f}, {lng:.4f}) is "
f"{dist_km:.1f}km from the receiving node, past {max_km:.1f}km — "
f"accepted anyway: the query names its own region, not the node's"
)
coords = {"lat": lat, "lng": lng, "location_type": location_type}
logger.info(
f"Geocoded '{location_str}' → {coords} "
f"({dist_km:.1f}km from {ref_label}) [{location_type}]"
)
return coords
except Exception as e:
logger.warning(f"Geocoding failed for '{location_str}': {e}")
return None
# server-26#298: geocode answers, keyed by the exact request (address +
# bounds). Only the first result's geometry is kept — the minimum the checks
# in _geocode_location read — so the cache holds coordinates, not Maps
# content. The distance/precision checks still run per call on a hit, so a
# cached answer is judged against whatever anchor or node asks.
_geocode_cache: "OrderedDict[tuple[str, str], tuple[float, Optional[dict]]]" = OrderedDict()
geocode_cache_stats = {"hits": 0, "misses": 0}
def clear_geocode_cache() -> None:
_geocode_cache.clear()
geocode_cache_stats.update(hits=0, misses=0)
async def _geocode_lookup(address: str, bounds: str) -> Optional[dict]:
"""
The Geocoding API's first result for `address` inside `bounds`, or None
when it has none. Cached in process (settings.geocode_cache_*). A network
error or non-200 raises and is NOT cached — the next call retries.
"""
import time
import httpx
from app.config import settings
key = (address.strip().lower(), bounds)
now = time.monotonic()
hit = _geocode_cache.get(key)
if hit is not None:
expires, value = hit
if expires > now:
_geocode_cache.move_to_end(key)
geocode_cache_stats["hits"] += 1
return value
_geocode_cache.pop(key, None)
geocode_cache_stats["misses"] += 1
async with httpx.AsyncClient(timeout=5.0) as client:
r = await client.get(
"https://maps.googleapis.com/maps/api/geocode/json",
params={"address": address, "bounds": bounds, "region": "us",
"key": settings.google_maps_api_key},
)
r.raise_for_status()
data = r.json()
status = data.get("status")
value: Optional[dict] = None
if status == "OK" and data.get("results"):
geometry = data["results"][0].get("geometry") or {}
value = {"geometry": {"location": geometry.get("location") or {},
"location_type": geometry.get("location_type", "")}}
ttl = settings.geocode_cache_ttl_seconds
elif status == "ZERO_RESULTS":
ttl = settings.geocode_cache_negative_ttl_seconds
else:
# OVER_QUERY_LIMIT / REQUEST_DENIED / UNKNOWN_ERROR: a key or quota
# problem, not an answer about this address.
return None
if ttl > 0 and settings.geocode_cache_max_entries > 0:
_geocode_cache[key] = (now + ttl, value)
while len(_geocode_cache) > settings.geocode_cache_max_entries:
_geocode_cache.popitem(last=False)
return value
def _municipality_from_tg(tg_name: Optional[str]) -> Optional[str]:
"""
Extract the municipality name from a talkgroup name.
@@ -943,7 +1031,7 @@ def _sync_extract(
ten_codes: Optional[dict[str, str]] = None,
unit_format_hint: Optional[str] = None,
) -> list[dict]:
"""Call GPT-4o-mini and return a list of scene dicts."""
"""Call the extraction model and return a list of scene dicts."""
from app.config import settings
from app.internal.ai_clients import openai_client
@@ -965,7 +1053,7 @@ def _sync_extract(
try:
client = openai_client()
response = client.chat.completions.create(
model="gpt-4o-mini",
model=settings.extraction_model,
messages=[{"role": "user", "content": prompt}],
response_format={"type": "json_object"},
)
@@ -980,7 +1068,7 @@ def _sync_extract(
return [raw]
except json.JSONDecodeError as e:
logger.warning(f"GPT-4o-mini returned non-JSON: {e}")
logger.warning(f"Extraction model returned non-JSON: {e}")
return []
# Any other exception is the API call itself failing (no credit, rate
# limit, outage) and propagates to extract_scenes, which reports it to
@@ -990,7 +1078,7 @@ def _sync_extract(
def _sync_embed(text: str) -> Optional[list[float]]:
"""Generate a text-embedding-3-small vector for semantic similarity."""
"""Generate an embedding (settings.embedding_model) for semantic similarity."""
from app.config import settings
from app.internal.ai_clients import openai_client
@@ -999,7 +1087,7 @@ def _sync_embed(text: str) -> Optional[list[float]]:
try:
client = openai_client()
result = client.embeddings.create(model="text-embedding-3-small", input=text)
result = client.embeddings.create(model=settings.embedding_model, input=text)
return result.data[0].embedding
except Exception as e:
logger.warning(f"Embedding generation failed: {e}")
+57 -4
View File
@@ -23,7 +23,8 @@ permanently improves the reference data for that channel.
NO ANCHOR MEANS SKIP, NOT ACCEPT. An anchor too wide to discriminate is not
stored at all (see `area_context`), and without one this module returns
immediately. A statewide radius would confirm anything inside it, which is worse
immediately. A channel nobody described may still get one from its own
talkgroup name plus the node's state (server-26#160, `_talkgroup_anchor`). A statewide radius would confirm anything inside it, which is worse
than not checking — it looks like verification and is not.
THE FREE TIER RUNS FIRST. A sound-alike among the terms the operator already
@@ -169,6 +170,50 @@ def _known_soundalike(heard: str, area: dict) -> Optional[dict]:
return best
async def _talkgroup_anchor(
talkgroup_name: Optional[str], node_id: Optional[str],
) -> tuple[Optional[dict], str]:
"""
An anchor derived from the talkgroup's own name plus the receiving node's
state, or (None, ""). Same width rule as an operator anchor
(area_context.resolve_anchor drops anything wider than
area_anchor_max_radius_km, and caches per place for the process).
A STATE IS REQUIRED. "Springfield PD" alone geocodes to whichever
Springfield Google prefers, and an anchor in the wrong half of the
country would let Maps rewrite real local streets into sound-alikes from
there (the failure server-26#36 added `state` to prevent). The node's
reverse-geocoded state is coarse enough to be right even when the node
sits far from the area the channel covers.
"""
from app.internal import firestore as fstore
from app.internal.intelligence import _get_node_state, _municipality_from_tg
muni = _municipality_from_tg(talkgroup_name)
if not muni or not node_id:
return None, ""
node = await fstore.doc_get_cached("nodes", node_id) or {}
lat, lon = node.get("lat"), node.get("lon")
if lat is None or lon is None:
return None, ""
state = await _get_node_state(node_id, lat, lon)
if not state:
return None, ""
place = {"municipality": muni, "state": state}
resolved = await area_context.resolve_anchor(place)
if not resolved:
return None, ""
try:
anchor = {
"lat": float(resolved["center"]["lat"]),
"lng": float(resolved["center"]["lng"]),
"radius_km": float(resolved["radius_km"]),
}
except (KeyError, TypeError, ValueError):
return None, ""
return anchor, f"{muni}, {state}"
# -- Public --------------------------------------------------------------------
def _substitute(text: str, swaps: list[tuple[str, str]]) -> str:
@@ -186,6 +231,8 @@ async def verify(
tg_area: Optional[dict],
system_id: Optional[str] = None,
talkgroup_id: Optional[Any] = None,
talkgroup_name: Optional[str] = None,
node_id: Optional[str] = None,
) -> tuple[Optional[str], Optional[list[dict]]]:
"""
Check the corrector's location nouns against the talkgroup's anchor.
@@ -197,12 +244,18 @@ async def verify(
if not settings.place_verification_enabled or not locations:
return None, None
area = area_context.effective(system_area, tg_area)
anchor = area_context.anchor_for(system_area, tg_area)
suffix = _place_suffix(area)
if not anchor and not area_context.has_place(area):
# server-26#160: nobody described this channel, but its own name may
# (e.g. "New York City - NYPD Citywide 2 Patch") — the same signal
# #159 trusted in intelligence.py's geocoder. Only when an operator set
# NOTHING: a place that is set but has no anchor was too wide or is
# stale, and that skip stays load-bearing.
anchor, suffix = await _talkgroup_anchor(talkgroup_name, node_id)
if not anchor:
return None, None # load-bearing: no anchor means skip, never accept
area = area_context.effective(system_area, tg_area)
suffix = _place_suffix(area)
swaps: list[tuple[str, str]] = []
proposals: list[dict] = []
@@ -65,7 +65,11 @@ def pipeline_worst_case_seconds(n_scenes: int = _BOUND_SCENES) -> float:
_run_intelligence_pipeline."""
openai_attempts = 1 + max(0, settings.openai_max_retries)
stt_rounds = 2 if settings.stt_retry_on_degenerate else 1
stt = stt_rounds * openai_attempts * settings.stt_timeout_seconds
# A failed primary STT provider falls back to OpenAI (server-26#297), so
# the worst case is every configured target timing out in turn.
from app.internal.transcription import stt_targets
stt_providers = max(1, len(stt_targets()))
stt = stt_providers * stt_rounds * openai_attempts * settings.stt_timeout_seconds
# Transcript correction (inside transcribe_call). gemini.generate_json only
# re-issues a request when the model rejects a thinking level — a fast
# 400, not a timeout — and google-genai itself does not retry.
+4
View File
@@ -74,6 +74,9 @@ _DERIVED = {
"vehicles", "cleared_units", "severity", "incident_type", "type",
"embedding", "skip_reason", "intelligence_started_at", "reassignment",
"resolved", "has_updates", "audio_url",
# server-26#297: which STT produced the transcript, and its confidence —
# a re-transcription that comes back empty must not keep the live values.
"stt_provider", "stt_model", "stt_audio_seconds", "stt_confidence",
}
_DERIVED_PREFIXES = ("corr_", "chatter_classifier_", "eval_")
_TRANSCRIPT_FIELDS = (
@@ -407,6 +410,7 @@ async def _prepare(call: dict, mode: str, source_scenes: dict[str, dict]) -> dic
transcript, segments = await transcription.transcribe_call(
call_id, call["audio_gcs_uri"], talkgroup_name,
system_id=call.get("system_id"), talkgroup_id=call.get("talkgroup_id"),
node_id=call.get("node_id"),
)
else:
transcript, segments = _stored_input(call)
@@ -116,11 +116,37 @@ def _dedupe(items: list[str]) -> list[str]:
# mishearing this pass exists to fix (server-26#162).
_CODE_TOKEN_RE = re.compile(r"\b\d{1,3}(?:-\d{1,3})+\b")
# Bare numbers (the rest of server-26#162): the same call had a bare "7"
# turned into "ShotSpotter". A unit number, a post, a house number or a count
# the recogniser DID hear must survive, in order. A number may still be
# added, because a sound-alike can legitimately be one ("back to Rose" ->
# "back to Route 9"). "4th" is not bare (it is a word).
_BARE_NUMBER_RE = re.compile(r"(?<![\w-])\d+(?![\w-])")
# "NEVER add information" is the other prompt rule nothing checked. A
# correction substitutes sound-alikes, so it may shift the word count a little
# ("Holmes Street and 4th and Rowe" -> "Home Street and Forest Ave"); growing
# by more than this is the model writing, not correcting.
_MAX_ADDED_WORDS = 4
_MAX_GROWTH_RATIO = 0.25
def _code_tokens(text: str) -> list[str]:
return _CODE_TOKEN_RE.findall(text or "")
def _lost_numbers(original: str, corrected: str) -> bool:
"""True unless every bare number of `original` is still in `corrected`, in order."""
remaining = iter(_BARE_NUMBER_RE.findall(corrected or ""))
return not all(any(n == m for m in remaining) for n in _BARE_NUMBER_RE.findall(original or ""))
def _grew(original: str, corrected: str) -> bool:
n_orig = len((original or "").split())
n_new = len((corrected or "").split())
return n_new - n_orig > max(_MAX_ADDED_WORDS, int(n_orig * _MAX_GROWTH_RATIO))
def _talkgroup_entry(system_doc: dict, talkgroup_id: Optional[int]) -> dict:
"""The config.talkgroups[] entry for this talkgroup, or {}."""
if talkgroup_id is None:
@@ -241,6 +267,7 @@ async def correct(
system_id: Optional[str] = None,
talkgroup_id: Optional[int] = None,
talkgroup_name: Optional[str] = None,
node_id: Optional[str] = None,
) -> tuple[Optional[str], Optional[list[dict]], bool]:
"""
Second-opinion pass over a transcript.
@@ -330,6 +357,8 @@ async def correct(
context.get("tg_area"),
system_id=system_id,
talkgroup_id=talkgroup_id,
talkgroup_name=talkgroup_name,
node_id=node_id,
)
except Exception as e:
logger.warning(f"Place verification failed for call {call_id}: {e}")
@@ -354,6 +383,18 @@ async def correct(
f"discarding the joined correction"
)
corrected = None
if corrected is not None and _lost_numbers(text, corrected):
logger.warning(
f"Transcript correction for call {call_id} dropped or changed a number — "
f"discarding the joined correction"
)
corrected = None
if corrected is not None and _grew(text, corrected):
logger.warning(
f"Transcript correction for call {call_id} grew the transcript "
f"({len(text.split())} -> {len(corrected.split())} words) — discarding the joined correction"
)
corrected = None
if corrected_segments is not None:
for seg, orig in zip(corrected_segments, segments or []):
if _code_tokens(seg["text"]) != _code_tokens(orig.get("text", "")):
@@ -363,6 +404,13 @@ async def correct(
)
corrected_segments = None
break
if _lost_numbers(orig.get("text", ""), seg["text"]) or _grew(orig.get("text", ""), seg["text"]):
logger.warning(
f"Transcript correction for call {call_id} grew a segment or dropped "
f"a number in one — discarding segment corrections"
)
corrected_segments = None
break
if corrected or corrected_segments or not_speech:
changed = raw.get("changed") or []
+471 -131
View File
@@ -1,18 +1,25 @@
"""
Speech-to-text transcription for recorded calls using OpenAI Whisper.
Speech-to-text transcription for recorded calls.
Audio is downloaded from GCS then sent to the Whisper API. Falls back to
returning None on any failure so the intelligence pipeline can still run.
Audio is downloaded from GCS once, then sent to an OpenAI-compatible
/audio/transcriptions endpoint: OpenAI whisper-1 by default, or a cheaper
provider (Groq whisper-large-v3-turbo, a self-hosted server) when configured,
with OpenAI as the fallback when that provider fails (server-26#297). Returns
None on any failure so the intelligence pipeline can still run.
"""
import asyncio
import math
import re
import tempfile
import os
from dataclasses import dataclass
from typing import Optional
from app.internal.logger import logger
from app.internal import firestore as fstore
from app.internal import ai_health
from app.internal import transcript_correction
from app.internal import area_context
from app.internal.chatter_classifier import gate_chatter
from app.config import settings
# Whisper treats `prompt` as preceding transcript text, not instructions.
@@ -43,6 +50,19 @@ _RUN_RATIO = 0.7 # share of consecutive pairs that must step by +1
_MIN_SEGMENTS_FOR_REPEAT = 6 # segments needed before repetition is considered
_UNIQUE_RATIO = 0.25 # unique/total segment texts at or below this is degenerate
_MAX_PHRASE_REPEATS = 8 # identical consecutive phrase repeats allowed in one blob
# Whisper's own failure test (openai/whisper transcribe.py, compression_ratio_
# threshold=2.4): text that gzips this well is a loop, whatever its shape. The
# two checks above only know two loop shapes; this catches the rest (two
# phrases alternating, a list re-emitted) without a model call, so a noise
# gate survives with transcript correction switched off (server-26#83). Only
# on longer text -- a short string has too little to compress to judge.
_MAX_COMPRESSION_RATIO = 2.4
_MIN_CHARS_FOR_COMPRESSION = 200
# Prompt echo (server-26#297): on silence Whisper can emit its own prompt.
# With a vocabulary glossary in the prompt that reads as a list of local
# street names. See _prompt_echo for why this is kept narrow.
_ECHO_MIN_TERMS = 3
_ECHO_WORD_RATIO = 0.6
def _ten_code_run(text: str) -> bool:
@@ -77,7 +97,74 @@ def _phrase_loop(text: str) -> bool:
return False
def _is_degenerate(text: str, segments: list[dict]) -> bool:
def _compression_ratio(text: str) -> float:
import zlib
raw = text.encode("utf-8")
return len(raw) / max(1, len(zlib.compress(raw)))
def _prompt_echo(text: str, glossary: list[str]) -> bool:
"""
True if the transcript is the prompt's own glossary read back.
Deliberately narrow (review of server-26#297): a real location readback
("Main Street, Croton Avenue, Route 9") names glossary terms too, and
that is where map pins come from. So either the lead-in itself came back
("Local names: ..."), or nearly the whole glossary did, in prompt order,
with little else around it.
"""
if not text or not glossary:
return False
low = text.lower()
if "local names" in low:
return True
if len(glossary) < _ECHO_MIN_TERMS:
return False
positions = []
for t in glossary:
m = re.search(rf"\b{re.escape(t.lower())}\b", low)
if m:
positions.append(m.start())
if len(positions) < max(_ECHO_MIN_TERMS, math.ceil(0.8 * len(glossary))):
return False
if positions != sorted(positions):
return False
words = re.findall(r"[\w'-]+", low)
term_words = {w for t in glossary for w in re.findall(r"[\w'-]+", t.lower())}
return bool(words) and sum(1 for w in words if w in term_words) / len(words) >= _ECHO_WORD_RATIO
def _compression_loop(text: str, segments: list[dict]) -> bool:
"""
Whisper applies its 2.4 threshold per 30s decode window, and zlib's ratio
grows with length, so judging a whole 5-minute call at once would drop
busy real traffic. Judge ~30s windows of segments (or ~500-char chunks of
a segment-less transcript) and call it a loop only when at least half of
the windows long enough to judge are loops.
"""
windows: list[str] = []
if segments:
cur, start = [], None
for s in segments:
st = float(s.get("start") or 0.0)
if start is None:
start = st
if st - start > 30.0 and cur:
windows.append(" ".join(cur))
cur, start = [], st
cur.append(s.get("text") or "")
if cur:
windows.append(" ".join(cur))
else:
windows = [text[i:i + 500] for i in range(0, len(text), 500)]
judged = [w for w in windows if len(w) >= _MIN_CHARS_FOR_COMPRESSION]
if not judged:
return False
loops = sum(1 for w in judged if _compression_ratio(w) > _MAX_COMPRESSION_RATIO)
return loops * 2 >= len(judged)
def _is_degenerate(text: str, segments: list[dict], glossary: Optional[list[str]] = None) -> bool:
"""True if a transcript looks like Whisper output rather than radio traffic.
Applied AFTER the per-segment no_speech_prob filter, which does not catch
@@ -88,6 +175,10 @@ def _is_degenerate(text: str, segments: list[dict]) -> bool:
return False
if _ten_code_run(text) or _phrase_loop(text):
return True
if _compression_loop(text, segments):
return True
if _prompt_echo(text, glossary or []):
return True
# Near-identical segments repeated across the whole recording.
if len(segments) >= _MIN_SEGMENTS_FOR_REPEAT:
normalised = {s["text"].strip().lower() for s in segments}
@@ -96,7 +187,155 @@ def _is_degenerate(text: str, segments: list[dict]) -> bool:
return False
async def _log_transcribe_failure(call_id: str, exc: Exception) -> None:
# ── Providers (server-26#297) ───────────────────────────────────────────────
_GROQ_BASE_URL = "https://api.groq.com/openai/v1"
@dataclass(frozen=True)
class SttTarget:
"""One OpenAI-compatible transcription endpoint to try."""
provider: str # "openai" | "groq" | "compat"
model: str
api_key: Optional[str]
base_url: Optional[str] # None -> the OpenAI SDK default
vocab_prompt: bool # whether the domain-vocabulary prompt is sent
tier: str # ai_health tier this target reports to
def stt_targets() -> list[SttTarget]:
"""
The endpoints to try, in order: the configured provider, then OpenAI as
fallback. Empty when nothing usable is configured.
`stt_provider="auto"` resolves to Groq only when GROQ_API_KEY is set, so a
deploy that has not provisioned the key yet keeps transcribing on OpenAI
exactly as before.
"""
provider = (settings.stt_provider or "auto").strip().lower()
if provider == "auto":
if settings.groq_api_key:
provider = "groq"
elif settings.stt_base_url:
provider = "compat" # an explicit endpoint is a compat server
else:
provider = "openai"
openai_target = SttTarget(
provider="openai", model=settings.stt_model, api_key=settings.openai_api_key,
base_url=None, vocab_prompt=False, tier="transcription",
)
if provider not in ("groq", "compat"):
if provider != "openai":
logger.warning(f"Unknown STT_PROVIDER {settings.stt_provider!r} — using OpenAI")
return [openai_target] if settings.openai_api_key else []
if provider == "groq":
key = settings.stt_api_key or settings.groq_api_key
base = settings.stt_base_url or _GROQ_BASE_URL
else:
# A self-hosted server may not check the key, but the SDK insists on one.
key = settings.stt_api_key or "unused"
base = settings.stt_base_url
targets: list[SttTarget] = []
if key and base:
targets.append(SttTarget(
provider=provider, model=settings.stt_compat_model, api_key=key, base_url=base,
vocab_prompt=settings.stt_vocab_prompt_enabled, tier="transcription",
))
else:
logger.warning(f"STT provider {provider!r} has no API key/base URL — using OpenAI only")
if settings.openai_api_key and (not targets or settings.stt_fallback_to_openai):
tier = "transcription_fallback" if targets else "transcription"
targets.append(SttTarget(**{**openai_target.__dict__, "tier": tier}))
return targets
# ── Prompt (server-26#297) ──────────────────────────────────────────────────
_CODE_SHAPED = re.compile(r"^\s*\d{1,3}(?:-\d{1,3})+\s*$")
_MAX_TERM_CHARS = 40
def _estimate_tokens(text: str) -> int:
"""
Conservative token estimate for Whisper's GPT-2 tokenizer, without the
tokenizer: one per punctuation mark and per digit, ~4 letters per token
for words. Over-counts real English slightly, which is the safe side —
Whisper drops prompt tokens past 224 from the FRONT, silently.
"""
n = 0
for piece in re.findall(r"[A-Za-z]+|\d|[^\sA-Za-z\d]", text or ""):
n += max(1, math.ceil(len(piece) / 4)) if piece[0].isalpha() else 1
return n
def _glossary_terms(context: Optional[dict]) -> list[str]:
"""
Local names worth spelling for the recogniser, most specific first:
the talkgroup/system local_knowledge terms, then vocabulary.
Never ten-codes. An enumerated run of codes in the prompt is what made
Whisper count "10-4. 10-5. 10-6." over silence (see _WHISPER_PROMPT), so
the ten_codes reference and any code-shaped vocabulary term stay out.
"""
if not context:
return []
area = context.get("area") or {}
terms = [e.get("term") or "" for e in (area.get("local_knowledge") or []) if isinstance(e, dict)]
terms += [t for t in (context.get("vocabulary") or []) if isinstance(t, str)]
out: list[str] = []
seen: set[str] = set()
for t in terms:
t = t.strip()
key = t.lower()
if not t or key in seen or len(t) > _MAX_TERM_CHARS or _CODE_SHAPED.match(t):
continue
seen.add(key)
out.append(t)
return out
def build_stt_prompt(
talkgroup_name: Optional[str], context: Optional[dict] = None,
) -> tuple[str, list[str]]:
"""
The STT prompt, and the glossary terms that made it in.
Without `context` this is exactly the prompt OpenAI has always been sent.
With it, the channel's area and local names go in front of the fixed
radio-speech priming, cut to settings.stt_prompt_max_tokens. A glossary
is how Whisper's prompting guide says to fix the spelling of proper
nouns; the priming text stays last because it is the part that sets style.
"""
tg_prefix = f"Talkgroup: {talkgroup_name}. " if talkgroup_name else ""
if not context:
return tg_prefix + _WHISPER_PROMPT, []
area = context.get("area") or {}
place = ", ".join(str(area[f]) for f in area_context.PLACE_FIELDS if area.get(f))
head = tg_prefix + (f"{place}. " if place else "")
budget = settings.stt_prompt_max_tokens - _estimate_tokens(head + _WHISPER_PROMPT)
chosen: list[str] = []
for term in _glossary_terms(context):
trial = "Local names: " + ", ".join(chosen + [term]) + ". "
if _estimate_tokens(trial) > budget:
continue # a shorter later term may still fit
chosen.append(term)
glossary = ("Local names: " + ", ".join(chosen) + ". ") if chosen else ""
prompt = head + glossary + _WHISPER_PROMPT
if _estimate_tokens(prompt) > settings.stt_prompt_max_tokens:
return tg_prefix + _WHISPER_PROMPT, []
return prompt, chosen
# ── Failure reporting ───────────────────────────────────────────────────────
async def _log_transcribe_failure(call_id: str, exc: Exception, target: SttTarget) -> None:
"""
Log a transcription failure, escalating a permanent condition to ERROR
once (via app.internal.ai_health, which also drives the /health/ai
@@ -115,47 +354,71 @@ async def _log_transcribe_failure(call_id: str, exc: Exception) -> None:
"""
text = str(exc)
kind = ai_health.classify(text)
who = f"{target.provider} ({target.model})"
billing_url = {
"openai": "https://platform.openai.com/settings/organization/billing",
"groq": "https://console.groq.com/settings/billing",
}.get(target.provider, "the provider's billing page")
if kind == "billing":
problem = "the OpenAI account cannot be billed"
fix = "top up at https://platform.openai.com/settings/organization/billing"
problem = f"the {target.provider} account cannot be billed"
fix = f"top up at {billing_url}"
logger.error(
"Transcription: the OpenAI account cannot be billed -- EVERY call is "
"now stored with no transcript, so extraction, correlation and "
"incidents are all dead downstream. Top up at "
f"https://platform.openai.com/settings/organization/billing. API said: {text}"
f"Transcription: the {target.provider} account cannot be billed -- calls on this "
f"provider get no transcript (OpenAI fallback, if configured, is carrying them). "
f"Top up at {billing_url}. API said: {text}"
)
await ai_health.report_degraded(
"transcription", "openai", settings.stt_model, problem, fix, permanent=True
target.tier, target.provider, target.model, problem, fix, permanent=True
)
return
if kind == "dead_model":
problem = "the STT model is unavailable"
fix = "update STT_MODEL in config.py"
fix = "update STT_MODEL / STT_COMPAT_MODEL"
logger.error(
f"Transcription: the configured model ({settings.stt_model!r}) is unavailable "
"-- EVERY call is now stored with no transcript, so extraction, correlation "
f"and incidents are all dead downstream. Update STT_MODEL in config.py. API said: {text}"
f"Transcription: the configured model {who} is unavailable -- calls on this provider "
f"get no transcript. Update STT_MODEL / STT_COMPAT_MODEL. API said: {text}"
)
await ai_health.report_degraded(
"transcription", "openai", settings.stt_model, problem, fix, permanent=True
target.tier, target.provider, target.model, problem, fix, permanent=True
)
return
logger.warning(f"Transcription failed for call {call_id}: {text}")
logger.warning(f"Transcription via {who} failed for call {call_id}: {text}")
await ai_health.report_degraded(
"transcription", "openai", settings.stt_model,
target.tier, target.provider, target.model,
"transient API error", "no action needed unless this persists", permanent=False,
)
# ── Pipeline entry point ────────────────────────────────────────────────────
async def _transcribe_with(
call_id: str, tmp_path: str, target: SttTarget, prompt: str, glossary: list[str],
) -> dict:
"""One target, with the one retry on degenerate output. Raises on API failure."""
result = await asyncio.to_thread(_sync_transcribe_file, tmp_path, target, prompt, glossary)
# A hallucination is a coin-flip, not a property of the clip: call
# e49ea32c produced a 56-word ten-code counting run on one attempt and
# ordinary speech on the next, same audio and temperature=0. Discarding
# on the first bad roll threw away a recoverable transcript, so spend
# one more request before giving up.
if result["degenerate"] and settings.stt_retry_on_degenerate:
logger.info(f"Retrying transcription for call {call_id} after degenerate output")
result = await asyncio.to_thread(_sync_transcribe_file, tmp_path, target, prompt, glossary)
if result["degenerate"]:
logger.warning(f"Transcription for call {call_id} was degenerate twice — giving up")
return result
async def transcribe_call(
call_id: str,
gcs_uri: str,
talkgroup_name: Optional[str] = None,
system_id: Optional[str] = None,
talkgroup_id: Optional[int] = None,
node_id: Optional[str] = None,
) -> tuple[Optional[str], list[dict]]:
"""
Transcribe audio at the given GCS URI and store the result in Firestore.
@@ -167,36 +430,78 @@ async def transcribe_call(
if not gcs_uri or not gcs_uri.startswith("gs://"):
return None, []
targets = stt_targets()
if not targets:
logger.warning("No STT provider configured (OPENAI_API_KEY / GROQ_API_KEY unset) — transcription disabled.")
return None, []
context: Optional[dict] = None
if any(t.vocab_prompt for t in targets):
try:
context = await transcript_correction.resolve_context(system_id, talkgroup_id)
except Exception as e:
logger.warning(f"Could not resolve STT vocabulary for call {call_id}: {e}")
try:
transcript, segments, degenerate = await asyncio.to_thread(
_sync_transcribe, gcs_uri, talkgroup_name
)
# A hallucination is a coin-flip, not a property of the clip: call
# e49ea32c produced a 56-word ten-code counting run on one attempt and
# ordinary speech on the next, same audio and temperature=0. Discarding
# on the first bad roll threw away a recoverable transcript, so spend
# one more request before giving up.
if degenerate and settings.stt_retry_on_degenerate:
logger.info(f"Retrying transcription for call {call_id} after degenerate output")
transcript, segments, degenerate = await asyncio.to_thread(
_sync_transcribe, gcs_uri, talkgroup_name
)
if degenerate:
logger.warning(
f"Transcription for call {call_id} was degenerate twice — giving up"
)
tmp_path = await asyncio.to_thread(_download, gcs_uri)
except Exception as e:
await _log_transcribe_failure(call_id, e)
# Not a provider failure: classify() would read a GCS "not found" as a
# retired model and page someone to change STT_MODEL.
logger.warning(f"Could not download audio for call {call_id} ({gcs_uri}): {e}")
return None, []
result: Optional[dict] = None
used: Optional[SttTarget] = None
try:
for i, target in enumerate(targets):
prompt, glossary = build_stt_prompt(talkgroup_name, context if target.vocab_prompt else None)
try:
result = await _transcribe_with(call_id, tmp_path, target, prompt, glossary)
except Exception as e:
await _log_transcribe_failure(call_id, e, target)
if i + 1 < len(targets):
logger.warning(
f"Transcription for call {call_id}: {target.provider} failed, "
f"falling back to {targets[i + 1].provider} ({targets[i + 1].model})"
)
continue
used = target
break
finally:
try:
os.unlink(tmp_path)
except OSError:
pass
if used is None or result is None:
return None, []
# No exception means the provider call itself succeeded (this also
# covers transcripts discarded as degenerate/hallucinated output —
# that's a filtering decision, not a provider failure), so the
# transcription tier is healthy and any prior degradation clears.
await ai_health.report_healthy("transcription")
# that's a filtering decision, not a provider failure), so that tier is
# healthy and any prior degradation clears. A fallback success clears
# only the fallback tier: the primary is still broken and must stay red.
await ai_health.report_healthy(used.tier)
transcript: Optional[str] = result["text"]
segments: list[dict] = result["segments"]
if transcript:
updates: dict = {"transcript": transcript}
updates: dict = {
"transcript": transcript,
# Which endpoint produced this text, and how much audio it billed,
# so cost (server-26#45) and an A/B of providers can be read off
# the call docs instead of guessed.
"stt_provider": used.provider,
"stt_model": used.model,
}
if result.get("duration") is not None:
updates["stt_audio_seconds"] = result["duration"]
if result.get("confidence") is not None:
# Mean per-token probability, exp(avg_logprob) weighted by segment
# length. The only STT-side confidence there is; the alert gate
# reads it (server-26#300).
updates["stt_confidence"] = result["confidence"]
if segments:
updates["segments"] = segments
@@ -212,17 +517,30 @@ async def transcribe_call(
from app.internal.feature_flags import resolve_flags
_, _ai_flag = await resolve_flags(system_id)
# server-26#298: radio housekeeping (roll call, a bare "Post 4, 10-8,
# copy") has no proper noun to correct, and extraction skips it too
# (intelligence.extract_scenes), so a correction pass here is pure
# spend. Same pure classifier both places, so they cannot disagree.
is_chatter, chatter_reason = (
gate_chatter(transcript) if settings.chatter_gate_enabled else (False, None)
)
corrected, corrected_segments, not_speech = (None, None, False)
if _ai_flag("transcript_correction_enabled"):
if not _ai_flag("transcript_correction_enabled"):
logger.info(
f"Transcript correction disabled — saving raw transcript for call {call_id}"
)
elif is_chatter:
logger.info(
f"Transcript correction skipped for call {call_id}: chatter ({chatter_reason})"
)
else:
corrected, corrected_segments, not_speech = await transcript_correction.correct(
call_id, transcript, segments,
system_id=system_id,
talkgroup_id=talkgroup_id,
talkgroup_name=talkgroup_name,
)
else:
logger.info(
f"Transcript correction disabled — saving raw transcript for call {call_id}"
node_id=node_id,
)
if corrected_segments:
# Raw stays as evidence; the corrected copy is what extraction reads.
@@ -241,7 +559,7 @@ async def transcribe_call(
try:
await fstore.doc_set("calls", call_id, updates)
logger.info(
f"Transcript saved for call {call_id} "
f"Transcript saved for call {call_id} via {used.provider}/{used.model} "
f"({len(transcript)} chars, {len(segments)} segment(s)"
f"{', corrected' if corrected and not not_speech else ''})"
)
@@ -259,27 +577,10 @@ async def transcribe_call(
return transcript, segments
def _sync_transcribe(
gcs_uri: str,
talkgroup_name: Optional[str] = None,
) -> tuple[Optional[str], list[dict], bool]:
"""Download audio from GCS and transcribe with OpenAI Whisper.
Third element is True when output was DISCARDED as degenerate, which the
caller distinguishes from ordinary silence so it can retry — the same clip
can hallucinate on one attempt and transcribe on the next.
"""
def _download(gcs_uri: str) -> str:
"""Download the recording to a temp file once; the caller deletes it."""
from google.cloud import storage as gcs
from google.oauth2 import service_account
from app.config import settings
from app.internal.ai_clients import openai_client
if not settings.openai_api_key:
logger.warning("OPENAI_API_KEY not set — transcription disabled.")
# Tuple, not a bare None: the caller unpacks two values, so returning
# None here raised a TypeError that surfaced as a misleading
# "Transcription failed" instead of the real missing-key warning.
return None, [], False
without_scheme = gcs_uri[len("gs://"):]
bucket_name, blob_path = without_scheme.split("/", 1)
@@ -293,77 +594,116 @@ def _sync_transcribe(
else:
gcs_client = gcs.Client()
bucket = gcs_client.bucket(bucket_name)
blob = bucket.blob(blob_path)
blob = gcs_client.bucket(bucket_name).blob(blob_path)
suffix = os.path.splitext(blob_path)[1] or ".mp3"
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
tmp_path = tmp.name
try:
blob.download_to_filename(tmp_path)
tg_prefix = f"Talkgroup: {talkgroup_name}. " if talkgroup_name else ""
# Vocabulary is intentionally excluded from the Whisper prompt.
# whisper-1 treats the prompt as a transcription prior and echoes
# vocabulary terms into noise/silence, polluting downstream extraction.
# Vocabulary context is applied in the GPT extraction step instead,
# where it is used as reference rather than a transcription prior.
prompt = tg_prefix + _WHISPER_PROMPT
# Only whisper-1 supports verbose_json (per-segment timestamps + no_speech_prob).
# gpt-4o-transcribe and gpt-4o-mini-transcribe only support json/text.
use_verbose = settings.stt_model == "whisper-1"
# Own timeout, not openai_timeout_seconds: this request uploads the
# whole recording (server-26#153).
stt_client = openai_client(timeout=settings.stt_timeout_seconds)
with open(tmp_path, "rb") as f:
response = stt_client.audio.transcriptions.create(
model=settings.stt_model,
file=f,
language="en",
prompt=prompt,
response_format="verbose_json" if use_verbose else "json",
temperature=0,
)
if use_verbose:
# Filter hallucinated segments. Two sources of hallucination in P25 recordings:
#
# 1. Trailing silence / static — Whisper fills silence past real content with
# sequential radio codes (10-4, 10-5...). Clamped by audio duration.
#
# 2. Leading silence — OP25 recordings typically have a short silence at the
# start before the first PTT press. Whisper sometimes hallucinates filler
# words or codes over this silence. Detected via no_speech_prob > 0.8
# (Whisper's own confidence that a segment contains no real speech).
audio_duration: float = getattr(response, "duration", None) or float("inf")
segments = [
{"start": round(s.start, 2), "end": round(s.end, 2), "text": s.text.strip()}
for s in (response.segments or [])
if s.text.strip()
and s.start < audio_duration
and getattr(s, "no_speech_prob", 0.0) < 0.8
]
# Reconstruct text from non-hallucinated segments only so the two stay
# in sync. If every segment was filtered, text becomes None which prevents
# the intelligence pipeline from running on hallucinated content.
text = " ".join(s["text"] for s in segments) or None
if _is_degenerate(text or "", segments):
logger.info(f"Discarded hallucinated transcript for {gcs_uri}: {(text or '')[:80]!r}")
return None, [], True
return text, segments, False
else:
# json format returns just {"text": "..."} — no segments or timestamps.
# Intelligence extraction falls back to treating the whole transcript as one block.
text = (response.text or "").strip() or None
if _is_degenerate(text or "", []):
logger.info(f"Discarded hallucinated transcript for {gcs_uri}: {(text or '')[:80]!r}")
return None, [], True
return text, [], False
finally:
except Exception:
try:
os.unlink(tmp_path)
except OSError:
pass
raise
return tmp_path
def _segment_confidence(raw_segments: list) -> Optional[float]:
"""exp(avg_logprob) per segment, weighted by segment length; None if absent."""
total, weight = 0.0, 0.0
for s in raw_segments:
lp = getattr(s, "avg_logprob", None)
if lp is None:
continue
w = max(0.01, float(getattr(s, "end", 0) or 0) - float(getattr(s, "start", 0) or 0))
total += math.exp(float(lp)) * w
weight += w
return round(total / weight, 3) if weight else None
def _sync_transcribe_file(
tmp_path: str, target: SttTarget, prompt: str, glossary: Optional[list[str]] = None,
) -> dict:
"""Transcribe one downloaded file on one target.
Returns {"text", "segments", "degenerate", "duration", "confidence"}.
`degenerate` is True when output was DISCARDED as degenerate, which the
caller distinguishes from ordinary silence so it can retry — the same clip
can hallucinate on one attempt and transcribe on the next. Raises on any
API failure, so the caller can fall back to the next target.
"""
from app.internal.ai_clients import stt_client
# verbose_json (per-segment timestamps + no_speech_prob) is a Whisper
# feature: whisper-1 on OpenAI, every whisper-* model on Groq. OpenAI's
# gpt-4o-transcribe / gpt-4o-mini-transcribe only support json/text.
use_verbose = target.model.startswith("whisper")
client = stt_client(target.api_key, target.base_url)
with open(tmp_path, "rb") as f:
response = client.audio.transcriptions.create(
model=target.model,
file=f,
language="en",
prompt=prompt,
response_format="verbose_json" if use_verbose else "json",
temperature=0,
)
if use_verbose:
# Filter hallucinated segments. Two sources of hallucination in P25 recordings:
#
# 1. Trailing silence / static — Whisper fills silence past real content with
# sequential radio codes (10-4, 10-5...). Clamped by audio duration.
#
# 2. Leading silence — OP25 recordings typically have a short silence at the
# start before the first PTT press. Whisper sometimes hallucinates filler
# words or codes over this silence. Detected via no_speech_prob > 0.8
# (Whisper's own confidence that a segment contains no real speech).
duration = getattr(response, "duration", None)
audio_duration: float = duration or float("inf")
kept = [
s for s in (response.segments or [])
if (_seg(s, "text") or "").strip()
and (_seg(s, "start") or 0.0) < audio_duration
and (_seg(s, "no_speech_prob") or 0.0) < 0.8
]
segments = [
{"start": round(_seg(s, "start") or 0.0, 2), "end": round(_seg(s, "end") or 0.0, 2),
"text": _seg(s, "text").strip()}
for s in kept
]
# Reconstruct text from non-hallucinated segments only so the two stay
# in sync. If every segment was filtered, text becomes None which prevents
# the intelligence pipeline from running on hallucinated content.
text = " ".join(s["text"] for s in segments) or None
base = {"duration": duration, "confidence": _segment_confidence([_SegView(s) for s in kept])}
if _is_degenerate(text or "", segments, glossary):
logger.info(f"Discarded hallucinated transcript from {target.provider}: {(text or '')[:80]!r}")
return {"text": None, "segments": [], "degenerate": True, **base}
return {"text": text, "segments": segments, "degenerate": False, **base}
# json format returns just {"text": "..."} — no segments or timestamps.
# Intelligence extraction falls back to treating the whole transcript as one block.
text = (response.text or "").strip() or None
if _is_degenerate(text or "", [], glossary):
logger.info(f"Discarded hallucinated transcript from {target.provider}: {(text or '')[:80]!r}")
return {"text": None, "segments": [], "degenerate": True, "duration": None, "confidence": None}
return {"text": text, "segments": [], "degenerate": False, "duration": None, "confidence": None}
def _seg(s, field: str):
"""Segment field from an SDK object or a plain dict (OpenAI-compatible
servers do not all come back through the same pydantic model)."""
if isinstance(s, dict):
return s.get(field)
return getattr(s, field, None)
class _SegView:
"""Attribute view over a segment, for _segment_confidence."""
def __init__(self, s):
self.avg_logprob = _seg(s, "avg_logprob")
self.start = _seg(s, "start")
self.end = _seg(s, "end")