Logan Cusano
97013e1505
Two independent sources of garbage in the AI pipeline, both visible in the 2026-08-16 correlation dump. 1. Hallucinated transcripts. The Whisper prompt opened with an enumerated run of ten-codes: 10-4, 10-23, 10-20, 10-97 and so on. Whisper treats prompt text as preceding transcript, so on noisy or silent audio it continued the series, emitting transcripts that count upward from 10-4 to 10-99. The existing no_speech_prob filter could not catch these: the model is highly confident in text it invented by continuing a pattern. The prompt no longer contains a series to extend, and _is_degenerate() rejects the three shapes this failure takes: ascending ten-code runs, one phrase looping, and near-identical segments across a whole recording. Verified against 13 transcripts from production: all four known hallucinations rejected, all nine real ones kept, including terse traffic containing legitimate codes. 2. Duplicate recordings. node-002 and node-PI-2 both cover TG 9048 and both uploaded the same transmissions, ~1.1s apart. Nine pairs appeared in one dump. Each was transcribed, billed and correlated twice, and the resulting incident listed two units where there was one. Canonical selection is by earliest started_at, tie-broken on call_id, NOT by upload order: upload order varies with encode time and network latency, so it would make the authoritative recording non-deterministic. Call documents are created from MQTT call_start before uploads arrive, so both nodes independently reach the same verdict. The loser keeps its audio (it may be the cleaner capture) but is excluded from STT, correlation, the re-correlation sweep and the orphan debug view. Also fixes _sync_transcribe returning a bare None when OPENAI_API_KEY is missing, where the caller unpacks two values. A missing key surfaced as a misleading "Transcription failed" instead of the real warning. Adds tests/test_dedup.py (15 cases). dedup.py reaches Firestore through an injected callable so it stays importable without firebase-admin present.
189 lines
6.1 KiB
Python
189 lines
6.1 KiB
Python
from pydantic import BaseModel, Field
|
|
from typing import Optional, List, Dict, Any
|
|
from datetime import datetime
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Nodes
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class NodeRecord(BaseModel):
|
|
node_id: str
|
|
name: str
|
|
lat: float = 0.0
|
|
lon: float = 0.0
|
|
status: str = "offline" # online / offline / recording / unconfigured
|
|
configured: bool = False
|
|
last_seen: Optional[datetime] = None
|
|
assigned_system_id: Optional[str] = None
|
|
node_type: str = "fixed" # fixed or portable
|
|
enforce_override_timeout: bool = True
|
|
is_overridden: bool = False
|
|
override_system_id: Optional[str] = None
|
|
override_timeout_at: Optional[datetime] = None
|
|
|
|
|
|
class CommandPayload(BaseModel):
|
|
action: str # discord_join / discord_leave / op25_restart
|
|
guild_id: Optional[str] = None
|
|
channel_id: Optional[str] = None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Systems
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class SystemRecord(BaseModel):
|
|
system_id: str
|
|
name: str
|
|
type: str # P25 / DMR / NBFM
|
|
config: Dict[str, Any] = {} # OP25-compatible config blob
|
|
ten_codes: Dict[str, str] = {} # {"10-10": "Commercial Alarm", ...}
|
|
|
|
|
|
class SystemCreate(BaseModel):
|
|
name: str
|
|
type: str
|
|
config: Dict[str, Any] = {}
|
|
ten_codes: Dict[str, str] = {}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Calls
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class CallRecord(BaseModel):
|
|
call_id: str
|
|
node_id: str
|
|
system_id: Optional[str] = None
|
|
talkgroup_id: Optional[int] = None
|
|
talkgroup_name: Optional[str] = None
|
|
freq: Optional[float] = None
|
|
srcaddr: Optional[str] = None
|
|
started_at: datetime
|
|
ended_at: Optional[datetime] = None
|
|
audio_gcs_uri: Optional[str] = None # canonical gs:// object location
|
|
audio_url: Optional[str] = None # NOT stored — minted per read, see internal/storage.py
|
|
duplicate_of: Optional[str] = None # another node recorded this same transmission first
|
|
transcript: Optional[str] = None # populated later by STT
|
|
incident_ids: List[str] = [] # one per scene detected in the recording
|
|
location: Optional[Dict[str, float]] = None # {lat, lng}
|
|
tags: List[str] = []
|
|
status: str = "active" # active / ended
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Incidents
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class IncidentRecord(BaseModel):
|
|
incident_id: str
|
|
title: Optional[str] = None
|
|
type: Optional[str] = None # fire / police / ems / etc.
|
|
status: str = "active" # active / resolved
|
|
location: Optional[Dict[str, float]] = None
|
|
call_ids: List[str] = []
|
|
started_at: datetime
|
|
updated_at: datetime
|
|
summary: Optional[str] = None
|
|
tags: List[str] = []
|
|
|
|
|
|
class IncidentCreate(BaseModel):
|
|
title: str
|
|
type: str = "other"
|
|
status: str = "active"
|
|
location: Optional[Dict[str, float]] = None
|
|
call_ids: List[str] = []
|
|
summary: Optional[str] = None
|
|
tags: List[str] = []
|
|
|
|
|
|
class IncidentUpdate(BaseModel):
|
|
title: Optional[str] = None
|
|
type: Optional[str] = None
|
|
status: Optional[str] = None
|
|
location: Optional[Dict[str, float]] = None
|
|
summary: Optional[str] = None
|
|
tags: Optional[List[str]] = None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Alerts
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class AlertRule(BaseModel):
|
|
rule_id: Optional[str] = None
|
|
name: str
|
|
keywords: List[str] = []
|
|
talkgroup_ids: List[int] = []
|
|
enabled: bool = True
|
|
discord_webhook: Optional[str] = None # POST here when rule fires
|
|
|
|
|
|
class AlertRuleUpdate(BaseModel):
|
|
name: Optional[str] = None
|
|
keywords: Optional[List[str]] = None
|
|
talkgroup_ids: Optional[List[int]] = None
|
|
enabled: Optional[bool] = None
|
|
discord_webhook: Optional[str] = None
|
|
|
|
|
|
class AlertEvent(BaseModel):
|
|
alert_id: Optional[str] = None
|
|
rule_id: str
|
|
rule_name: str
|
|
call_id: str
|
|
node_id: str
|
|
talkgroup_id: Optional[int] = None
|
|
talkgroup_name: Optional[str] = None
|
|
matched_keywords: List[str] = []
|
|
transcript_snippet: Optional[str] = None
|
|
triggered_at: Optional[datetime] = None
|
|
acknowledged: bool = False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Trips
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TripCreate(BaseModel):
|
|
name: str
|
|
location: str
|
|
maps_link: Optional[str] = None
|
|
start_date: str # YYYY-MM-DD
|
|
end_date: str # YYYY-MM-DD
|
|
available_tags: List[str] = [] # tag labels configured for this trip
|
|
overlap_tags: List[str] = [] # subset of available_tags that allow time overlap
|
|
visibility: str = "public" # "public" | "private"
|
|
invited_discord_ids: List[str] = [] # discord user IDs allowed on private trips
|
|
|
|
|
|
class TripEventCreate(BaseModel):
|
|
title: str
|
|
date: str # YYYY-MM-DD, must fall within parent trip range
|
|
start_time: Optional[str] = None # HH:MM (24h)
|
|
end_time: Optional[str] = None # HH:MM (24h)
|
|
location: Optional[str] = None # inherits trip location if None
|
|
maps_link: Optional[str] = None
|
|
place_id: Optional[str] = None # Google Place ID
|
|
notes: Optional[str] = None
|
|
tags: List[str] = [] # tag labels applied to this event
|
|
|
|
|
|
class TripEventUpdate(BaseModel):
|
|
title: Optional[str] = None
|
|
date: Optional[str] = None
|
|
start_time: Optional[str] = None
|
|
end_time: Optional[str] = None
|
|
location: Optional[str] = None
|
|
maps_link: Optional[str] = None
|
|
place_id: Optional[str] = None
|
|
notes: Optional[str] = None
|
|
tags: Optional[List[str]] = None
|
|
|
|
|
|
class AttendeeAction(BaseModel):
|
|
discord_user_id: str
|
|
discord_username: Optional[str] = None
|