Files
server-26/drb-c2-core/app/routers/systems.py
T
Logan CusanoandClaude Opus 5 58efdbd6eb
Build & Deploy / Build & push images (push) Successful in 4m0s
Build & Deploy / Deploy to VM (push) Successful in 2m28s
Build & Deploy / Report a failed deploy (push) Skipped
Correct the transcript before anything reads it
Correction existed, but as a line in intelligence.py's EXTRACTION_PROMPT --
which put it in the wrong place twice over. The same model call that extracted
units, location and severity emitted the correction afterwards, so extraction
reasoned over text already known to be wrong; and it sat behind
correlation_enabled, so during a cost-controlled STT-only window nothing was
ever corrected at all. That is the normal state during development.

internal/transcript_correction.py is now its own pass, between the degenerate
filter and the Firestore write. It receives an already-produced transcript plus
a reference list, so unlike a Whisper prompt it has no series to extend -- the
distinction that keeps vocabulary out of the recogniser's prompt, where an
enumerated ten-code list once made it hallucinate ten-code runs.

Reference data is merged from the talkgroup and the system, TALKGROUP FIRST. A
system spanning several counties can have a talkgroup covering one
municipality, and that municipality's streets must not be buried under a
county-wide list. A single-municipality system is the degenerate case: populate
the system level and every talkgroup inherits it. Area context is now SET --
municipality, county, roads, landmarks, on both scopes -- rather than guessed
from talkgroup names, which is what vocabulary_learner did and which is close
to useless across multiple counties.

Segments are corrected too, not just the joined text. extract_scenes builds its
prompt from numbered segments whenever there is more than one, so a correction
that only fixed the transcript would have been discarded on exactly the
multi-transmission calls carrying the most content. Alignment is enforced: an
array of the wrong length or type is dropped whole, because scenes map back to
transmissions by index and a shifted array would misattribute audio silently.

Whisper is also retried once on degenerate output. Call e49ea32c produced a
56-word ten-code counting run on one attempt and ordinary speech on the next --
same clip, same temperature=0 -- so a hallucination is a coin-flip, and
discarding on the first bad roll threw away a recoverable transcript.

Two things found on the way:

PUT /systems/{id} wiped ten_codes on every save. The systems form sends only
{name, type, config}, and model_dump() wrote every omitted field as its default
over the top. Now exclude_unset. area_context would have been the next victim,
which is why it gets its own route alongside ten-codes rather than a field on
that payload.

Closes server-26#36.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-23 14:23:41 -04:00

267 lines
10 KiB
Python

import uuid
from fastapi import APIRouter, HTTPException, Depends, Query
from pydantic import BaseModel
from typing import Dict, List, Optional
from app.models import SystemCreate, SystemRecord
from app.internal import firestore as fstore
from app.internal.auth import (
require_admin_token,
require_node_service_or_firebase_token,
resolve_caller_org_id,
bootstrap_limiter,
)
from app.internal.tenancy import FOUNDING_ORG_ID
router = APIRouter(prefix="/systems", tags=["systems"])
class VocabularyTermBody(BaseModel):
term: str
class TenCodesBody(BaseModel):
ten_codes: Dict[str, str]
class AreaContextBody(BaseModel):
"""Ground truth about the area a system covers — see PUT /{id}/area-context."""
municipality: Optional[str] = None
county: Optional[str] = None
roads: List[str] = []
landmarks: List[str] = []
class AiFlagsBody(BaseModel):
stt_enabled: Optional[bool] = None
correlation_enabled: Optional[bool] = None
@router.get("")
async def list_systems(decoded: dict = Depends(require_node_service_or_firebase_token)):
org_id = await resolve_caller_org_id(decoded)
if org_id is None: # service key or platform admin — unrestricted, matches prior behaviour
return await fstore.collection_list("systems")
return await fstore.collection_list("systems", org_id=org_id)
@router.get("/{system_id}")
async def get_system(system_id: str, decoded: dict = Depends(require_node_service_or_firebase_token)):
system = await fstore.doc_get("systems", system_id)
if not system:
raise HTTPException(404, f"System '{system_id}' not found.")
org_id = await resolve_caller_org_id(decoded)
if org_id is not None and system.get("org_id") != org_id:
raise HTTPException(404, f"System '{system_id}' not found.")
return system
@router.post("", status_code=201)
async def create_system(
body: SystemCreate,
org_id: Optional[str] = Query(None, description="Platform-admin only — defaults to the founding org."),
_: dict = Depends(require_admin_token),
):
system_id = str(uuid.uuid4())
doc = SystemRecord(system_id=system_id, org_id=org_id or FOUNDING_ORG_ID, **body.model_dump())
await fstore.doc_set("systems", system_id, doc.model_dump(), merge=False)
return doc
@router.put("/{system_id}")
async def update_system(system_id: str, body: SystemCreate, _: dict = Depends(require_admin_token)):
existing = await fstore.doc_get("systems", system_id)
if not existing:
raise HTTPException(404, f"System '{system_id}' not found.")
# exclude_unset, or every field the caller omitted gets written as its
# default and silently erases what was there. The systems page PUTs only
# {name, type, config}, so a plain model_dump() wiped ten_codes on every
# save — they are edited through PUT /{id}/ten-codes and were never in this
# payload. area_context (server-26#36) would have been the second casualty.
patch = body.model_dump(exclude_unset=True)
await fstore.doc_update("systems", system_id, patch)
return {**existing, **patch}
@router.delete("/{system_id}", status_code=204)
async def delete_system(system_id: str, _: dict = Depends(require_admin_token)):
existing = await fstore.doc_get("systems", system_id)
if not existing:
raise HTTPException(404, f"System '{system_id}' not found.")
await fstore.doc_delete("systems", system_id)
# ── Per-system AI flag overrides ──────────────────────────────────────────────
@router.put("/{system_id}/ai-flags")
async def update_system_ai_flags(
system_id: str,
body: AiFlagsBody,
_: dict = Depends(require_admin_token),
):
"""
Set per-system AI flag overrides. Only fields included in the body are
written; omitted fields remain unchanged (or absent, meaning inherit global).
Pass null to clear an override and fall back to the global flag.
"""
existing = await fstore.doc_get("systems", system_id)
if not existing:
raise HTTPException(404, f"System '{system_id}' not found.")
current: dict = existing.get("ai_flags") or {}
for field, value in body.model_dump(exclude_unset=True).items():
if value is None:
current.pop(field, None) # clear override → inherit global
else:
current[field] = value
await fstore.doc_update("systems", system_id, {"ai_flags": current})
return {"ok": True, "ai_flags": current}
# ── Ten-codes endpoints ────────────────────────────────────────────────────────
@router.get("/{system_id}/ten-codes")
async def get_ten_codes(system_id: str):
"""Return the ten-code dictionary for a system."""
system = await fstore.doc_get("systems", system_id)
if not system:
raise HTTPException(404, f"System '{system_id}' not found.")
return {"ten_codes": system.get("ten_codes") or {}}
@router.put("/{system_id}/ten-codes")
async def update_ten_codes(
system_id: str,
body: TenCodesBody,
_: dict = Depends(require_admin_token),
):
"""Replace the ten-code dictionary for a system."""
existing = await fstore.doc_get("systems", system_id)
if not existing:
raise HTTPException(404, f"System '{system_id}' not found.")
await fstore.doc_update("systems", system_id, {"ten_codes": body.ten_codes})
return {"ok": True, "ten_codes": body.ten_codes}
# ── Area context ──────────────────────────────────────────────────────────────
@router.get("/{system_id}/area-context")
async def get_area_context(system_id: str, _: dict = Depends(require_admin_token)):
system = await fstore.doc_get("systems", system_id)
if not system:
raise HTTPException(404, f"System '{system_id}' not found.")
return {"area_context": system.get("area_context") or {}}
@router.put("/{system_id}/area-context")
async def update_area_context(
system_id: str,
body: AreaContextBody,
_: dict = Depends(require_admin_token),
):
"""
Replace the system-wide area context used by the transcript corrector.
Ground truth about where this system operates — municipality, county, the
roads and landmarks whose names Whisper mangles. Per-talkgroup overrides
live inside config.talkgroups[] and rank ABOVE this (server-26#36), so a
multi-county system narrows per channel rather than replacing this wholesale.
Its own route rather than a field on PUT /systems/{id} for the same reason
ten-codes has one: the systems form does not carry it, and folding it into
that payload is how ten_codes kept getting wiped.
"""
existing = await fstore.doc_get("systems", system_id)
if not existing:
raise HTTPException(404, f"System '{system_id}' not found.")
area = body.model_dump(exclude_none=True)
await fstore.doc_update("systems", system_id, {"area_context": area})
return {"ok": True, "area_context": area}
# ── Vocabulary endpoints ───────────────────────────────────────────────────────
@router.get("/{system_id}/vocabulary")
async def get_vocabulary(system_id: str):
"""Return approved vocabulary and pending induction suggestions."""
existing = await fstore.doc_get("systems", system_id)
if not existing:
raise HTTPException(404, f"System '{system_id}' not found.")
from app.internal.vocabulary_learner import get_vocabulary as _get
return await _get(system_id)
@router.post("/{system_id}/vocabulary/bootstrap", status_code=202)
async def bootstrap_vocabulary(
system_id: str,
decoded: dict = Depends(require_admin_token),
):
"""Trigger a one-shot GPT-4o bootstrap to seed the vocabulary from local knowledge."""
existing = await fstore.doc_get("systems", system_id)
if not existing:
raise HTTPException(404, f"System '{system_id}' not found.")
bootstrap_limiter.check(system_id)
from app.internal.vocabulary_learner import bootstrap_system_vocabulary
terms = await bootstrap_system_vocabulary(system_id)
return {"added": len(terms), "terms": terms}
@router.post("/{system_id}/vocabulary/terms")
async def add_vocabulary_term(
system_id: str,
body: VocabularyTermBody,
_: dict = Depends(require_admin_token),
):
"""Manually add a term to the approved vocabulary."""
existing = await fstore.doc_get("systems", system_id)
if not existing:
raise HTTPException(404, f"System '{system_id}' not found.")
from app.internal.vocabulary_learner import add_term
await add_term(system_id, body.term.strip())
return {"ok": True}
@router.delete("/{system_id}/vocabulary/terms")
async def remove_vocabulary_term(
system_id: str,
body: VocabularyTermBody,
_: dict = Depends(require_admin_token),
):
"""Remove a term from the approved vocabulary."""
existing = await fstore.doc_get("systems", system_id)
if not existing:
raise HTTPException(404, f"System '{system_id}' not found.")
from app.internal.vocabulary_learner import remove_term
await remove_term(system_id, body.term)
return {"ok": True}
@router.post("/{system_id}/vocabulary/pending/approve")
async def approve_pending(
system_id: str,
body: VocabularyTermBody,
_: dict = Depends(require_admin_token),
):
"""Move a pending induction suggestion into the approved vocabulary."""
existing = await fstore.doc_get("systems", system_id)
if not existing:
raise HTTPException(404, f"System '{system_id}' not found.")
from app.internal.vocabulary_learner import approve_pending_term
await approve_pending_term(system_id, body.term)
return {"ok": True}
@router.post("/{system_id}/vocabulary/pending/dismiss")
async def dismiss_pending(
system_id: str,
body: VocabularyTermBody,
_: dict = Depends(require_admin_token),
):
"""Dismiss a pending induction suggestion without adding it."""
existing = await fstore.doc_get("systems", system_id)
if not existing:
raise HTTPException(404, f"System '{system_id}' not found.")
from app.internal.vocabulary_learner import dismiss_pending_term
await dismiss_pending_term(system_id, body.term)
return {"ok": True}