Compare commits
69
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e79b8bc37d | ||
|
|
c72c28f5dc | ||
|
|
02b5b7b5a5 | ||
|
|
40014a47a3 | ||
|
|
6c0e7a4f8e | ||
|
|
6479174022 | ||
|
|
c043298902 | ||
|
|
fa194e0f0a | ||
|
|
5f85a878fa | ||
|
|
241a15b8da | ||
|
|
f91d4559f3 | ||
|
|
66bbf5b473 | ||
|
|
6c095083fc | ||
|
|
2e67d1bad6 | ||
|
|
1ffff25cd2 | ||
|
|
3d2b722c64 | ||
|
|
5537b095df | ||
|
|
8892e824fc | ||
|
|
3f69879437 | ||
|
|
fb0bb15c22 | ||
|
|
a9197709f8 | ||
|
|
8dd636af8f | ||
|
|
e27f8f6636 | ||
|
|
f23026b9ab | ||
|
|
7717fcccdd | ||
|
|
454fe7e81c | ||
|
|
422e9a4dc8 | ||
|
|
b1884852d5 | ||
|
|
0473e6a583 | ||
|
|
4df801c5e0 | ||
|
|
c50bfda8db | ||
|
|
833cfade4e | ||
|
|
0fe6d3b567 | ||
|
|
fae84a45c3 | ||
|
|
b7701b6d49 | ||
|
|
3ae0bb2d5b | ||
|
|
76db41adf7 | ||
|
|
e97dab22ce | ||
|
|
05ddec8284 | ||
|
|
11c98daed0 | ||
|
|
83beb2bf35 | ||
|
|
598054746a | ||
|
|
400b74b519 | ||
|
|
07ff9ba193 | ||
|
|
15a9d10666 | ||
|
|
dd426572fc | ||
|
|
ca1d8fbdae | ||
|
|
7f4d684966 | ||
|
|
bd04bdbd69 | ||
|
|
775244bbde | ||
|
|
bc3251e8df | ||
|
|
7a5bd5dbbb | ||
|
|
629bd1c340 | ||
|
|
cea094d66b | ||
|
|
01c146e21e | ||
|
|
8a0412b529 | ||
|
|
52edbf105c | ||
|
|
77f1d2f93f | ||
|
|
d60fef67ad | ||
|
|
fe643924c7 | ||
|
|
bccb3e0316 | ||
|
|
1a631d65d0 | ||
|
|
3a944f35c1 | ||
|
|
0712e7a437 | ||
|
|
a739fa64f0 | ||
|
|
7189ba03e4 | ||
|
|
d67b2057e6 | ||
|
|
ef1e3d7f9d | ||
|
|
c1c3e89e1d |
@@ -63,6 +63,7 @@ jobs:
|
|||||||
NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID=${{ secrets.FIREBASE_MESSAGING_SENDER_ID }}
|
NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID=${{ secrets.FIREBASE_MESSAGING_SENDER_ID }}
|
||||||
NEXT_PUBLIC_FIREBASE_APP_ID=${{ secrets.FIREBASE_APP_ID }}
|
NEXT_PUBLIC_FIREBASE_APP_ID=${{ secrets.FIREBASE_APP_ID }}
|
||||||
NEXT_PUBLIC_FIRESTORE_DATABASE=${{ secrets.FIRESTORE_DATABASE }}
|
NEXT_PUBLIC_FIRESTORE_DATABASE=${{ secrets.FIRESTORE_DATABASE }}
|
||||||
|
NEXT_PUBLIC_MAP_TILE_URL=https://tile.openstreetmap.org/{z}/{x}/{y}.png
|
||||||
|
|
||||||
deploy:
|
deploy:
|
||||||
name: Deploy to VM
|
name: Deploy to VM
|
||||||
@@ -96,9 +97,33 @@ jobs:
|
|||||||
set -e
|
set -e
|
||||||
cd /opt/drb
|
cd /opt/drb
|
||||||
|
|
||||||
|
# server-26#129: every deploy pushes 3 freshly SHA-tagged images and
|
||||||
|
# nothing ever removed the old ones except a prune that only ran
|
||||||
|
# AFTER a successful `compose pull` -- so a run that never got that
|
||||||
|
# far (this one) left the leak unaddressed forever. That silently
|
||||||
|
# filled the disk to 100% over ~week of deploys (2026-09-12: 29G/29G
|
||||||
|
# used, 96 of 100 local images unreferenced, 23.76GB reclaimable) and
|
||||||
|
# took `git pull` itself down with "No space left on device" before
|
||||||
|
# the deploy could even determine a rollback target. Prune BEFORE
|
||||||
|
# doing anything else, not after: `docker image prune -af` only
|
||||||
|
# removes images with no container referencing them, so it can never
|
||||||
|
# touch what's currently running -- there is nothing here for a
|
||||||
|
# mid-flight deploy to lose. Warn-not-fail: a prune failure must not
|
||||||
|
# block a deploy that doesn't actually need the space this time.
|
||||||
|
docker image prune -af || echo "WARNING: pre-deploy image prune failed (server-26#129) -- disk pressure may persist"
|
||||||
|
|
||||||
# Update compose files + mosquitto config
|
# Update compose files + mosquitto config
|
||||||
git pull origin main
|
git pull origin main
|
||||||
|
|
||||||
|
# server-26#51: Firestore rules/indexes deploy used to be attempted
|
||||||
|
# HERE, over SSH, gated on the VM having firebase-tools installed.
|
||||||
|
# It never did (no node on the VM), so this silently warned and
|
||||||
|
# skipped on every deploy for weeks -- PR #124 even auto-closed
|
||||||
|
# #13/#51 as if it were fixed. Moved to a standalone
|
||||||
|
# deploy-firestore-rules job below that runs on the Gitea runner
|
||||||
|
# itself (which always has node), so it no longer depends on
|
||||||
|
# anything being pre-installed on this VM.
|
||||||
|
|
||||||
# server-26#65: capture what is actually live BEFORE switching, so
|
# server-26#65: capture what is actually live BEFORE switching, so
|
||||||
# a bad deploy has something concrete to fall back to. This reads
|
# a bad deploy has something concrete to fall back to. This reads
|
||||||
# from a state file rather than re-deriving it from git log,
|
# from a state file rather than re-deriving it from git log,
|
||||||
@@ -108,7 +133,20 @@ jobs:
|
|||||||
# has confirmed the tag it names actually answered /health. A
|
# has confirmed the tag it names actually answered /health. A
|
||||||
# fresh VM with no file yet falls back to :latest, same escape
|
# fresh VM with no file yet falls back to :latest, same escape
|
||||||
# hatch as a manual `up -d` with no TAG set.
|
# hatch as a manual `up -d` with no TAG set.
|
||||||
PREV_TAG=$(cat /opt/drb/.last_good_tag 2>/dev/null || echo latest)
|
#
|
||||||
|
# server-26#156: `cat missing-file || echo latest` only falls back
|
||||||
|
# when cat itself fails (nonzero exit) -- a file that EXISTS but is
|
||||||
|
# EMPTY (the state this file was found in, 2026-09-20) makes cat
|
||||||
|
# succeed with empty output, so PREV_TAG became "" instead of
|
||||||
|
# "latest". That "" then failed the emptiness check below and
|
||||||
|
# exited 1 -- AFTER git pull + up -d had already succeeded -- which
|
||||||
|
# skips the Health check step entirely (later steps don't run after
|
||||||
|
# a failure), and Health check is the ONLY thing that ever writes a
|
||||||
|
# real value here. Self-perpetuating: every deploy failed the same
|
||||||
|
# way forever, with the app itself deploying fine underneath it.
|
||||||
|
# ${VAR:-default} covers empty AND unset in one expansion.
|
||||||
|
PREV_TAG=$(cat /opt/drb/.last_good_tag 2>/dev/null)
|
||||||
|
PREV_TAG="${PREV_TAG:-latest}"
|
||||||
echo "PREV_TAG=$PREV_TAG"
|
echo "PREV_TAG=$PREV_TAG"
|
||||||
|
|
||||||
# Deploy THIS commit's images, not :latest. Overlapping runs are
|
# Deploy THIS commit's images, not :latest. Overlapping runs are
|
||||||
@@ -136,7 +174,12 @@ jobs:
|
|||||||
$COMPOSE pull
|
$COMPOSE pull
|
||||||
fi
|
fi
|
||||||
$COMPOSE up -d --remove-orphans
|
$COMPOSE up -d --remove-orphans
|
||||||
docker image prune -f
|
# server-26#129: -f alone only removes dangling (untagged) images --
|
||||||
|
# the SHA-tagged image from every PAST deploy is not dangling, just
|
||||||
|
# unreferenced once `up -d` swaps the running container to the new
|
||||||
|
# tag, so it survived this indefinitely. -a catches those too; see
|
||||||
|
# the pre-pull prune above for why this can't touch anything live.
|
||||||
|
docker image prune -af
|
||||||
ENDSSH
|
ENDSSH
|
||||||
)
|
)
|
||||||
echo "$OUTPUT"
|
echo "$OUTPUT"
|
||||||
@@ -248,9 +291,48 @@ jobs:
|
|||||||
echo "status=success" >> "$GITHUB_OUTPUT"
|
echo "status=success" >> "$GITHUB_OUTPUT"
|
||||||
echo "rolled_back_to=$PREV_TAG" >> "$GITHUB_OUTPUT"
|
echo "rolled_back_to=$PREV_TAG" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
deploy-firestore-rules:
|
||||||
|
name: Deploy Firestore rules & indexes
|
||||||
|
needs: build
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
# Deliberately independent of the `deploy` job (app containers) and its
|
||||||
|
# health-check/rollback chain above: a rules/indexes deploy failure has
|
||||||
|
# nothing to roll back (there is no previous "build" of a ruleset to
|
||||||
|
# revert to via this pipeline) and must never be conflated with an app
|
||||||
|
# deploy failure by triggering that job's rollback logic. This job
|
||||||
|
# failing is its own, separate red run -- picked up by notify-failure
|
||||||
|
# below -- not a signal to touch the running containers.
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Deploy firestore rules and indexes
|
||||||
|
env:
|
||||||
|
FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -e
|
||||||
|
# server-26#51: this used to run over SSH on the deploy VM, gated
|
||||||
|
# on the VM having firebase-tools installed. It never did, so it
|
||||||
|
# silently warned-and-skipped on every single deploy for weeks.
|
||||||
|
# Running it here instead means the only prerequisite is a secret
|
||||||
|
# -- FIREBASE_TOKEN, from `firebase login:ci` -- rather than
|
||||||
|
# something installed by hand on a machine this pipeline doesn't
|
||||||
|
# otherwise touch. A missing token now fails this job LOUDLY
|
||||||
|
# (picked up by notify-failure) instead of a buried warning line
|
||||||
|
# nobody reads in the app deploy's logs.
|
||||||
|
if [ -z "$FIREBASE_TOKEN" ]; then
|
||||||
|
echo "FIREBASE_TOKEN secret is not set -- cannot deploy Firestore rules/indexes." >&2
|
||||||
|
echo "Generate one with 'firebase login:ci' and add it as a Gitea Actions secret." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
npm install -g firebase-tools
|
||||||
|
cd infra/firestore
|
||||||
|
firebase deploy --only firestore:rules,firestore:indexes \
|
||||||
|
--project ${{ secrets.FIREBASE_PROJECT_ID }} \
|
||||||
|
--token "$FIREBASE_TOKEN" --non-interactive
|
||||||
|
|
||||||
notify-failure:
|
notify-failure:
|
||||||
name: Report a failed deploy
|
name: Report a failed deploy
|
||||||
needs: [build, deploy]
|
needs: [build, deploy, deploy-firestore-rules]
|
||||||
if: failure()
|
if: failure()
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
@@ -267,6 +349,8 @@ jobs:
|
|||||||
SHA: ${{ gitea.sha }}
|
SHA: ${{ gitea.sha }}
|
||||||
ROLLBACK_STATUS: ${{ needs.deploy.outputs.rollback_status }}
|
ROLLBACK_STATUS: ${{ needs.deploy.outputs.rollback_status }}
|
||||||
ROLLBACK_SHA: ${{ needs.deploy.outputs.rollback_sha }}
|
ROLLBACK_SHA: ${{ needs.deploy.outputs.rollback_sha }}
|
||||||
|
DEPLOY_RESULT: ${{ needs.deploy.result }}
|
||||||
|
RULES_RESULT: ${{ needs.deploy-firestore-rules.result }}
|
||||||
run: |
|
run: |
|
||||||
if [ -z "$WEBHOOK" ]; then
|
if [ -z "$WEBHOOK" ]; then
|
||||||
echo "DEPLOY_ALERT_WEBHOOK is not set - skipping notification."
|
echo "DEPLOY_ALERT_WEBHOOK is not set - skipping notification."
|
||||||
@@ -278,6 +362,16 @@ jobs:
|
|||||||
run_url = os.environ["RUN_URL"]
|
run_url = os.environ["RUN_URL"]
|
||||||
status = os.environ.get("ROLLBACK_STATUS", "")
|
status = os.environ.get("ROLLBACK_STATUS", "")
|
||||||
rollback_sha = os.environ.get("ROLLBACK_SHA", "")
|
rollback_sha = os.environ.get("ROLLBACK_SHA", "")
|
||||||
|
deploy_result = os.environ.get("DEPLOY_RESULT", "")
|
||||||
|
rules_result = os.environ.get("RULES_RESULT", "")
|
||||||
|
|
||||||
|
# deploy-firestore-rules runs independent of the app deploy/rollback
|
||||||
|
# chain (see its own job comment), so its failure needs its own
|
||||||
|
# branch here -- otherwise this fell through to the generic "Build
|
||||||
|
# failed before any deploy was attempted" text even when the app
|
||||||
|
# deployed fine and only the Firestore rules/indexes push failed.
|
||||||
|
if deploy_result != "failure" and rules_result == "failure":
|
||||||
|
detail = "App deploy succeeded; Firestore rules/indexes deploy FAILED (server-26#51). Rules may be stale — check FIREBASE_TOKEN and the job log."
|
||||||
|
|
||||||
# server-26#65: the old text here unconditionally claimed
|
# server-26#65: the old text here unconditionally claimed
|
||||||
# "production is still running the previous build" -- true only
|
# "production is still running the previous build" -- true only
|
||||||
@@ -286,7 +380,7 @@ jobs:
|
|||||||
# class of bug the correlator instrumentation exists to catch), or
|
# class of bug the correlator instrumentation exists to catch), or
|
||||||
# once the deploy job's own rollback path has run. Say what
|
# once the deploy job's own rollback path has run. Say what
|
||||||
# actually happened instead.
|
# actually happened instead.
|
||||||
if status == "success":
|
elif status == "success":
|
||||||
detail = "Automatic rollback to `%s` succeeded. Production is back on the previous good build." % rollback_sha[:8]
|
detail = "Automatic rollback to `%s` succeeded. Production is back on the previous good build." % rollback_sha[:8]
|
||||||
elif status == "failed":
|
elif status == "failed":
|
||||||
detail = ("Automatic rollback to `%s` FAILED. Production state is UNKNOWN -- "
|
detail = ("Automatic rollback to `%s` FAILED. Production state is UNKNOWN -- "
|
||||||
|
|||||||
@@ -47,3 +47,4 @@ Thumbs.db
|
|||||||
|
|
||||||
# Out of scope - not a deployed service (server-26#56)
|
# Out of scope - not a deployed service (server-26#56)
|
||||||
drb-telegram-bot/
|
drb-telegram-bot/
|
||||||
|
.claude/worktrees/
|
||||||
|
|||||||
@@ -33,6 +33,13 @@ SUMMARY_INTERVAL_MINUTES=15
|
|||||||
CORRELATION_WINDOW_HOURS=4
|
CORRELATION_WINDOW_HOURS=4
|
||||||
EMBEDDING_SIMILARITY_THRESHOLD=0.82
|
EMBEDDING_SIMILARITY_THRESHOLD=0.82
|
||||||
|
|
||||||
|
# Browser origins allowed to call this API cross-origin (JSON list). The only
|
||||||
|
# browser caller is the frontend's Archive page (GET /calls/search). Set this
|
||||||
|
# to the exact origin the frontend is served from — scheme + host, no path.
|
||||||
|
# Defaults to https://drb.cusano.net. A "*" entry works for local dev but is
|
||||||
|
# logged as a probable misconfiguration and never gets a credentialed response.
|
||||||
|
CORS_ORIGINS=["https://drb.cusano.net"]
|
||||||
|
|
||||||
# Fleet-wide token edge nodes present as X-Enrollment-Token on first boot
|
# Fleet-wide token edge nodes present as X-Enrollment-Token on first boot
|
||||||
# (POST /nodes/enroll). Shared across every node — NOT a per-node secret.
|
# (POST /nodes/enroll). Shared across every node — NOT a per-node secret.
|
||||||
# Generate with: openssl rand -hex 32
|
# Generate with: openssl rand -hex 32
|
||||||
|
|||||||
+23
-26
@@ -90,24 +90,19 @@ class Settings(BaseSettings):
|
|||||||
unit_continuity_max_idle_minutes: int = 20 # unit-continuity path: skip if incident idle > this
|
unit_continuity_max_idle_minutes: int = 20 # unit-continuity path: skip if incident idle > this
|
||||||
recorrelation_scan_minutes: int = 60 # re-examine orphaned calls ended within this window
|
recorrelation_scan_minutes: int = 60 # re-examine orphaned calls ended within this window
|
||||||
tg_fast_path_idle_minutes: int = 90 # fast path: max minutes since incident last updated
|
tg_fast_path_idle_minutes: int = 90 # fast path: max minutes since incident last updated
|
||||||
# Dispatch channels only: tier-2 thin calls attach to a lone candidate idle < this.
|
# Tier-2 thin calls attach to a lone candidate idle < this, on every
|
||||||
# Was 10, which is long enough for the channel to have moved on to something else:
|
# channel (server-26#133/#134 removed the dispatch/tactical split — a
|
||||||
# on 2026-08-16 a "72 at Holland Station" incident absorbed a Grand Central train
|
# channel's name doesn't change how much scrutiny it gets). Was 10, which
|
||||||
# meet 9.6 min later, and a status check absorbed a records lookup at 9.7 min.
|
# is long enough for the channel to have moved on to something else: on
|
||||||
# Across that dump every correct thin attach was <= 3.4 min idle and every wrong
|
# 2026-08-16 a "72 at Holland Station" incident absorbed a Grand Central
|
||||||
# one was >= 8.2, so 5 separates them with room on both sides. Genuine
|
# train meet 9.6 min later, and a status check absorbed a records lookup
|
||||||
# back-and-forth is handled by the 30-second tier-1 path above this.
|
# at 9.7 min. Every correct thin attach in that dump was <= 3.4 min idle
|
||||||
|
# and every wrong one was >= 8.2, so 5 separates them with room on both
|
||||||
|
# sides. Genuine back-and-forth is handled by the 30-second tier-1 path
|
||||||
|
# above this. Also the escape hatch in routers/upload.py's LLM-orphan gate
|
||||||
|
# (_recent_incident_on_same_talkgroup, server-26#115) — check both call
|
||||||
|
# sites before retuning this.
|
||||||
tg_dispatch_thin_idle_minutes: int = 5
|
tg_dispatch_thin_idle_minutes: int = 5
|
||||||
# Every other channel: tier-2 thin calls attach to a lone candidate idle < this.
|
|
||||||
# Non-dispatch talkgroups previously had NO tier-2 bound at all — they used the
|
|
||||||
# whole 90-minute tg_fast_path_idle_minutes window with no single-candidate
|
|
||||||
# requirement and no fit test, which is the widest version of the 2026-08-20
|
|
||||||
# over-merge. A tactical channel really is dedicated to one scene, so it earns
|
|
||||||
# a longer window than a dispatch backbone, but not an unbounded one: 15 sits
|
|
||||||
# inside the 20-minute tactical-default window in _call_fits_incident, so the
|
|
||||||
# no-evidence thin path is never more permissive than the fit-tested path on
|
|
||||||
# the same channel.
|
|
||||||
tg_thin_idle_minutes: int = 15
|
|
||||||
|
|
||||||
# ── Hard caps: an incident past either of these stops accepting calls ──────
|
# ── Hard caps: an incident past either of these stops accepting calls ──────
|
||||||
# Enforced on every correlation path (see _incident_at_capacity). Pairwise fit
|
# Enforced on every correlation path (see _incident_at_capacity). Pairwise fit
|
||||||
@@ -180,16 +175,18 @@ class Settings(BaseSettings):
|
|||||||
# between genuinely separate transmissions on a busy dispatch channel.
|
# between genuinely separate transmissions on a busy dispatch channel.
|
||||||
duplicate_window_seconds: int = 10
|
duplicate_window_seconds: int = 10
|
||||||
|
|
||||||
# CORS — set to your frontend origin(s) in production, e.g. ["https://app.example.com"]
|
# Browser origins allowed to call this API cross-origin. The only browser
|
||||||
# Defaults to "*" for local development only.
|
# caller is the frontend's Archive page (GET /calls/search) — every other
|
||||||
|
# page reads Firestore directly. The frontend is served on the BARE domain
|
||||||
|
# (see infra Caddyfile.j2 — only drb. and api. have DNS records), so the
|
||||||
|
# default is that origin, not app.<domain>. Override via CORS_ORIGINS (JSON
|
||||||
|
# list) if the frontend ever moves; keep infra/.../c2-core.env.j2 in sync.
|
||||||
#
|
#
|
||||||
# Leaving this as "*" is not merely permissive: main.py turns OFF
|
# A "*" entry here still works for local dev but is refused a credentialed
|
||||||
# allow_credentials when it sees a wildcard, because Starlette would
|
# response: main.py never enables allow_credentials (auth is a Bearer
|
||||||
# otherwise reflect each caller's origin back WITH
|
# header, not a cookie), and it logs a loud ERROR when it sees a wildcard
|
||||||
# Access-Control-Allow-Credentials. So a production deployment that
|
# in a deployment so a forgotten override is visible.
|
||||||
# forgets to set this gets a loud ERROR at startup and loses credentialed
|
cors_origins: list[str] = ["https://drb.cusano.net"]
|
||||||
# cross-origin requests, rather than silently accepting every origin.
|
|
||||||
cors_origins: list[str] = ["*"]
|
|
||||||
|
|
||||||
# Discord webhook URL that app/internal/ai_health.py posts to when an AI
|
# Discord webhook URL that app/internal/ai_health.py posts to when an AI
|
||||||
# tier (transcription/correlation) transitions into or out of degraded
|
# tier (transcription/correlation) transitions into or out of degraded
|
||||||
|
|||||||
@@ -0,0 +1,135 @@
|
|||||||
|
"""
|
||||||
|
Upstream dispatch-vs-chatter classifier — SHADOW MODE (server-26#115 follow-up).
|
||||||
|
|
||||||
|
Three live measurement windows (CORRELATION_REVIEW_0907.md, _0907b.md, _0912.md)
|
||||||
|
and two consensus-layer fixes (#125, #126) all converged on the same conclusion:
|
||||||
|
the actual non-event-promotion problem lives upstream of correlation entirely.
|
||||||
|
Radio housekeeping — unit check-ins, roll call, bare 10-4/10-8/98 acknowledgements
|
||||||
|
— has no incident content for `intelligence.extract_scenes` to find, but nothing
|
||||||
|
stops it from being sent to the scene-extraction LLM and coming out the other end
|
||||||
|
as a thin "scene" for the correlator to then judge. See CORRELATION_REVIEW_0912.md
|
||||||
|
("Reminder: the real fix is still unscoped") and issue #115.
|
||||||
|
|
||||||
|
This module is that classifier. It is a PURE function of the transcript text —
|
||||||
|
no Firestore, no LLM call, no side effects — so it is cheap to run on every
|
||||||
|
transcript and cheap to test against real dumps offline.
|
||||||
|
|
||||||
|
SHADOW MODE ONLY. As of this module's introduction, nothing skips scene
|
||||||
|
extraction based on this verdict. `intelligence.extract_scenes` calls
|
||||||
|
`classify_chatter` purely to record the verdict on the call doc
|
||||||
|
(`chatter_classifier_verdict` / `chatter_classifier_reason`) so it becomes
|
||||||
|
observable in the next `/admin` correlation-debug dump, exactly like
|
||||||
|
`corr_gate_veto` (server-26#115 / PR #126). See the TODO at that call site for
|
||||||
|
what has to be true before this flips live.
|
||||||
|
|
||||||
|
Precision over recall, deliberately. A false positive here — flagging a REAL
|
||||||
|
event as chatter — would, once live, silently mean that event never gets a
|
||||||
|
scene, never gets tags/location/severity, and never has a chance to become an
|
||||||
|
incident. That is a much bigger, harder-to-notice failure than a false
|
||||||
|
negative (a housekeeping call that still goes through the existing expensive
|
||||||
|
pipeline and gets judged "not an incident" the same way it is today). When a
|
||||||
|
transcript doesn't clearly match one of the shapes below, this returns
|
||||||
|
(False, None) and the existing pipeline runs exactly as it does today.
|
||||||
|
|
||||||
|
Patterns are drawn from hand-labeled examples in CORRELATION_REVIEW_0907b.md
|
||||||
|
and CORRELATION_REVIEW_0912.md, cross-referenced against the real transcripts
|
||||||
|
in corr_dump_9-7_0437am.json / corr_dump_9-7_pm.json / corr_dump_9-12.json —
|
||||||
|
not invented regexes. See the backtest script referenced in the PR for the
|
||||||
|
per-dump catch rate and false-positive count.
|
||||||
|
"""
|
||||||
|
import re
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
# Police/law-enforcement phonetic alphabet words (APCO + NATO). Deliberately
|
||||||
|
# duplicated from intelligence.py's `_PHONETIC_ALPHA_WORDS` rather than
|
||||||
|
# imported — intelligence.py imports this module (to write the shadow-mode
|
||||||
|
# verdict onto the call doc), so importing back would be circular. Keep the
|
||||||
|
# two sets in sync if either changes; they're small and rarely touched.
|
||||||
|
_PHONETIC_ALPHA_WORDS = frozenset({
|
||||||
|
# APCO (law enforcement)
|
||||||
|
"adam", "baker", "charles", "david", "edward", "frank", "george", "henry",
|
||||||
|
"ida", "john", "king", "lincoln", "mary", "nora", "ocean", "paul", "queen",
|
||||||
|
"robert", "sam", "tom", "union", "victor", "william", "x-ray", "young", "zebra",
|
||||||
|
# NATO
|
||||||
|
"alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel",
|
||||||
|
"india", "juliet", "kilo", "lima", "mike", "november", "oscar", "papa",
|
||||||
|
"quebec", "romeo", "sierra", "tango", "uniform", "whiskey", "yankee", "zulu",
|
||||||
|
})
|
||||||
|
|
||||||
|
_TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9\-]*")
|
||||||
|
|
||||||
|
# Bare radio-procedure words that carry zero incident content by themselves.
|
||||||
|
# Deliberately small and literal — this is not a general stopword list, it's
|
||||||
|
# the exact vocabulary observed in hand-labeled chatter transcripts. Words
|
||||||
|
# that are ambiguous outside a pure-procedure context (e.g. "location",
|
||||||
|
# "call", "phone", "number", "go") are left OUT on purpose: including them
|
||||||
|
# risks reducing a real, substantive transcript down to nothing.
|
||||||
|
_FILLER_WORDS = frozenset({
|
||||||
|
"to", "this", "is", "the", "a", "and", "for", "you", "can", "i", "in",
|
||||||
|
"on", "of", "that", "just", "from", "out", "ok", "okay", "at", "be",
|
||||||
|
"show", "me", "mark", "marked", "charge", "standby", "stand", "by",
|
||||||
|
"clear", "available", "affirm", "affirmative", "negative", "copy",
|
||||||
|
"copies", "received", "roger",
|
||||||
|
})
|
||||||
|
|
||||||
|
# Agency/procedural designators — who's being addressed, not what happened.
|
||||||
|
_RADIO_DESIGNATORS = frozenset({
|
||||||
|
"central", "dispatch", "headquarters", "hq", "post", "unit", "sergeant",
|
||||||
|
"sgt", "metro", "mta", "division", "county",
|
||||||
|
})
|
||||||
|
|
||||||
|
_ROLL_CALL_RE = re.compile(r"\broll\s*call\b")
|
||||||
|
|
||||||
|
|
||||||
|
def _tokenize(transcript: str) -> list[str]:
|
||||||
|
return _TOKEN_RE.findall(transcript.lower())
|
||||||
|
|
||||||
|
|
||||||
|
def _is_filler_token(token: str) -> bool:
|
||||||
|
# Any token starting with a digit is a unit ID, 10-code, badge/post
|
||||||
|
# number, or call-number fragment ("10-4", "6-8", "72-holland",
|
||||||
|
# "11-victor", "98", "114") — procedural, not incident content. This is
|
||||||
|
# deliberately broad: a real event transcript that happens to include a
|
||||||
|
# digit-led token (an address number, a case number) still has other,
|
||||||
|
# non-digit descriptive words left over, so this alone never reduces a
|
||||||
|
# real transcript to nothing. See the backtest for confirmation.
|
||||||
|
if token[0].isdigit():
|
||||||
|
return True
|
||||||
|
return (
|
||||||
|
token in _FILLER_WORDS
|
||||||
|
or token in _RADIO_DESIGNATORS
|
||||||
|
or token in _PHONETIC_ALPHA_WORDS
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def classify_chatter(transcript: Optional[str]) -> tuple[bool, Optional[str]]:
|
||||||
|
"""
|
||||||
|
Pure classification of a transcript as non-event radio housekeeping.
|
||||||
|
|
||||||
|
Returns (is_chatter, reason):
|
||||||
|
(True, "roll_call") — contains a roll-call announcement
|
||||||
|
(True, "bare_acknowledgement") — every token is a callsign/10-code/
|
||||||
|
procedural filler word; nothing else
|
||||||
|
(False, None) — not confidently chatter; let the
|
||||||
|
existing pipeline run as today
|
||||||
|
|
||||||
|
Takes only the transcript. Other call metadata (talkgroup, severity, tags)
|
||||||
|
doesn't exist yet at the point this needs to run — this classifier is
|
||||||
|
upstream of the scene-extraction call that produces those fields — so it
|
||||||
|
deliberately doesn't take them as input.
|
||||||
|
"""
|
||||||
|
if not transcript or not transcript.strip():
|
||||||
|
return False, None
|
||||||
|
|
||||||
|
lowered = transcript.lower()
|
||||||
|
if _ROLL_CALL_RE.search(lowered):
|
||||||
|
return True, "roll_call"
|
||||||
|
|
||||||
|
tokens = _tokenize(transcript)
|
||||||
|
if not tokens:
|
||||||
|
return False, None
|
||||||
|
|
||||||
|
if any(not _is_filler_token(t) for t in tokens):
|
||||||
|
return False, None
|
||||||
|
|
||||||
|
return True, "bare_acknowledgement"
|
||||||
@@ -7,6 +7,15 @@ from google.cloud.firestore_v1.base_query import FieldFilter
|
|||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.internal.logger import logger
|
from app.internal.logger import logger
|
||||||
|
|
||||||
|
# Re-exported so callers never need their own `firebase_admin.firestore` import
|
||||||
|
# just to delete a field. server-26#96/#114 review: `doc_set(..., merge=True)`
|
||||||
|
# merges nested maps by key but can never REMOVE one — writing `{"scenes": {}}`
|
||||||
|
# to clear a map is a no-op, not a delete. Use `doc_update(coll, id, {"field":
|
||||||
|
# fstore.DELETE_FIELD})` (or doc_set + merge, DELETE_FIELD works under both)
|
||||||
|
# whenever a re-extraction/reprocess path needs a stale nested field gone
|
||||||
|
# rather than merged over.
|
||||||
|
DELETE_FIELD = fs.DELETE_FIELD
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# In-memory TTL cache for rarely-changing documents (systems, nodes config)
|
# In-memory TTL cache for rarely-changing documents (systems, nodes config)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -91,12 +91,12 @@ def _max_severity(current: Optional[str], new: Optional[str]) -> str:
|
|||||||
_MAX_PURSUIT_SPEED_KM_PER_MIN = 8.0 # ~300 km/h, intentionally generous
|
_MAX_PURSUIT_SPEED_KM_PER_MIN = 8.0 # ~300 km/h, intentionally generous
|
||||||
_PURSUIT_PROXIMITY_KM = 20.0 # expanded radius for moving incidents
|
_PURSUIT_PROXIMITY_KM = 20.0 # expanded radius for moving incidents
|
||||||
|
|
||||||
_DISPATCH_TG_RE = re.compile(
|
# server-26#115 — the location path linked on `location_proximity_km` (0.5 km)
|
||||||
r"\bdispatch\b|\bdisp\b"
|
# alone, with no unit or content check. In a dense village two unrelated events
|
||||||
r"|\bpatched\b" # patched channels aggregate multiple call streams
|
# routinely geocode that close (a vehicle lockout stitched to a station-restroom
|
||||||
r"|\bprimary\b", # "Primary" channels serve as shared backbones
|
# slip; two different churches an hour apart). A location link now needs unit
|
||||||
re.IGNORECASE,
|
# overlap with the candidate OR a distance under this tighter bar.
|
||||||
)
|
_LOCATION_TIGHT_PROXIMITY_KM = 0.2
|
||||||
|
|
||||||
# Matches route/road identifiers in location strings for cross-system parent detection.
|
# Matches route/road identifiers in location strings for cross-system parent detection.
|
||||||
# Groups: numbered routes (Route 202, NY-9, US-6, I-87, CR-35) and named parkways/highways.
|
# Groups: numbered routes (Route 202, NY-9, US-6, I-87, CR-35) and named parkways/highways.
|
||||||
@@ -108,16 +108,31 @@ _ROAD_RE = re.compile(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# Street-type synonyms collapsed to one token so "Mohegan Park Avenue" and
|
||||||
|
# "Mohegan Park Ave" produce the same road id (server-26#115 — that one
|
||||||
|
# difference was splitting a car-alarm incident into two).
|
||||||
|
_ROAD_SUFFIX_CANON = {
|
||||||
|
"avenue": "ave", "street": "st", "road": "rd", "drive": "dr",
|
||||||
|
"boulevard": "blvd", "lane": "ln", "court": "ct", "place": "pl",
|
||||||
|
"highway": "hwy", "parkway": "pkwy",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _extract_road_ids(text: str) -> set[str]:
|
def _extract_road_ids(text: str) -> set[str]:
|
||||||
"""
|
"""
|
||||||
Extract normalised road/route identifiers from a location string.
|
Extract normalised road/route identifiers from a location string.
|
||||||
e.g. "suspect east on Route 202" → {"route 202"}
|
e.g. "suspect east on Route 202" → {"route 202"}
|
||||||
"at Main Street and Oak Ave" → {"main street", "oak ave"}
|
"at Main Street and Oak Ave" → {"main st", "oak ave"}
|
||||||
"""
|
"""
|
||||||
return {
|
ids: set[str] = set()
|
||||||
re.sub(r"[\s.\-]+", " ", m.group().lower()).strip()
|
for m in _ROAD_RE.finditer(text):
|
||||||
for m in _ROAD_RE.finditer(text)
|
key = re.sub(r"[\s.\-]+", " ", m.group().lower()).strip()
|
||||||
}
|
parts = key.split()
|
||||||
|
if parts and parts[-1] in _ROAD_SUFFIX_CANON:
|
||||||
|
parts[-1] = _ROAD_SUFFIX_CANON[parts[-1]]
|
||||||
|
key = " ".join(parts)
|
||||||
|
ids.add(key)
|
||||||
|
return ids
|
||||||
|
|
||||||
|
|
||||||
def _location_mentions_road_overlap(new_location: str, inc_mentions: list[str]) -> bool:
|
def _location_mentions_road_overlap(new_location: str, inc_mentions: list[str]) -> bool:
|
||||||
@@ -224,6 +239,22 @@ def _matching_units(call_units: Optional[list[str]], inc_units: Optional[list[st
|
|||||||
return [u for u in (call_units or []) if _normalize_unit(u) in inc_keys]
|
return [u for u in (call_units or []) if _normalize_unit(u) in inc_keys]
|
||||||
|
|
||||||
|
|
||||||
|
def has_event_substance(ctx: dict) -> bool:
|
||||||
|
"""
|
||||||
|
True when the call carries content beyond who-was-speaking-and-where:
|
||||||
|
a vehicle, a geocode, or a tag.
|
||||||
|
|
||||||
|
This is the substance half of the incident-creation gate (see
|
||||||
|
`_run_decision`, "Severity, not type, decides..."), factored out so the
|
||||||
|
consensus LLM-orphan gate in routers/upload.py mirrors it exactly and can
|
||||||
|
never drop a call the creation gate would have opened. `call_units` and
|
||||||
|
`location` are deliberately excluded — radio protocol puts a unit ID and a
|
||||||
|
place name in almost every transmission, so counting them as substance
|
||||||
|
makes the check trivially true.
|
||||||
|
"""
|
||||||
|
return bool(ctx.get("call_vehicles") or ctx.get("coords") or ctx.get("tags"))
|
||||||
|
|
||||||
|
|
||||||
def _infer_type_from_tags(tags: list[str]) -> Optional[str]:
|
def _infer_type_from_tags(tags: list[str]) -> Optional[str]:
|
||||||
"""Return an incident type inferred from tags, or None if ambiguous."""
|
"""Return an incident type inferred from tags, or None if ambiguous."""
|
||||||
for tag in tags:
|
for tag in tags:
|
||||||
@@ -456,13 +487,6 @@ def _resolve_incident_title(
|
|||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
|
||||||
def _is_dispatch_channel(talkgroup_name: Optional[str]) -> bool:
|
|
||||||
"""True when the talkgroup is a shared dispatch backbone (not a tactical/working channel)."""
|
|
||||||
if not talkgroup_name:
|
|
||||||
return False
|
|
||||||
return bool(_DISPATCH_TG_RE.search(talkgroup_name))
|
|
||||||
|
|
||||||
|
|
||||||
def _incident_idle_minutes(inc: dict, now: datetime) -> float:
|
def _incident_idle_minutes(inc: dict, now: datetime) -> float:
|
||||||
"""Minutes since the incident was last updated (or started). Returns 9999 on parse error."""
|
"""Minutes since the incident was last updated (or started). Returns 9999 on parse error."""
|
||||||
try:
|
try:
|
||||||
@@ -670,6 +694,8 @@ async def correlate_call(
|
|||||||
reassignment: bool = False,
|
reassignment: bool = False,
|
||||||
embedding: Optional[list] = None,
|
embedding: Optional[list] = None,
|
||||||
severity: Optional[str] = None,
|
severity: Optional[str] = None,
|
||||||
|
transcript: Optional[str] = None,
|
||||||
|
scene_index: int = 0,
|
||||||
) -> Optional[str]:
|
) -> Optional[str]:
|
||||||
"""
|
"""
|
||||||
Link call_id to an existing incident or create a new one.
|
Link call_id to an existing incident or create a new one.
|
||||||
@@ -679,6 +705,11 @@ async def correlate_call(
|
|||||||
Callers that re-correlate a whole call rather than a scene — the
|
Callers that re-correlate a whole call rather than a scene — the
|
||||||
recorrelation sweep — pass the call doc's stored values explicitly; they are
|
recorrelation sweep — pass the call doc's stored values explicitly; they are
|
||||||
no longer read from the doc inside _build_context.
|
no longer read from the doc inside _build_context.
|
||||||
|
|
||||||
|
``scene_index`` (server-26#96) identifies which scene of the call this
|
||||||
|
decision belongs to for the per-scene ``scenes`` map written by
|
||||||
|
_apply_and_log. Defaults to 0 — correct for every caller here, since this
|
||||||
|
entry point always re-correlates a call as a single unit, not a scene loop.
|
||||||
"""
|
"""
|
||||||
ctx = await _build_context(
|
ctx = await _build_context(
|
||||||
call_id=call_id, units=units, vehicles=vehicles, cleared_units=cleared_units,
|
call_id=call_id, units=units, vehicles=vehicles, cleared_units=cleared_units,
|
||||||
@@ -686,7 +717,8 @@ async def correlate_call(
|
|||||||
system_id=system_id, talkgroup_id=talkgroup_id, talkgroup_name=talkgroup_name,
|
system_id=system_id, talkgroup_id=talkgroup_id, talkgroup_name=talkgroup_name,
|
||||||
tags=tags, incident_type=incident_type, location=location,
|
tags=tags, incident_type=incident_type, location=location,
|
||||||
reassignment=reassignment, create_if_new=create_if_new,
|
reassignment=reassignment, create_if_new=create_if_new,
|
||||||
embedding=embedding, severity=severity,
|
embedding=embedding, severity=severity, transcript=transcript,
|
||||||
|
scene_index=scene_index,
|
||||||
)
|
)
|
||||||
decision = _run_decision(ctx)
|
decision = _run_decision(ctx)
|
||||||
return await _apply_and_log(decision, ctx)
|
return await _apply_and_log(decision, ctx)
|
||||||
@@ -710,6 +742,8 @@ async def preview_correlation(
|
|||||||
reassignment: bool = False,
|
reassignment: bool = False,
|
||||||
embedding: Optional[list] = None,
|
embedding: Optional[list] = None,
|
||||||
severity: Optional[str] = None,
|
severity: Optional[str] = None,
|
||||||
|
transcript: Optional[str] = None,
|
||||||
|
scene_index: int = 0,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""
|
"""
|
||||||
Run the rules engine and return the decision WITHOUT committing to Firestore.
|
Run the rules engine and return the decision WITHOUT committing to Firestore.
|
||||||
@@ -723,6 +757,15 @@ async def preview_correlation(
|
|||||||
matched_incident the candidate incident doc (action == "link")
|
matched_incident the candidate incident doc (action == "link")
|
||||||
incident_type resolved type after tag inference (action == "new")
|
incident_type resolved type after tag inference (action == "new")
|
||||||
corr_debug fields to persist on the call doc
|
corr_debug fields to persist on the call doc
|
||||||
|
|
||||||
|
``scene_index`` (server-26#96) — which scene of the call (upload.py's
|
||||||
|
``for scene_index, scene in enumerate(scenes):`` loop) this call is. It
|
||||||
|
rides through ctx to _apply_and_log, which uses it as the key under the
|
||||||
|
call doc's ``scenes`` map so each scene's corr_debug/transcript lands in
|
||||||
|
its own map entry instead of colliding on the shared flat fields. Defaults
|
||||||
|
to 0 for callers with no scene concept (a single-scene call, or the
|
||||||
|
no-scenes-extracted correlation attempt) — equivalent to today's
|
||||||
|
behaviour for those calls.
|
||||||
"""
|
"""
|
||||||
ctx = await _build_context(
|
ctx = await _build_context(
|
||||||
call_id=call_id, units=units, vehicles=vehicles, cleared_units=cleared_units,
|
call_id=call_id, units=units, vehicles=vehicles, cleared_units=cleared_units,
|
||||||
@@ -730,7 +773,8 @@ async def preview_correlation(
|
|||||||
system_id=system_id, talkgroup_id=talkgroup_id, talkgroup_name=talkgroup_name,
|
system_id=system_id, talkgroup_id=talkgroup_id, talkgroup_name=talkgroup_name,
|
||||||
tags=tags, incident_type=incident_type, location=location,
|
tags=tags, incident_type=incident_type, location=location,
|
||||||
reassignment=reassignment, create_if_new=create_if_new,
|
reassignment=reassignment, create_if_new=create_if_new,
|
||||||
embedding=embedding, severity=severity,
|
embedding=embedding, severity=severity, transcript=transcript,
|
||||||
|
scene_index=scene_index,
|
||||||
)
|
)
|
||||||
decision = _run_decision(ctx)
|
decision = _run_decision(ctx)
|
||||||
return {"decision": decision, "ctx": ctx}
|
return {"decision": decision, "ctx": ctx}
|
||||||
@@ -765,6 +809,8 @@ async def _build_context(
|
|||||||
create_if_new: bool,
|
create_if_new: bool,
|
||||||
embedding: Optional[list] = None,
|
embedding: Optional[list] = None,
|
||||||
severity: Optional[str] = None,
|
severity: Optional[str] = None,
|
||||||
|
transcript: Optional[str] = None,
|
||||||
|
scene_index: int = 0,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
now = reference_time or datetime.now(timezone.utc)
|
now = reference_time or datetime.now(timezone.utc)
|
||||||
window = timedelta(hours=settings.correlation_window_hours)
|
window = timedelta(hours=settings.correlation_window_hours)
|
||||||
@@ -804,6 +850,13 @@ async def _build_context(
|
|||||||
call_vehicles = vehicles if vehicles is not None else (call_doc.get("vehicles") or [])
|
call_vehicles = vehicles if vehicles is not None else (call_doc.get("vehicles") or [])
|
||||||
call_cleared = cleared_units if cleared_units is not None else (call_doc.get("cleared_units") or [])
|
call_cleared = cleared_units if cleared_units is not None else (call_doc.get("cleared_units") or [])
|
||||||
call_severity = severity or "routine"
|
call_severity = severity or "routine"
|
||||||
|
# The transcript the LLM correlation tier reasons over. Prefer the SCENE's
|
||||||
|
# own words (server-26#102) — passed by upload.py's scene loop — and fall
|
||||||
|
# back to the call doc only when no scene text was supplied (the
|
||||||
|
# recorrelation sweep, and single-scene calls where the two are identical).
|
||||||
|
# Without this, every non-primary scene of a multi-scene call was judged by
|
||||||
|
# the LLM against a transcript containing the OTHER scenes.
|
||||||
|
scene_transcript = transcript or call_doc.get("transcript_corrected") or call_doc.get("transcript")
|
||||||
# A string that is not a place is not a location anywhere downstream — not
|
# A string that is not a place is not a location anywhere downstream — not
|
||||||
# in the fit tests, not in the thin-call test, not in the LLM prompt, and
|
# in the fit tests, not in the thin-call test, not in the LLM prompt, and
|
||||||
# not on the incident. Its coordinates go with it: coords are geocoded
|
# not on the incident. Its coordinates go with it: coords are geocoded
|
||||||
@@ -822,10 +875,17 @@ async def _build_context(
|
|||||||
is_thin_call = _is_thin_call(
|
is_thin_call = _is_thin_call(
|
||||||
call_units, call_vehicles, coords, tags, location, call_severity, reassignment
|
call_units, call_vehicles, coords, tags, location, call_severity, reassignment
|
||||||
)
|
)
|
||||||
|
# server-26#158: the P25 source radio ID. Captured on every call by the
|
||||||
|
# edge node's metadata_watcher.py independent of transcript content, so
|
||||||
|
# it survives even when transcript_too_short skips GPT extraction
|
||||||
|
# entirely and leaves call_units empty — exactly the population the
|
||||||
|
# thin-call path below has no other identity signal for.
|
||||||
|
call_srcaddr = call_doc.get("srcaddr")
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"call_id": call_id, "org_id": org_id, "all_active": all_active, "recent": recent,
|
"call_id": call_id, "org_id": org_id, "all_active": all_active, "recent": recent,
|
||||||
"call_doc": call_doc, "call_embedding": call_embedding,
|
"call_doc": call_doc, "call_embedding": call_embedding,
|
||||||
|
"scene_transcript": scene_transcript, "call_srcaddr": call_srcaddr,
|
||||||
"call_units": call_units, "call_vehicles": call_vehicles,
|
"call_units": call_units, "call_vehicles": call_vehicles,
|
||||||
"call_cleared": call_cleared, "call_severity": call_severity,
|
"call_cleared": call_cleared, "call_severity": call_severity,
|
||||||
"coords": coords, "is_thin_call": is_thin_call, "now": now,
|
"coords": coords, "is_thin_call": is_thin_call, "now": now,
|
||||||
@@ -834,6 +894,9 @@ async def _build_context(
|
|||||||
"incident_type": incident_type, "location": location,
|
"incident_type": incident_type, "location": location,
|
||||||
"location_coords": location_coords, "reassignment": reassignment,
|
"location_coords": location_coords, "reassignment": reassignment,
|
||||||
"create_if_new": create_if_new,
|
"create_if_new": create_if_new,
|
||||||
|
# server-26#96 — which scene of the call this decision is for. Carried
|
||||||
|
# through so _apply_and_log can key the per-scene write correctly.
|
||||||
|
"scene_index": scene_index,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -876,6 +939,7 @@ def _run_decision(ctx: dict) -> dict:
|
|||||||
call_severity = ctx["call_severity"]
|
call_severity = ctx["call_severity"]
|
||||||
coords = ctx["coords"]
|
coords = ctx["coords"]
|
||||||
is_thin_call = ctx["is_thin_call"]
|
is_thin_call = ctx["is_thin_call"]
|
||||||
|
call_srcaddr = ctx.get("call_srcaddr")
|
||||||
system_id = ctx["system_id"]
|
system_id = ctx["system_id"]
|
||||||
talkgroup_id = ctx["talkgroup_id"]
|
talkgroup_id = ctx["talkgroup_id"]
|
||||||
talkgroup_name = ctx["talkgroup_name"]
|
talkgroup_name = ctx["talkgroup_name"]
|
||||||
@@ -902,14 +966,13 @@ def _run_decision(ctx: dict) -> dict:
|
|||||||
if talkgroup_id is not None and system_id:
|
if talkgroup_id is not None and system_id:
|
||||||
tg_str = str(talkgroup_id)
|
tg_str = str(talkgroup_id)
|
||||||
# talkgroup_name may be None when the upload form omits it (node sets it
|
# talkgroup_name may be None when the upload form omits it (node sets it
|
||||||
# directly on the Firestore call doc). Fall back to the call doc so that
|
# directly on the Firestore call doc). Fall back to the call doc so the
|
||||||
# dispatch-channel strictness works regardless of how the call arrived.
|
# log lines below still name the channel.
|
||||||
effective_talkgroup_name = talkgroup_name or call_doc.get("talkgroup_name")
|
effective_talkgroup_name = talkgroup_name or call_doc.get("talkgroup_name")
|
||||||
is_dispatch = _is_dispatch_channel(effective_talkgroup_name)
|
|
||||||
if effective_talkgroup_name != talkgroup_name:
|
if effective_talkgroup_name != talkgroup_name:
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Correlator: talkgroup_name missing from request for call {call_id}, "
|
f"Correlator: talkgroup_name missing from request for call {call_id}, "
|
||||||
f"resolved from call doc: {effective_talkgroup_name!r} → is_dispatch={is_dispatch}"
|
f"resolved from call doc: {effective_talkgroup_name!r}"
|
||||||
)
|
)
|
||||||
|
|
||||||
tg_matches = [
|
tg_matches = [
|
||||||
@@ -952,11 +1015,28 @@ def _run_decision(ctx: dict) -> dict:
|
|||||||
# incident idle up to tg_fast_path_idle_minutes (90) with no
|
# incident idle up to tg_fast_path_idle_minutes (90) with no
|
||||||
# single-candidate requirement and no fit test of any kind. Four
|
# single-candidate requirement and no fit test of any kind. Four
|
||||||
# hours is not a bound, and neither is ninety minutes.
|
# hours is not a bound, and neither is ninety minutes.
|
||||||
|
# server-26#158: identity beats guesswork. A thin call has no
|
||||||
|
# extracted units (GPT never ran), but it still carries the P25
|
||||||
|
# radio ID that transmitted it — stronger, cheaper evidence than
|
||||||
|
# "most recently active" and immune to the exact failure this
|
||||||
|
# path exists to guard against: two incidents both live on one
|
||||||
|
# busy dispatch channel. If the radio that sent this call already
|
||||||
|
# has calls on one of the TG-matched incidents, that IS the
|
||||||
|
# thread, regardless of which incident is more recently updated
|
||||||
|
# or how many candidates are in the window.
|
||||||
|
srcaddr_matches = [
|
||||||
|
inc for inc in tg_recent
|
||||||
|
if call_srcaddr and call_srcaddr in (inc.get("srcaddrs") or [])
|
||||||
|
]
|
||||||
THIN_CONVERSATIONAL_SECS = 30
|
THIN_CONVERSATIONAL_SECS = 30
|
||||||
thin_window_min = (
|
thin_window_min = settings.tg_dispatch_thin_idle_minutes
|
||||||
settings.tg_dispatch_thin_idle_minutes if is_dispatch
|
if srcaddr_matches:
|
||||||
else settings.tg_thin_idle_minutes
|
thin_pool = [max(srcaddr_matches, key=lambda inc: inc.get("updated_at", ""))]
|
||||||
|
logger.info(
|
||||||
|
f"Correlator fast-path thin (srcaddr match): radio {call_srcaddr} "
|
||||||
|
f"already on {len(srcaddr_matches)} candidate(s) for call {call_id}"
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
very_recent = [
|
very_recent = [
|
||||||
inc for inc in tg_recent
|
inc for inc in tg_recent
|
||||||
if _idle_gate_minutes(inc, now) * 60 <= THIN_CONVERSATIONAL_SECS
|
if _idle_gate_minutes(inc, now) * 60 <= THIN_CONVERSATIONAL_SECS
|
||||||
@@ -978,8 +1058,7 @@ def _run_decision(ctx: dict) -> dict:
|
|||||||
if len(thin_pool) > 1:
|
if len(thin_pool) > 1:
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Correlator fast-path thin (tier-2): {len(thin_pool)} active incidents "
|
f"Correlator fast-path thin (tier-2): {len(thin_pool)} active incidents "
|
||||||
f"on {'dispatch' if is_dispatch else 'tactical'} channel — "
|
f"— ambiguous, skipping thin call {call_id}"
|
||||||
f"ambiguous, skipping thin call {call_id}"
|
|
||||||
)
|
)
|
||||||
thin_pool = []
|
thin_pool = []
|
||||||
|
|
||||||
@@ -997,8 +1076,10 @@ def _run_decision(ctx: dict) -> dict:
|
|||||||
# no fit signal, so the admin debug view's "fit_signal
|
# no fit signal, so the admin debug view's "fit_signal
|
||||||
# distribution" panel read empty on 95% of calls and looked
|
# distribution" panel read empty on 95% of calls and looked
|
||||||
# broken. Name what actually decided it: recency on this
|
# broken. Name what actually decided it: recency on this
|
||||||
# talkgroup, with no content to check a fit against.
|
# talkgroup, with no content to check a fit against — or,
|
||||||
"corr_fit_signal": "thin_recency",
|
# when the same radio ID already touched a candidate
|
||||||
|
# (server-26#158), that identity match instead of a guess.
|
||||||
|
"corr_fit_signal": "thin_srcaddr_match" if srcaddr_matches else "thin_recency",
|
||||||
"corr_candidates": len(thin_pool),
|
"corr_candidates": len(thin_pool),
|
||||||
}
|
}
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -1009,14 +1090,14 @@ def _run_decision(ctx: dict) -> dict:
|
|||||||
candidate = tg_recent[0]
|
candidate = tg_recent[0]
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Correlator fast/single: call {call_id} vs incident {candidate['incident_id']} "
|
f"Correlator fast/single: call {call_id} vs incident {candidate['incident_id']} "
|
||||||
f"tg_name={effective_talkgroup_name!r} is_dispatch={is_dispatch} "
|
f"tg_name={effective_talkgroup_name!r} "
|
||||||
f"idle={round(_incident_idle_minutes(candidate, now), 1)}min "
|
f"idle={round(_incident_idle_minutes(candidate, now), 1)}min "
|
||||||
f"call_units={call_units} inc_units={candidate.get('units')} "
|
f"call_units={call_units} inc_units={candidate.get('units')} "
|
||||||
f"call_coords={'yes' if coords else 'no'} inc_coords={'yes' if candidate.get('location_coords') else 'no'}"
|
f"call_coords={'yes' if coords else 'no'} inc_coords={'yes' if candidate.get('location_coords') else 'no'}"
|
||||||
)
|
)
|
||||||
fit, fit_signal = _call_fits_incident(
|
fit, fit_signal = _call_fits_incident(
|
||||||
candidate, call_units, call_vehicles, coords,
|
candidate, call_units, call_vehicles, coords,
|
||||||
settings.location_proximity_km, is_dispatch=is_dispatch,
|
settings.location_proximity_km,
|
||||||
call_embedding=call_embedding, now=now,
|
call_embedding=call_embedding, now=now,
|
||||||
reassignment=reassignment,
|
reassignment=reassignment,
|
||||||
)
|
)
|
||||||
@@ -1026,13 +1107,12 @@ def _run_decision(ctx: dict) -> dict:
|
|||||||
"corr_path": "fast/single",
|
"corr_path": "fast/single",
|
||||||
"corr_incident_idle_min": round(_incident_idle_minutes(candidate, now), 1),
|
"corr_incident_idle_min": round(_incident_idle_minutes(candidate, now), 1),
|
||||||
"corr_fit_signal": fit_signal,
|
"corr_fit_signal": fit_signal,
|
||||||
"corr_is_dispatch": is_dispatch,
|
|
||||||
}
|
}
|
||||||
if fit_signal == "unit_overlap" and call_units:
|
if fit_signal == "unit_overlap" and call_units:
|
||||||
corr_debug["corr_matched_units"] = _matching_units(call_units, candidate.get("units"))
|
corr_debug["corr_matched_units"] = _matching_units(call_units, candidate.get("units"))
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Correlator fast-path: call {call_id} → {candidate['incident_id']} "
|
f"Correlator fast-path: call {call_id} → {candidate['incident_id']} "
|
||||||
f"(signal={fit_signal}, is_dispatch={is_dispatch})"
|
f"(signal={fit_signal})"
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -1048,14 +1128,14 @@ def _run_decision(ctx: dict) -> dict:
|
|||||||
# dispatch channel should create its own incident, not be force-merged.
|
# dispatch channel should create its own incident, not be force-merged.
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Correlator fast/disambig: call {call_id} vs incident {candidate['incident_id']} "
|
f"Correlator fast/disambig: call {call_id} vs incident {candidate['incident_id']} "
|
||||||
f"tg_name={effective_talkgroup_name!r} is_dispatch={is_dispatch} "
|
f"tg_name={effective_talkgroup_name!r} "
|
||||||
f"idle={round(_incident_idle_minutes(candidate, now), 1)}min "
|
f"idle={round(_incident_idle_minutes(candidate, now), 1)}min "
|
||||||
f"call_units={call_units} inc_units={candidate.get('units')} "
|
f"call_units={call_units} inc_units={candidate.get('units')} "
|
||||||
f"call_coords={'yes' if coords else 'no'} inc_coords={'yes' if candidate.get('location_coords') else 'no'}"
|
f"call_coords={'yes' if coords else 'no'} inc_coords={'yes' if candidate.get('location_coords') else 'no'}"
|
||||||
)
|
)
|
||||||
fit, fit_signal = _call_fits_incident(
|
fit, fit_signal = _call_fits_incident(
|
||||||
candidate, call_units, call_vehicles, coords,
|
candidate, call_units, call_vehicles, coords,
|
||||||
settings.location_proximity_km, is_dispatch=is_dispatch,
|
settings.location_proximity_km,
|
||||||
call_embedding=call_embedding, now=now,
|
call_embedding=call_embedding, now=now,
|
||||||
reassignment=reassignment,
|
reassignment=reassignment,
|
||||||
)
|
)
|
||||||
@@ -1066,7 +1146,6 @@ def _run_decision(ctx: dict) -> dict:
|
|||||||
"corr_incident_idle_min": round(_incident_idle_minutes(candidate, now), 1),
|
"corr_incident_idle_min": round(_incident_idle_minutes(candidate, now), 1),
|
||||||
"corr_candidates": len(tg_recent),
|
"corr_candidates": len(tg_recent),
|
||||||
"corr_fit_signal": fit_signal,
|
"corr_fit_signal": fit_signal,
|
||||||
"corr_is_dispatch": is_dispatch,
|
|
||||||
}
|
}
|
||||||
if fit_signal == "unit_overlap" and call_units:
|
if fit_signal == "unit_overlap" and call_units:
|
||||||
corr_debug["corr_matched_units"] = _matching_units(call_units, candidate.get("units"))
|
corr_debug["corr_matched_units"] = _matching_units(call_units, candidate.get("units"))
|
||||||
@@ -1147,6 +1226,10 @@ def _run_decision(ctx: dict) -> dict:
|
|||||||
|
|
||||||
# ── 2. Location path: proximity match (time-limited, cross-type) ─────────
|
# ── 2. Location path: proximity match (time-limited, cross-type) ─────────
|
||||||
if not matched_incident and coords:
|
if not matched_incident and coords:
|
||||||
|
# server-26#115 — score every in-radius candidate and link the NEAREST
|
||||||
|
# that carries corroboration, rather than whichever incident happened to
|
||||||
|
# come first in an unsorted `recent`.
|
||||||
|
loc_candidates: list[tuple] = []
|
||||||
for inc in recent:
|
for inc in recent:
|
||||||
inc_coords = inc.get("location_coords")
|
inc_coords = inc.get("location_coords")
|
||||||
if not inc_coords:
|
if not inc_coords:
|
||||||
@@ -1163,18 +1246,49 @@ def _run_decision(ctx: dict) -> dict:
|
|||||||
elapsed_min = max(_incident_idle_minutes(inc, now), 0.1)
|
elapsed_min = max(_incident_idle_minutes(inc, now), 0.1)
|
||||||
if (dist_km / elapsed_min) > _MAX_PURSUIT_SPEED_KM_PER_MIN:
|
if (dist_km / elapsed_min) > _MAX_PURSUIT_SPEED_KM_PER_MIN:
|
||||||
continue # implausible speed — skip this candidate
|
continue # implausible speed — skip this candidate
|
||||||
if dist_km <= radius:
|
if dist_km > radius:
|
||||||
|
continue
|
||||||
|
# server-26#115 — a bare sub-radius distance is not enough on its
|
||||||
|
# own. Require corroboration: unit overlap with the candidate, OR
|
||||||
|
# a much tighter proximity. Pursuit incidents keep their
|
||||||
|
# movement-speed-validated wide radius (they passed the speed
|
||||||
|
# check above), so they are exempt.
|
||||||
|
unit_overlap = bool(
|
||||||
|
_unit_keys(call_units) & _unit_keys(inc.get("units"))
|
||||||
|
)
|
||||||
|
tight_proximity = dist_km <= _LOCATION_TIGHT_PROXIMITY_KM
|
||||||
|
if not (is_pursuit_inc or unit_overlap or tight_proximity):
|
||||||
|
logger.info(
|
||||||
|
f"Correlator location-path skipped: call {call_id} vs "
|
||||||
|
f"{inc['incident_id']} — dist={dist_km:.2f}km within radius "
|
||||||
|
f"but no unit overlap and not tight-proximity "
|
||||||
|
f"(<= {_LOCATION_TIGHT_PROXIMITY_KM}km)"
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
loc_candidates.append((dist_km, unit_overlap, is_pursuit_inc, inc))
|
||||||
|
|
||||||
|
if loc_candidates:
|
||||||
|
loc_candidates.sort(key=lambda c: c[0])
|
||||||
|
dist_km, unit_overlap, is_pursuit_inc, inc = loc_candidates[0]
|
||||||
matched_incident = inc
|
matched_incident = inc
|
||||||
|
# Distinct from the fast path's "unit_overlap" so the admin
|
||||||
|
# corr_fit_signal histogram (routers/admin.py) does not merge a
|
||||||
|
# location-path link into the fast-path bucket (#35).
|
||||||
|
fit_signal = "location_unit_overlap" if unit_overlap else "location_proximity"
|
||||||
corr_debug = {
|
corr_debug = {
|
||||||
"corr_path": "location",
|
"corr_path": "location",
|
||||||
"corr_distance_km": round(dist_km, 3),
|
"corr_distance_km": round(dist_km, 3),
|
||||||
"corr_pursuit_mode": is_pursuit_inc,
|
"corr_pursuit_mode": is_pursuit_inc,
|
||||||
|
"corr_fit_signal": fit_signal,
|
||||||
}
|
}
|
||||||
|
if unit_overlap and call_units:
|
||||||
|
corr_debug["corr_matched_units"] = _matching_units(
|
||||||
|
call_units, inc.get("units")
|
||||||
|
)
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Correlator location-path: call {call_id} → {inc['incident_id']} "
|
f"Correlator location-path: call {call_id} → {inc['incident_id']} "
|
||||||
f"(dist={dist_km:.2f}km, pursuit={is_pursuit_inc})"
|
f"(dist={dist_km:.2f}km, pursuit={is_pursuit_inc}, signal={fit_signal})"
|
||||||
)
|
)
|
||||||
break
|
|
||||||
|
|
||||||
# ── 2.5. Cross-TG path: same department, overlapping units, moderate similarity ──
|
# ── 2.5. Cross-TG path: same department, overlapping units, moderate similarity ──
|
||||||
#
|
#
|
||||||
@@ -1311,7 +1425,7 @@ def _run_decision(ctx: dict) -> dict:
|
|||||||
# each. A vehicle, a geocode, or a tag means the extractor found something
|
# each. A vehicle, a geocode, or a tag means the extractor found something
|
||||||
# beyond who was speaking and where they stood.
|
# beyond who was speaking and where they stood.
|
||||||
if not resolved_type:
|
if not resolved_type:
|
||||||
has_substance = bool(call_vehicles or coords or tags)
|
has_substance = has_event_substance(ctx)
|
||||||
if call_severity in ("minor", "moderate", "major") or has_substance:
|
if call_severity in ("minor", "moderate", "major") or has_substance:
|
||||||
resolved_type = "other"
|
resolved_type = "other"
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -1335,12 +1449,68 @@ def _run_decision(ctx: dict) -> dict:
|
|||||||
# ─────────────────────────────────────────────────────────────────────────────
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
async def _apply_and_log(decision: dict, ctx: dict) -> Optional[str]:
|
async def _apply_and_log(decision: dict, ctx: dict) -> Optional[str]:
|
||||||
"""Commit a rules decision and persist the corr_debug fields to the call doc."""
|
"""
|
||||||
|
Commit a rules decision and persist the corr_debug fields to the call doc.
|
||||||
|
|
||||||
|
server-26#96: every scene of a multi-scene call reaches this function
|
||||||
|
independently (upload.py's ``for scene_index, scene in enumerate(scenes):``
|
||||||
|
loop → _correlate_with_consensus → preview_correlation/apply_correlation),
|
||||||
|
and every scene writes to the SAME call doc. Writing corr_debug only at
|
||||||
|
the flat top level meant scene 2's write silently clobbered scene 1's
|
||||||
|
corr_path/corr_consensus/etc — the call doc ended up describing a splice
|
||||||
|
of decisions, not any one of them.
|
||||||
|
|
||||||
|
Fix: write the flat fields exactly as before (kept for any reader that
|
||||||
|
doesn't know about `scenes` yet — last-scene-wins, same as pre-#96
|
||||||
|
behaviour, a safe backward-compatible default) AND additionally nest the
|
||||||
|
same corr_debug — plus this scene's own transcript and the incident_id it
|
||||||
|
resolved to — under scenes.<scene_index>. Firestore's
|
||||||
|
`DocumentReference.set(data, merge=True)` recursively merges nested map
|
||||||
|
fields by key. Verified against `internal/firestore.py`'s `doc_set`
|
||||||
|
wrapper (a straight `ref.set(data, merge=merge)` pass-through — no
|
||||||
|
`update()`, no read-modify-write, nothing that would change this) and
|
||||||
|
against the documented set-with-merge semantics; NOT exercised against a
|
||||||
|
live Firestore instance (no SDK available in the sandboxes this landed
|
||||||
|
from — review flagged this distinction explicitly). A write of
|
||||||
|
{"scenes": {"1": {...}}} merges into an existing
|
||||||
|
{"scenes": {"0": {...}}} to produce {"scenes": {"0": {...}, "1": {...}}}
|
||||||
|
rather than replacing the whole `scenes` map, so scene 0's and scene 1's
|
||||||
|
entries land side by side instead of colliding like the flat fields do.
|
||||||
|
`scene_index` defaults to 0 (see preview_correlation/correlate_call), so a
|
||||||
|
plain single-scene call still gets a `scenes` map — just with one entry,
|
||||||
|
equivalent to reading the flat fields today.
|
||||||
|
"""
|
||||||
incident_id = await _apply_decision(decision, ctx)
|
incident_id = await _apply_decision(decision, ctx)
|
||||||
|
if ctx.get("reassignment"):
|
||||||
|
await _release_reassigned_units(ctx, incident_id)
|
||||||
corr_debug = decision.get("corr_debug") or {}
|
corr_debug = decision.get("corr_debug") or {}
|
||||||
if corr_debug:
|
if corr_debug:
|
||||||
|
scene_index = ctx.get("scene_index", 0)
|
||||||
|
updates = dict(corr_debug)
|
||||||
|
updates["scenes"] = {
|
||||||
|
str(scene_index): {
|
||||||
|
"transcript": ctx.get("scene_transcript"),
|
||||||
|
"incident_id": incident_id,
|
||||||
|
"corr_debug": corr_debug,
|
||||||
|
# server-26#139: this scene's OWN extracted incident_type/
|
||||||
|
# severity, as read by _call_is_substanceless's ctx at
|
||||||
|
# decision time — not the call doc's flat top-level field,
|
||||||
|
# which is last-scene-wins (server-26#96) and was the reason
|
||||||
|
# #138's "type" veto couldn't be told apart from cross-scene
|
||||||
|
# contamination without re-guessing from a live dump.
|
||||||
|
# NOTE: unlike incident_type, call_severity is already
|
||||||
|
# coerced to "routine" when extraction emitted nothing
|
||||||
|
# (ctx build: `severity or "routine"`) — a scene reading
|
||||||
|
# "routine" here doesn't distinguish "extraction said
|
||||||
|
# routine" from "extraction said nothing". Don't split a
|
||||||
|
# severity veto the way #138 splits the type veto without
|
||||||
|
# accounting for that.
|
||||||
|
"incident_type": ctx.get("incident_type"),
|
||||||
|
"severity": ctx.get("call_severity"),
|
||||||
|
}
|
||||||
|
}
|
||||||
try:
|
try:
|
||||||
await fstore.doc_set("calls", ctx["call_id"], corr_debug)
|
await fstore.doc_set("calls", ctx["call_id"], updates)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Could not write corr_debug for call {ctx['call_id']}: {e}")
|
logger.warning(f"Could not write corr_debug for call {ctx['call_id']}: {e}")
|
||||||
return incident_id
|
return incident_id
|
||||||
@@ -1370,6 +1540,7 @@ async def _apply_decision(decision: dict, ctx: dict) -> Optional[str]:
|
|||||||
call_cleared = ctx["call_cleared"]
|
call_cleared = ctx["call_cleared"]
|
||||||
coords = ctx["coords"]
|
coords = ctx["coords"]
|
||||||
now = ctx["now"]
|
now = ctx["now"]
|
||||||
|
call_srcaddr = ctx.get("call_srcaddr")
|
||||||
incident_type = decision["incident_type"]
|
incident_type = decision["incident_type"]
|
||||||
|
|
||||||
if action == "link":
|
if action == "link":
|
||||||
@@ -1381,7 +1552,7 @@ async def _apply_decision(decision: dict, ctx: dict) -> Optional[str]:
|
|||||||
location, location_coords, call_units, call_vehicles, call_embedding, now,
|
location, location_coords, call_units, call_vehicles, call_embedding, now,
|
||||||
talkgroup_name=talkgroup_name, incident_type=incident_type,
|
talkgroup_name=talkgroup_name, incident_type=incident_type,
|
||||||
cleared_units=call_cleared, refresh_activity=not thin_link,
|
cleared_units=call_cleared, refresh_activity=not thin_link,
|
||||||
call_severity=call_severity,
|
call_severity=call_severity, call_srcaddr=call_srcaddr,
|
||||||
)
|
)
|
||||||
return matched_incident["incident_id"]
|
return matched_incident["incident_id"]
|
||||||
|
|
||||||
@@ -1413,6 +1584,7 @@ async def _apply_decision(decision: dict, ctx: dict) -> Optional[str]:
|
|||||||
call_id, org_id, incident_type, talkgroup_id, talkgroup_name, system_id,
|
call_id, org_id, incident_type, talkgroup_id, talkgroup_name, system_id,
|
||||||
tags, location, location_coords,
|
tags, location, location_coords,
|
||||||
call_units, call_vehicles, call_embedding, call_severity, now,
|
call_units, call_vehicles, call_embedding, call_severity, now,
|
||||||
|
call_srcaddr=call_srcaddr,
|
||||||
)
|
)
|
||||||
|
|
||||||
if existing_master_id:
|
if existing_master_id:
|
||||||
@@ -1458,6 +1630,7 @@ async def _apply_decision(decision: dict, ctx: dict) -> Optional[str]:
|
|||||||
call_id, org_id, incident_type, talkgroup_id, talkgroup_name, system_id,
|
call_id, org_id, incident_type, talkgroup_id, talkgroup_name, system_id,
|
||||||
tags, location, location_coords,
|
tags, location, location_coords,
|
||||||
call_units, call_vehicles, call_embedding, call_severity, now,
|
call_units, call_vehicles, call_embedding, call_severity, now,
|
||||||
|
call_srcaddr=call_srcaddr,
|
||||||
)
|
)
|
||||||
decision["corr_debug"]["corr_path"] = "new"
|
decision["corr_debug"]["corr_path"] = "new"
|
||||||
|
|
||||||
@@ -1571,7 +1744,6 @@ def _call_fits_incident(
|
|||||||
call_vehicles: list[str],
|
call_vehicles: list[str],
|
||||||
call_coords: Optional[dict],
|
call_coords: Optional[dict],
|
||||||
proximity_km: float,
|
proximity_km: float,
|
||||||
is_dispatch: bool = False,
|
|
||||||
call_embedding: Optional[list] = None,
|
call_embedding: Optional[list] = None,
|
||||||
now: Optional[datetime] = None,
|
now: Optional[datetime] = None,
|
||||||
reassignment: bool = False,
|
reassignment: bool = False,
|
||||||
@@ -1581,48 +1753,24 @@ def _call_fits_incident(
|
|||||||
the incident; signal names the specific evidence that drove the decision.
|
the incident; signal names the specific evidence that drove the decision.
|
||||||
|
|
||||||
fits=True signals: "unit_overlap" | "vehicle_overlap" | "location_proximity"
|
fits=True signals: "unit_overlap" | "vehicle_overlap" | "location_proximity"
|
||||||
| "time_fallback" | "tactical_default"
|
|
||||||
fits=False signals: "unit_loc_conflict" | "content_divergence"
|
fits=False signals: "unit_loc_conflict" | "content_divergence"
|
||||||
| "location_conflict" | "no_signal" | "tactical_idle"
|
| "location_conflict" | "no_signal"
|
||||||
|
|
||||||
Original docstring (logic unchanged):
|
Evaluation order:
|
||||||
Return True if this call plausibly belongs to the given incident.
|
1. Unit overlap. Same officer = same call. Also runs a location-conflict
|
||||||
|
guard: geocoded on both sides and clearly different → the officer has
|
||||||
Evaluation order for dispatch channels (is_dispatch=True):
|
moved to a new scene, false positive. No geocode on the call (or on
|
||||||
─────────────────────────────────────────────────────────
|
the incident, with a call geocode) AND the incident is old (≥ 15 min)
|
||||||
1. Unit overlap
|
→ content divergence as a location proxy (embedding similarity < 0.82
|
||||||
Same officer = same call. On dispatch channels, also run a location
|
→ different scene). Skipped for recent incidents (< 15 min) — an
|
||||||
conflict guard: if both sides carry geocoded coords and they differ
|
update without re-stating the address is normal.
|
||||||
significantly, the officer has moved to a new scene and the unit match
|
2. Vehicle overlap → True.
|
||||||
is a false positive.
|
3. Location proximity. Both geocoded and close → True; far apart with no
|
||||||
When the call has NO geocoded coordinates AND the incident is old
|
other positive signal → False.
|
||||||
(≥ 15 min), use content divergence as a location proxy: an officer at
|
4. No positive signal at all → False. A shared channel must not absorb
|
||||||
a genuinely different scene will be talking about clearly different
|
calls by default (server-26#134 — this used to default True within
|
||||||
things. For recent incidents (< 15 min) we skip this proxy — the
|
20 min on any channel not name-matched as "dispatch"; a talkgroup
|
||||||
officer may simply be giving an update without mentioning the address.
|
named tac/tactical is no less scrutinized in practice than any other).
|
||||||
|
|
||||||
2. Vehicle overlap → True
|
|
||||||
|
|
||||||
3. Location proximity
|
|
||||||
Both sides geocoded and close → True.
|
|
||||||
Both sides geocoded and far apart (no other positive signal) → False.
|
|
||||||
|
|
||||||
4. No positive signals fired → dispatch fallback
|
|
||||||
a. Conversational continuity: idle < 2 min → True.
|
|
||||||
A call arriving within 2 minutes of the last incident activity almost
|
|
||||||
certainly belongs to the same dispatch thread. "Baker, head over
|
|
||||||
there too" or "copy that" carries no incident-specific vocabulary but
|
|
||||||
is unambiguously a response to what was just said. We do not require
|
|
||||||
embedding similarity here — embeddings capture word meaning, not
|
|
||||||
conversational context, and short operational messages will always
|
|
||||||
have lower similarity than the incident's accumulated content.
|
|
||||||
b. Older incident, no positive signals → False.
|
|
||||||
A shared dispatch channel must not absorb calls by default.
|
|
||||||
|
|
||||||
Tactical / working channel (is_dispatch=False):
|
|
||||||
────────────────────────────────────────────────
|
|
||||||
Channel is dedicated to one scene. No evidence of separation ≈ same call.
|
|
||||||
Default → True.
|
|
||||||
|
|
||||||
Thin calls (no units/vehicles/coords) never reach this function —
|
Thin calls (no units/vehicles/coords) never reach this function —
|
||||||
they are intercepted before it in correlate_call.
|
they are intercepted before it in correlate_call.
|
||||||
@@ -1631,7 +1779,7 @@ def _call_fits_incident(
|
|||||||
# signed value: the re-correlation sweep anchors `now` to the call's own
|
# signed value: the re-correlation sweep anchors `now` to the call's own
|
||||||
# started_at, which can be earlier than the incident's last activity and
|
# started_at, which can be earlier than the incident's last activity and
|
||||||
# send the signed value negative — silently defeating every `idle_min`
|
# send the signed value negative — silently defeating every `idle_min`
|
||||||
# gate below (content-divergence veto and the tactical default alike).
|
# gate below, the content-divergence veto included.
|
||||||
# See `_idle_gate_minutes` docstring. The signed value is reported to
|
# See `_idle_gate_minutes` docstring. The signed value is reported to
|
||||||
# callers separately as `corr_incident_idle_min` (they compute it via
|
# callers separately as `corr_incident_idle_min` (they compute it via
|
||||||
# `_incident_idle_minutes` themselves) — nothing here needs it.
|
# `_incident_idle_minutes` themselves) — nothing here needs it.
|
||||||
@@ -1642,7 +1790,6 @@ def _call_fits_incident(
|
|||||||
inc_units = _unit_keys(inc.get("units"))
|
inc_units = _unit_keys(inc.get("units"))
|
||||||
matched_units = _matching_units(call_units, inc.get("units"))
|
matched_units = _matching_units(call_units, inc.get("units"))
|
||||||
if matched_units:
|
if matched_units:
|
||||||
if is_dispatch:
|
|
||||||
if call_coords:
|
if call_coords:
|
||||||
# Hard location conflict: geocoded on both sides and clearly different.
|
# Hard location conflict: geocoded on both sides and clearly different.
|
||||||
inc_coords_u = inc.get("location_coords")
|
inc_coords_u = inc.get("location_coords")
|
||||||
@@ -1679,7 +1826,7 @@ def _call_fits_incident(
|
|||||||
if sim < 0.82:
|
if sim < 0.82:
|
||||||
logger.info(f" fits[{inc_id}]: unit_overlap({matched_units}) but content_divergence sim={sim:.3f} → content_divergence")
|
logger.info(f" fits[{inc_id}]: unit_overlap({matched_units}) but content_divergence sim={sim:.3f} → content_divergence")
|
||||||
return False, "content_divergence"
|
return False, "content_divergence"
|
||||||
logger.info(f" fits[{inc_id}]: unit_overlap matched={matched_units} is_dispatch={is_dispatch} → unit_overlap")
|
logger.info(f" fits[{inc_id}]: unit_overlap matched={matched_units} → unit_overlap")
|
||||||
return True, "unit_overlap"
|
return True, "unit_overlap"
|
||||||
|
|
||||||
# ── 2. Vehicle overlap ────────────────────────────────────────────────────
|
# ── 2. Vehicle overlap ────────────────────────────────────────────────────
|
||||||
@@ -1733,29 +1880,84 @@ def _call_fits_incident(
|
|||||||
return False, "location_conflict"
|
return False, "location_conflict"
|
||||||
|
|
||||||
# ── 4. No positive signals ────────────────────────────────────────────────
|
# ── 4. No positive signals ────────────────────────────────────────────────
|
||||||
|
# Requires at least one positive signal (unit, vehicle, or location match).
|
||||||
|
# A substantive call with no matching signals is more likely a separate
|
||||||
|
# incident than a follow-up — two dispatches can arrive within seconds of
|
||||||
|
# each other on a busy channel. Content-free thin calls are handled before
|
||||||
|
# this function via the thin path in correlate_call, with a tighter
|
||||||
|
# 30-second recency window.
|
||||||
logger.info(
|
logger.info(
|
||||||
f" fits[{inc_id}]: no positive signal — is_dispatch={is_dispatch} idle={idle_min:.1f}min "
|
f" fits[{inc_id}]: no positive signal — idle={idle_min:.1f}min "
|
||||||
f"inc_units={list(inc_units)} call_units={call_units} "
|
f"inc_units={list(inc_units)} call_units={call_units} "
|
||||||
f"inc_vehicles={list(inc_vehicles)} call_vehicles={call_vehicles} "
|
f"inc_vehicles={list(inc_vehicles)} call_vehicles={call_vehicles} "
|
||||||
f"call_coords={call_coords is not None} inc_coords={inc_coords is not None}"
|
f"call_coords={call_coords is not None} inc_coords={inc_coords is not None}"
|
||||||
)
|
)
|
||||||
if is_dispatch:
|
|
||||||
# Dispatch channels require at least one positive signal (unit, vehicle,
|
|
||||||
# or location match). A substantive call with no matching signals is more
|
|
||||||
# likely a separate incident than a follow-up to the current one — two
|
|
||||||
# dispatches can arrive within seconds of each other on a busy channel.
|
|
||||||
# Content-free thin calls are handled before this function via the thin
|
|
||||||
# path in correlate_call, with a tighter 30-second recency window.
|
|
||||||
return False, "no_signal"
|
return False, "no_signal"
|
||||||
|
|
||||||
# Tactical channel: one scene per channel.
|
|
||||||
# Within 20 min of the last incident activity, link by default — same
|
def _apply_unit_clearance(inc: dict, cleared: list[str]) -> tuple[list[str], list[str], bool]:
|
||||||
# working channel almost certainly means same scene.
|
"""
|
||||||
# After 20 min of silence, require at least one positive signal; the same
|
Merge `cleared` into inc's units_active/units_cleared. Shared by
|
||||||
# frequency can be reused for a new unrelated incident later in the shift.
|
_update_incident (explicit 10-8/back-in-service extraction) and
|
||||||
if idle_min < 20.0:
|
_release_reassigned_units (server-26#<pending> pattern B: a unit accepting
|
||||||
return True, "tactical_default"
|
a new dispatch, reassignment=True, is real-world evidence they're off
|
||||||
return False, "tactical_idle"
|
their prior call even without an explicit clearance phrase).
|
||||||
|
|
||||||
|
Returns (units_active, units_cleared, auto_resolved) — auto_resolved is
|
||||||
|
True when every tracked unit has now cleared, matching the resolve gate
|
||||||
|
at the bottom of _update_incident.
|
||||||
|
"""
|
||||||
|
units_active = list(inc.get("units_active") or [])
|
||||||
|
units_cleared = list(inc.get("units_cleared") or [])
|
||||||
|
for u in cleared:
|
||||||
|
if u in units_active:
|
||||||
|
units_active.remove(u)
|
||||||
|
if u not in units_cleared:
|
||||||
|
units_cleared.append(u)
|
||||||
|
auto_resolved = bool(units_cleared) and not units_active
|
||||||
|
return units_active, units_cleared, auto_resolved
|
||||||
|
|
||||||
|
|
||||||
|
async def _release_reassigned_units(ctx: dict, exclude_incident_id: Optional[str]) -> None:
|
||||||
|
"""
|
||||||
|
server-26#<pending>: reassignment=True means a unit is accepting a NEW
|
||||||
|
dispatch — real-world evidence they're off whatever they were on before,
|
||||||
|
even when they never say an explicit 10-8/clear phrase (dispatch: "are
|
||||||
|
you able to clear and take a run at X" / unit: "10-4" carries no
|
||||||
|
self-reported clearance language intelligence.py's cleared_units
|
||||||
|
extraction looks for). Without this, that unit's prior incident is only
|
||||||
|
ever closed by the 90-minute idle sweep, not a real clear.
|
||||||
|
|
||||||
|
Scoped to OTHER active incidents (exclude_incident_id keeps this call's
|
||||||
|
own outcome untouched) with unit overlap in units_active — mirrors the
|
||||||
|
unit-continuity candidate scan at :1142 but releases instead of links.
|
||||||
|
"""
|
||||||
|
call_units = ctx.get("call_units")
|
||||||
|
if not call_units:
|
||||||
|
return
|
||||||
|
system_id = ctx.get("system_id")
|
||||||
|
now = ctx["now"]
|
||||||
|
unit_set = _unit_keys(call_units)
|
||||||
|
for inc in ctx.get("all_active") or []:
|
||||||
|
if inc.get("incident_id") == exclude_incident_id:
|
||||||
|
continue
|
||||||
|
if system_id and system_id not in (inc.get("system_ids") or []):
|
||||||
|
continue
|
||||||
|
matched = [u for u in (inc.get("units_active") or []) if _normalize_unit(u) in unit_set]
|
||||||
|
if not matched:
|
||||||
|
continue
|
||||||
|
units_active, units_cleared, auto_resolved = _apply_unit_clearance(inc, matched)
|
||||||
|
updates = {"units_active": units_active, "units_cleared": units_cleared}
|
||||||
|
if auto_resolved:
|
||||||
|
updates["status"] = "resolved"
|
||||||
|
updates["resolved_at"] = now.isoformat()
|
||||||
|
await fstore.doc_set("incidents", inc["incident_id"], updates)
|
||||||
|
logger.info(
|
||||||
|
f"Correlator: reassignment released unit(s) {matched} from incident "
|
||||||
|
f"{inc['incident_id']}" + (" (auto-resolved)" if auto_resolved else "")
|
||||||
|
)
|
||||||
|
if auto_resolved:
|
||||||
|
await maybe_resolve_parent(inc["incident_id"])
|
||||||
|
|
||||||
|
|
||||||
async def _update_incident(
|
async def _update_incident(
|
||||||
@@ -1775,6 +1977,7 @@ async def _update_incident(
|
|||||||
cleared_units: Optional[list[str]] = None,
|
cleared_units: Optional[list[str]] = None,
|
||||||
refresh_activity: bool = True,
|
refresh_activity: bool = True,
|
||||||
call_severity: Optional[str] = None,
|
call_severity: Optional[str] = None,
|
||||||
|
call_srcaddr: Optional[str] = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
incident_id = inc["incident_id"]
|
incident_id = inc["incident_id"]
|
||||||
|
|
||||||
@@ -1793,6 +1996,12 @@ async def _update_incident(
|
|||||||
merged_tags = list(dict.fromkeys((inc.get("tags") or []) + tags))
|
merged_tags = list(dict.fromkeys((inc.get("tags") or []) + tags))
|
||||||
merged_units = list(dict.fromkeys((inc.get("units") or []) + call_units))
|
merged_units = list(dict.fromkeys((inc.get("units") or []) + call_units))
|
||||||
merged_vehicles = list(dict.fromkeys((inc.get("vehicles") or []) + call_vehicles))
|
merged_vehicles = list(dict.fromkeys((inc.get("vehicles") or []) + call_vehicles))
|
||||||
|
# server-26#158: accumulate every radio ID that has transmitted on this
|
||||||
|
# incident, so a later thin call from the same radio can identity-match
|
||||||
|
# instead of guessing off recency alone.
|
||||||
|
merged_srcaddrs = list(dict.fromkeys(
|
||||||
|
(inc.get("srcaddrs") or []) + ([call_srcaddr] if call_srcaddr else [])
|
||||||
|
))
|
||||||
|
|
||||||
# Unit activity tracking: units_active / units_cleared
|
# Unit activity tracking: units_active / units_cleared
|
||||||
# units_active = units currently on scene; units_cleared = units back in service
|
# units_active = units currently on scene; units_cleared = units back in service
|
||||||
@@ -1801,11 +2010,8 @@ async def _update_incident(
|
|||||||
for u in call_units:
|
for u in call_units:
|
||||||
if u not in units_cleared and u not in units_active:
|
if u not in units_cleared and u not in units_active:
|
||||||
units_active.append(u)
|
units_active.append(u)
|
||||||
for u in (cleared_units or []):
|
inc_with_active_update = {**inc, "units_active": units_active, "units_cleared": units_cleared}
|
||||||
if u in units_active:
|
units_active, units_cleared, _ = _apply_unit_clearance(inc_with_active_update, cleared_units or [])
|
||||||
units_active.remove(u)
|
|
||||||
if u not in units_cleared:
|
|
||||||
units_cleared.append(u)
|
|
||||||
|
|
||||||
# The incident's label and its pin are resolved together, as one value.
|
# The incident's label and its pin are resolved together, as one value.
|
||||||
location = clean_location(location)
|
location = clean_location(location)
|
||||||
@@ -1826,6 +2032,7 @@ async def _update_incident(
|
|||||||
"tags": merged_tags,
|
"tags": merged_tags,
|
||||||
"units": merged_units,
|
"units": merged_units,
|
||||||
"vehicles": merged_vehicles,
|
"vehicles": merged_vehicles,
|
||||||
|
"srcaddrs": merged_srcaddrs,
|
||||||
"units_active": units_active,
|
"units_active": units_active,
|
||||||
"units_cleared": units_cleared,
|
"units_cleared": units_cleared,
|
||||||
"location_mentions": location_mentions,
|
"location_mentions": location_mentions,
|
||||||
@@ -1892,6 +2099,7 @@ async def _create_incident(
|
|||||||
call_embedding: Optional[list],
|
call_embedding: Optional[list],
|
||||||
call_severity: str,
|
call_severity: str,
|
||||||
now: datetime,
|
now: datetime,
|
||||||
|
call_srcaddr: Optional[str] = None,
|
||||||
) -> str:
|
) -> str:
|
||||||
incident_id = str(uuid.uuid4())
|
incident_id = str(uuid.uuid4())
|
||||||
tg_label = (
|
tg_label = (
|
||||||
@@ -1935,6 +2143,7 @@ async def _create_incident(
|
|||||||
"units_active": list(call_units),
|
"units_active": list(call_units),
|
||||||
"units_cleared": [],
|
"units_cleared": [],
|
||||||
"vehicles": call_vehicles,
|
"vehicles": call_vehicles,
|
||||||
|
"srcaddrs": [call_srcaddr] if call_srcaddr else [],
|
||||||
"severity": call_severity,
|
"severity": call_severity,
|
||||||
"summary": None,
|
"summary": None,
|
||||||
"summary_stale": True,
|
"summary_stale": True,
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from typing import Optional
|
|||||||
from app.internal.logger import logger
|
from app.internal.logger import logger
|
||||||
from app.internal import firestore as fstore
|
from app.internal import firestore as fstore
|
||||||
from app.internal import area_context
|
from app.internal import area_context
|
||||||
|
from app.internal.chatter_classifier import classify_chatter
|
||||||
# Location validity is defined once, by the module that owns the incident's
|
# Location validity is defined once, by the module that owns the incident's
|
||||||
# location/pin invariant. incident_correlator does not import this module, so
|
# location/pin invariant. incident_correlator does not import this module, so
|
||||||
# this is not a cycle.
|
# this is not a cycle.
|
||||||
@@ -63,9 +64,9 @@ Response format — a JSON object with a "scenes" array. Each scene:
|
|||||||
Rules:
|
Rules:
|
||||||
- location: prefer intersections > addresses > mile markers > route+town > route alone > town alone. Dispatch-provided addresses take priority over unit-reported positions. Empty string if none.
|
- location: prefer intersections > addresses > mile markers > route+town > route alone > town alone. Dispatch-provided addresses take priority over unit-reported positions. Empty string if none.
|
||||||
- tags: describe WHAT happened, not WHERE. Specific, lowercase, hyphenated. Do not use location names, road names, talkgroup names, or place names as tags (wrong: "lower-macy's", "canvas-route-6", "route-202"; right: "suspect-search", "shoplifting", "vehicle-pursuit"). Do not repeat incident_type as a tag.
|
- tags: describe WHAT happened, not WHERE. Specific, lowercase, hyphenated. Do not use location names, road names, talkgroup names, or place names as tags (wrong: "lower-macy's", "canvas-route-6", "route-202"; right: "suspect-search", "shoplifting", "vehicle-pursuit"). Do not repeat incident_type as a tag.
|
||||||
- units: ONLY identifiers that appear verbatim in the transcript. Use speaker role inference to distinguish units being dispatched from units acknowledging — both should be included. Never infer or guess unit IDs not present in the text.
|
- units: ONLY identifiers that appear verbatim in the transcript. Use speaker role inference to distinguish units being dispatched from units acknowledging — both should be included. Never infer or guess unit IDs not present in the text. If a unit ID format is given below, use it to recognise a unit spoken in a shortened or partial form (e.g. just the phonetic name alone) as the same unit — but still only extract what is actually said, never fabricate the full form.
|
||||||
- Do not invent details not present in the transcript.
|
- Do not invent details not present in the transcript.
|
||||||
- incident_type: let the talkgroup channel be your primary signal. Use "fire" ONLY if the talkgroup is clearly a fire/rescue channel OR the transcript explicitly describes active fire, smoke, flames, or structure fire activation. Police or EMS referencing a fire scene → use "police" or "ems". When the channel is a police channel and nothing in the transcript contradicts it, return "police" — do NOT fall back to "other" merely because the transmission is administrative. Reserve "other" for traffic that genuinely belongs to no emergency service (rail operations, public works, utility coordination). Reserve "unknown" for transcripts too garbled to place at all.
|
- incident_type: FIRST decide whether this transmission has any incident behind it at all, using the same bar as the "routine" severity rule below — pure administrative/status traffic with nothing describable happening: post/unit check-ins, roll call, bare acknowledgements ("10-4", "copy", "received"), records/report exchanges, "show me admin"/"show me available", a status ten-code with no event attached. If it is administrative/status-only, return "unknown" — this applies on EVERY channel, including a police channel; do not let the channel default override it (server-26#138: forcing a channel default onto content-free chatter is what let radio housekeeping open incidents). Only once real event content is present, let the talkgroup channel be your primary signal for WHICH type. Use "fire" ONLY if the talkgroup is clearly a fire/rescue channel OR the transcript explicitly describes active fire, smoke, flames, or structure fire activation. Police or EMS referencing a fire scene → use "police" or "ems". When the channel is a police channel, a real event is present, and nothing in the transcript contradicts it, return "police". Reserve "other" for a real event that genuinely belongs to no emergency service (rail operations, public works, utility coordination) — not for administrative chatter, which is "unknown" per above regardless of channel. Also reserve "unknown" for transcripts too garbled to place at all.
|
||||||
- severity: ALWAYS return one of the four values. Judge the underlying event, not how dramatic the words sound.
|
- severity: ALWAYS return one of the four values. Judge the underlying event, not how dramatic the words sound.
|
||||||
"routine" — administrative/status traffic with no incident behind it: mileage and transport logging, radio checks, acknowledgements, shift changes, track block/power requests, records lookups.
|
"routine" — administrative/status traffic with no incident behind it: mileage and transport logging, radio checks, acknowledgements, shift changes, track block/power requests, records lookups.
|
||||||
"minor" — a real but low-stakes call: lift assist, parking complaint, past-tense larceny report, noise complaint, welfare check.
|
"minor" — a real but low-stakes call: lift assist, parking complaint, past-tense larceny report, noise complaint, welfare check.
|
||||||
@@ -73,18 +74,15 @@ Rules:
|
|||||||
"major" — life safety or major property loss: structure fire, vehicle pursuit, shots fired, entrapment, cardiac arrest, officer needing assistance.
|
"major" — life safety or major property loss: structure fire, vehicle pursuit, shots fired, entrapment, cardiac arrest, officer needing assistance.
|
||||||
- ten_codes: interpret radio codes using the department reference provided below. Do not guess codes not listed.
|
- ten_codes: interpret radio codes using the department reference provided below. Do not guess codes not listed.
|
||||||
- resolved: true only when the scene explicitly signals "Code 4", "all clear", "10-42", "in custody", "patient transported", "fire out", "GOA", "negative contact", "scene clear".
|
- resolved: true only when the scene explicitly signals "Code 4", "all clear", "10-42", "in custody", "patient transported", "fire out", "GOA", "negative contact", "scene clear".
|
||||||
- cleared_units: only include units that explicitly stated their own back-in-service status in this recording (e.g. "Unit 7, 10-8", "Baker-1 available", "E-14 back in service", or the department ten-code for available/back-in-service listed above). Silence or absence of a unit is NOT clearance. A scene-wide Code 4 belongs in resolved=true, not here — cleared_units is for individual unit availability signals only.
|
- cleared_units: include a unit whose back-in-service/available status is stated in this recording — either the unit self-reporting (e.g. "Unit 7, 10-8", "Baker-1 available", "E-14 back in service", or the department ten-code for available/back-in-service listed above) OR dispatch confirming that SPECIFIC unit's status back to them (e.g. the unit asks "how do you show me" and dispatch replies "showing you available" / "in service"). The unit ID must be identifiable either way — a bare "clear" or "10-8" with no unit attached to it is NOT clearance; do not guess which unit said it. Silence or absence of a unit is NOT clearance. A scene-wide Code 4 belongs in resolved=true, not here — cleared_units is for individual unit availability signals only.
|
||||||
- reassignment: only true when a unit is explicitly being pulled to a completely new call or location. A unit going en route to their first dispatch is NOT a reassignment. Routine status updates, acknowledgements, and scene updates are NOT reassignments.
|
- reassignment: only true when a unit is explicitly being pulled to a completely new call or location. A unit going en route to their first dispatch is NOT a reassignment. Routine status updates, acknowledgements, and scene updates are NOT reassignments.
|
||||||
|
|
||||||
System: {system_id}
|
System: {system_id}
|
||||||
Talkgroup: {talkgroup_name}
|
Talkgroup: {talkgroup_name}
|
||||||
{ten_codes_block}{vocabulary_block}{transcript_block}"""
|
{ten_codes_block}{vocabulary_block}{unit_format_block}{transcript_block}"""
|
||||||
|
|
||||||
# The incident_type enum offered to the model in EXTRACTION_PROMPT. Kept here
|
# "unknown" is deliberately absent — normalises to None, which is what lets
|
||||||
# rather than only in the prompt so a model that invents a value cannot write it
|
# the creation gate veto a content-free call (server-26#138).
|
||||||
# into incident.type. "unknown" is deliberately absent — it is a real answer
|
|
||||||
# from the model but not a usable type, and is normalised to None alongside
|
|
||||||
# anything unrecognised.
|
|
||||||
_VALID_INCIDENT_TYPES = frozenset({"fire", "ems", "police", "accident", "other"})
|
_VALID_INCIDENT_TYPES = frozenset({"fire", "ems", "police", "accident", "other"})
|
||||||
|
|
||||||
# Geographic bias radius for geocoding — half-width in degrees (~55 km)
|
# Geographic bias radius for geocoding — half-width in degrees (~55 km)
|
||||||
@@ -155,6 +153,23 @@ def _build_ten_codes_block(ten_codes: dict[str, str]) -> str:
|
|||||||
return f"Department ten-codes:\n{lines}\n\n"
|
return f"Department ten-codes:\n{lines}\n\n"
|
||||||
|
|
||||||
|
|
||||||
|
def _build_unit_format_block(unit_format_hint: Optional[str]) -> str:
|
||||||
|
"""
|
||||||
|
server-26#<pending> — unit ID formats vary per department (e.g. Yorktown:
|
||||||
|
"<district>-<phonetic>", "5-David", sometimes spoken as bare "David";
|
||||||
|
County: "<location>-<number>", "SAM-1", "airport-3", "parks-4") with no
|
||||||
|
shared pattern across systems. Without a per-system hint, the model has
|
||||||
|
no way to recognise a unit ID it hasn't seen phrased that way before, and
|
||||||
|
that failure compounds into cleared_units and reassignment detection,
|
||||||
|
both of which depend on first recognising which token IS the unit.
|
||||||
|
Owner-authored free text per system (systems/{id}.unit_format_hint via
|
||||||
|
PUT /systems/{id}/unit-format) — no auto-induction yet.
|
||||||
|
"""
|
||||||
|
if not unit_format_hint:
|
||||||
|
return ""
|
||||||
|
return f"This system's unit ID format: {unit_format_hint}\n\n"
|
||||||
|
|
||||||
|
|
||||||
async def extract_scenes(
|
async def extract_scenes(
|
||||||
call_id: str,
|
call_id: str,
|
||||||
transcript: str,
|
transcript: str,
|
||||||
@@ -172,7 +187,7 @@ async def extract_scenes(
|
|||||||
|
|
||||||
Each scene dict contains:
|
Each scene dict contains:
|
||||||
tags, incident_type, location, location_coords, resolved,
|
tags, incident_type, location, location_coords, resolved,
|
||||||
severity, vehicles, units, transcript_corrected,
|
severity, vehicles, units, transcript, transcript_corrected,
|
||||||
segment_indices, embedding
|
segment_indices, embedding
|
||||||
|
|
||||||
Side-effect: updates calls/{call_id} in Firestore with merged tags,
|
Side-effect: updates calls/{call_id} in Firestore with merged tags,
|
||||||
@@ -181,12 +196,15 @@ async def extract_scenes(
|
|||||||
"""
|
"""
|
||||||
vocabulary: list[str] = []
|
vocabulary: list[str] = []
|
||||||
ten_codes: dict[str, str] = {}
|
ten_codes: dict[str, str] = {}
|
||||||
|
unit_format_hint: str = ""
|
||||||
if system_id:
|
if system_id:
|
||||||
# Single cached read — vocabulary and ten_codes live on the same document.
|
# Single cached read — vocabulary, ten_codes and unit_format_hint all
|
||||||
|
# live on the same document.
|
||||||
system_doc = await fstore.doc_get_cached("systems", system_id)
|
system_doc = await fstore.doc_get_cached("systems", system_id)
|
||||||
if system_doc:
|
if system_doc:
|
||||||
vocabulary = system_doc.get("vocabulary") or []
|
vocabulary = system_doc.get("vocabulary") or []
|
||||||
ten_codes = system_doc.get("ten_codes") or {}
|
ten_codes = system_doc.get("ten_codes") or {}
|
||||||
|
unit_format_hint = system_doc.get("unit_format_hint") or ""
|
||||||
|
|
||||||
if _is_garbage_transcript(transcript):
|
if _is_garbage_transcript(transcript):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
@@ -199,6 +217,28 @@ async def extract_scenes(
|
|||||||
pass
|
pass
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
# server-26#127 — SHADOW MODE ONLY. Computes whether this transcript looks
|
||||||
|
# like non-event radio housekeeping (roll call, bare 10-4/10-8/98
|
||||||
|
# acknowledgements, unit check-ins) and records the verdict on the call
|
||||||
|
# doc, but does NOT skip extraction anywhere below — every path runs
|
||||||
|
# exactly as it did before this landed. Deliberately ahead of the ≤5-word
|
||||||
|
# skip: most bare acknowledgements ARE ≤5 words, and the first pass of
|
||||||
|
# this feature put the classifier after that return, so it never saw the
|
||||||
|
# bulk of its own target population — a review backtest against three
|
||||||
|
# live dumps found 82% of what it would have flagged already exits above
|
||||||
|
# as transcript_too_short, meaning a shadow-mode window would have shown
|
||||||
|
# roughly a fifth of the real catch rate. Computing it once, here, and
|
||||||
|
# folding the result into whichever skip/continue path runs below fixes
|
||||||
|
# that without adding a second Firestore write.
|
||||||
|
# TODO(server-26#127): flip this from shadow to live (skip extraction and
|
||||||
|
# write skip_reason="non_event_chatter" instead of just recording the
|
||||||
|
# verdict) once a live shadow-mode window confirms 0 false positives on
|
||||||
|
# real production traffic — pay particular attention to whole-transcript
|
||||||
|
# vs contains-anywhere matching for "roll call" and to digit-hyphen street
|
||||||
|
# addresses (e.g. "72-Holland"), both flagged as classifier risks that the
|
||||||
|
# dump backtest could not surface on its own.
|
||||||
|
chatter_is_chatter, chatter_reason = classify_chatter(transcript)
|
||||||
|
|
||||||
# Transcripts with ≤5 words carry no extractable intelligence — GPT hallucinates
|
# Transcripts with ≤5 words carry no extractable intelligence — GPT hallucinates
|
||||||
# units and tags from thin context (e.g. "Main Lot", "10-4", "David").
|
# units and tags from thin context (e.g. "Main Lot", "10-4", "David").
|
||||||
if len(transcript.split()) <= 5:
|
if len(transcript.split()) <= 5:
|
||||||
@@ -213,14 +253,25 @@ async def extract_scenes(
|
|||||||
await fstore.doc_set("calls", call_id, {
|
await fstore.doc_set("calls", call_id, {
|
||||||
"skip_reason": "transcript_too_short",
|
"skip_reason": "transcript_too_short",
|
||||||
"severity": "routine",
|
"severity": "routine",
|
||||||
|
"chatter_classifier_verdict": chatter_is_chatter,
|
||||||
|
"chatter_classifier_reason": chatter_reason,
|
||||||
})
|
})
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
try:
|
||||||
|
await fstore.doc_set("calls", call_id, {
|
||||||
|
"chatter_classifier_verdict": chatter_is_chatter,
|
||||||
|
"chatter_classifier_reason": chatter_reason,
|
||||||
|
})
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
raw_scenes: list[dict] = await asyncio.to_thread(
|
raw_scenes: list[dict] = await asyncio.to_thread(
|
||||||
_sync_extract,
|
_sync_extract,
|
||||||
transcript, talkgroup_name, talkgroup_id, system_id, segments, vocabulary, ten_codes,
|
transcript, talkgroup_name, talkgroup_id, system_id, segments, vocabulary, ten_codes,
|
||||||
|
unit_format_hint,
|
||||||
)
|
)
|
||||||
|
|
||||||
if not raw_scenes:
|
if not raw_scenes:
|
||||||
@@ -317,18 +368,20 @@ async def extract_scenes(
|
|||||||
# the country.
|
# the country.
|
||||||
location_coords: Optional[dict] = None
|
location_coords: Optional[dict] = None
|
||||||
if location:
|
if location:
|
||||||
parts = [location]
|
node_state, node_county = "", ""
|
||||||
if tg_area.get("municipality") or tg_area.get("county") or tg_area.get("state"):
|
if not area_context.has_place(tg_area) and node_id and node_lat is not None and node_lon is not None:
|
||||||
parts += [tg_area[f] for f in area_context.PLACE_FIELDS if tg_area.get(f)]
|
# Only worth the (cached-after-first-call) reverse-geocode
|
||||||
elif node_lat is not None and node_lon is not None:
|
# when nothing better already describes this talkgroup.
|
||||||
muni = _municipality_from_tg(talkgroup_name)
|
node_state = await _get_node_state(node_id, node_lat, node_lon)
|
||||||
state = await _get_node_state(node_id or "", node_lat, node_lon) if node_id else ""
|
node_county = _node_county_cache.get(node_id) or ""
|
||||||
county = _node_county_cache.get(node_id or "") if node_id else ""
|
parts, tg_named_region = _location_query_parts(
|
||||||
parts += [p for p in (muni, county, state) if p]
|
location, tg_area, talkgroup_name, node_state, node_county,
|
||||||
|
)
|
||||||
query = ", ".join(parts)
|
query = ", ".join(parts)
|
||||||
if tg_anchor or (node_lat is not None and node_lon is not None):
|
if tg_anchor or (node_lat is not None and node_lon is not None):
|
||||||
location_coords = await _geocode_location(
|
location_coords = await _geocode_location(
|
||||||
query, node_lat, node_lon, anchor=tg_anchor
|
query, node_lat, node_lon, anchor=tg_anchor,
|
||||||
|
trust_named_region=tg_named_region,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Embed this scene's content
|
# Embed this scene's content
|
||||||
@@ -337,6 +390,10 @@ async def extract_scenes(
|
|||||||
)
|
)
|
||||||
embedding = await asyncio.to_thread(_sync_embed, scene_text)
|
embedding = await asyncio.to_thread(_sync_embed, scene_text)
|
||||||
|
|
||||||
|
scene_transcript = _scene_transcript_text(
|
||||||
|
transcript, segments, segment_indices, transcript_corrected
|
||||||
|
)
|
||||||
|
|
||||||
processed.append({
|
processed.append({
|
||||||
"tags": tags,
|
"tags": tags,
|
||||||
"incident_type": incident_type,
|
"incident_type": incident_type,
|
||||||
@@ -348,6 +405,7 @@ async def extract_scenes(
|
|||||||
"severity": severity,
|
"severity": severity,
|
||||||
"resolved": resolved,
|
"resolved": resolved,
|
||||||
"reassignment": reassignment,
|
"reassignment": reassignment,
|
||||||
|
"transcript": scene_transcript,
|
||||||
"transcript_corrected": transcript_corrected,
|
"transcript_corrected": transcript_corrected,
|
||||||
"segment_indices": segment_indices,
|
"segment_indices": segment_indices,
|
||||||
"embedding": embedding,
|
"embedding": embedding,
|
||||||
@@ -458,6 +516,7 @@ async def _geocode_location(
|
|||||||
node_lat: Optional[float] = None,
|
node_lat: Optional[float] = None,
|
||||||
node_lon: Optional[float] = None,
|
node_lon: Optional[float] = None,
|
||||||
anchor: Optional[dict] = None,
|
anchor: Optional[dict] = None,
|
||||||
|
trust_named_region: bool = False,
|
||||||
) -> Optional[dict]:
|
) -> Optional[dict]:
|
||||||
"""
|
"""
|
||||||
Geocode using Google Maps Geocoding API, biased toward the channel's area.
|
Geocode using Google Maps Geocoding API, biased toward the channel's area.
|
||||||
@@ -473,6 +532,26 @@ async def _geocode_location(
|
|||||||
talkgroup has a resolved anchor, that is the reference and its own radius is
|
talkgroup has a resolved anchor, that is the reference and its own radius is
|
||||||
the bound. Distance-from-node stays only as the fallback for a system nobody
|
the bound. Distance-from-node stays only as the fallback for a system nobody
|
||||||
has described yet — it was always a stand-in for this.
|
has described yet — it was always a stand-in for this.
|
||||||
|
|
||||||
|
server-26#159: "a system nobody has described yet" turned out to include
|
||||||
|
systems that describe themselves — "New York City - NYPD Citywide 2 Patch"
|
||||||
|
names its own coverage area right in the talkgroup name, parsed into the
|
||||||
|
query by `_municipality_from_tg`, but a large aggregated/patched feed like
|
||||||
|
this is routinely received 40-70km from an antenna that happens to sit
|
||||||
|
wherever the node owner lives. Real, correctly-geocoded addresses on that
|
||||||
|
feed were being rejected by the node-distance check every single time —
|
||||||
|
location_coords stayed permanently null for the whole system, which killed
|
||||||
|
the location_proximity correlation signal and let duplicate incidents form
|
||||||
|
for the same event reported at two nearby addresses two minutes apart.
|
||||||
|
|
||||||
|
`trust_named_region` is True exactly when the query already carries a place
|
||||||
|
name that isn't the node's own position — operator-set area_context, or a
|
||||||
|
municipality parsed from the talkgroup's own name. In that case a distant
|
||||||
|
node is not evidence of a bad geocode, so the node-distance check is
|
||||||
|
skipped and precision is judged by `location_type` alone (still required
|
||||||
|
to be ROOFTOP/RANGE_INTERPOLATED/GEOMETRIC_CENTER, below). This does not
|
||||||
|
touch the anchor path at all — an anchor's own radius is always authoritative
|
||||||
|
when one has been resolved.
|
||||||
"""
|
"""
|
||||||
import httpx
|
import httpx
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
@@ -538,11 +617,21 @@ async def _geocode_location(
|
|||||||
lat, lng = float(loc["lat"]), float(loc["lng"])
|
lat, lng = float(loc["lat"]), float(loc["lng"])
|
||||||
dist_km = _geo_dist_km(ref_lat, ref_lon, lat, lng)
|
dist_km = _geo_dist_km(ref_lat, ref_lon, lat, lng)
|
||||||
if dist_km > max_km:
|
if dist_km > max_km:
|
||||||
|
# server-26#159: the node-distance bound is a proxy for "is
|
||||||
|
# this plausible" that only makes sense when the node's own
|
||||||
|
# position is our best guess at the area — never when the
|
||||||
|
# query already names a different region on its own terms.
|
||||||
|
if not (ref_label == "node" and trust_named_region):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f"Geocoding rejected '{location_str}' → ({lat:.4f}, {lng:.4f}) "
|
f"Geocoding rejected '{location_str}' → ({lat:.4f}, {lng:.4f}) "
|
||||||
f"— {dist_km:.1f}km from {ref_label} exceeds {max_km:.1f}km"
|
f"— {dist_km:.1f}km from {ref_label} exceeds {max_km:.1f}km"
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
logger.info(
|
||||||
|
f"Geocoding '{location_str}' → ({lat:.4f}, {lng:.4f}) is "
|
||||||
|
f"{dist_km:.1f}km from the receiving node, past {max_km:.1f}km — "
|
||||||
|
f"accepted anyway: the query names its own region, not the node's"
|
||||||
|
)
|
||||||
coords = {"lat": lat, "lng": lng}
|
coords = {"lat": lat, "lng": lng}
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Geocoded '{location_str}' → {coords} "
|
f"Geocoded '{location_str}' → {coords} "
|
||||||
@@ -568,14 +657,89 @@ def _municipality_from_tg(tg_name: Optional[str]) -> Optional[str]:
|
|||||||
return cleaned
|
return cleaned
|
||||||
|
|
||||||
|
|
||||||
|
def _location_query_parts(
|
||||||
|
location: str,
|
||||||
|
tg_area: dict,
|
||||||
|
talkgroup_name: Optional[str],
|
||||||
|
node_state: str,
|
||||||
|
node_county: str,
|
||||||
|
) -> tuple[list[str], bool]:
|
||||||
|
"""
|
||||||
|
Build the geocode query parts for `location`, plus whether the query names
|
||||||
|
a region the *talkgroup itself* covers (operator-set area_context, or a
|
||||||
|
municipality parsed from the talkgroup's own name) rather than one guessed
|
||||||
|
from wherever the receiving node happens to sit (server-26#159).
|
||||||
|
|
||||||
|
That distinction matters downstream: `_geocode_location`'s node-distance
|
||||||
|
sanity check is only a valid proxy for "is this plausible" when the node's
|
||||||
|
own position is the best guess we have at the area. A citywide/patched
|
||||||
|
feed ("New York City - NYPD Citywide 2 Patch") names its own coverage area
|
||||||
|
right in the talkgroup name — grafting the node's own county onto that
|
||||||
|
(Ossining-style: valid when the feed genuinely is local to the node,
|
||||||
|
actively wrong when it names a distant region of its own) would make the
|
||||||
|
query self-contradictory, so the node's COUNTY is used only when nothing
|
||||||
|
better names the place. The node's STATE is coarse enough to still be
|
||||||
|
correct either way and is kept in both branches.
|
||||||
|
"""
|
||||||
|
parts = [location]
|
||||||
|
if area_context.has_place(tg_area):
|
||||||
|
parts += [tg_area[f] for f in area_context.PLACE_FIELDS if tg_area.get(f)]
|
||||||
|
return parts, True
|
||||||
|
|
||||||
|
muni = _municipality_from_tg(talkgroup_name)
|
||||||
|
if muni:
|
||||||
|
parts += [p for p in (muni, node_state) if p]
|
||||||
|
else:
|
||||||
|
parts += [p for p in (node_county, node_state) if p]
|
||||||
|
return parts, muni is not None
|
||||||
|
|
||||||
|
|
||||||
def _build_transcript_block(transcript: str, segments: Optional[list[dict]]) -> str:
|
def _build_transcript_block(transcript: str, segments: Optional[list[dict]]) -> str:
|
||||||
"""Format transcript as numbered transmissions if segments are available."""
|
"""Format transcript as numbered transmissions if segments are available."""
|
||||||
if segments and len(segments) > 1:
|
if segments and len(segments) > 1:
|
||||||
lines = [f"{i+1}. [{s['start']}s] {s['text']}" for i, s in enumerate(segments)]
|
# 0-based labels, matching the prompt's "0-based indices into the
|
||||||
|
# numbered transmissions" — the model echoes these back as
|
||||||
|
# `segment_indices`, which _build_scene_embed_text and the per-scene
|
||||||
|
# `transcript` (server-26#102) then slice with directly.
|
||||||
|
lines = [f"{i}. [{s['start']}s] {s['text']}" for i, s in enumerate(segments)]
|
||||||
return f"Transmissions ({len(segments)}):\n" + "\n".join(lines)
|
return f"Transmissions ({len(segments)}):\n" + "\n".join(lines)
|
||||||
return f"Transcript:\n{transcript}"
|
return f"Transcript:\n{transcript}"
|
||||||
|
|
||||||
|
|
||||||
|
def _scene_transcript_text(
|
||||||
|
transcript: str,
|
||||||
|
segments: Optional[list[dict]],
|
||||||
|
segment_indices: Optional[list[int]],
|
||||||
|
transcript_corrected: Optional[str],
|
||||||
|
) -> str:
|
||||||
|
"""
|
||||||
|
This scene's own words, unprefixed — the segments it owns, joined.
|
||||||
|
|
||||||
|
server-26#102: the correlator's LLM tier reads this per scene instead of
|
||||||
|
the call doc's whole-call transcript, so on a multi-scene call scene N is
|
||||||
|
no longer judged against scenes 1..N-1's text.
|
||||||
|
|
||||||
|
Never returns "". Anything that would leave the slice empty — no
|
||||||
|
`segment_indices` (a single-segment call is never numbered by
|
||||||
|
`_build_transcript_block`), or indices that are out of range / not ints —
|
||||||
|
falls back to the whole-call transcript, which for a single-scene call is
|
||||||
|
the same text and for a mis-sliced multi-scene call is at least this
|
||||||
|
call's own words. `_sync_extract`'s prompt documents 0-based indices and
|
||||||
|
`_build_transcript_block` numbers to match, so no base normalisation here.
|
||||||
|
"""
|
||||||
|
if transcript_corrected:
|
||||||
|
return transcript_corrected
|
||||||
|
if segments and segment_indices:
|
||||||
|
joined = " ".join(
|
||||||
|
segments[i]["text"]
|
||||||
|
for i in segment_indices
|
||||||
|
if isinstance(i, int) and 0 <= i < len(segments)
|
||||||
|
)
|
||||||
|
if joined:
|
||||||
|
return joined
|
||||||
|
return transcript
|
||||||
|
|
||||||
|
|
||||||
def _build_scene_embed_text(
|
def _build_scene_embed_text(
|
||||||
transcript: str,
|
transcript: str,
|
||||||
segments: Optional[list[dict]],
|
segments: Optional[list[dict]],
|
||||||
@@ -601,6 +765,7 @@ def _sync_extract(
|
|||||||
segments: Optional[list[dict]],
|
segments: Optional[list[dict]],
|
||||||
vocabulary: Optional[list[str]] = None,
|
vocabulary: Optional[list[str]] = None,
|
||||||
ten_codes: Optional[dict[str, str]] = None,
|
ten_codes: Optional[dict[str, str]] = None,
|
||||||
|
unit_format_hint: Optional[str] = None,
|
||||||
) -> list[dict]:
|
) -> list[dict]:
|
||||||
"""Call GPT-4o-mini and return a list of scene dicts."""
|
"""Call GPT-4o-mini and return a list of scene dicts."""
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
@@ -618,6 +783,7 @@ def _sync_extract(
|
|||||||
system_id=system_id or "unknown",
|
system_id=system_id or "unknown",
|
||||||
ten_codes_block=_build_ten_codes_block(ten_codes or {}),
|
ten_codes_block=_build_ten_codes_block(ten_codes or {}),
|
||||||
vocabulary_block=build_gpt_vocab_block(vocabulary or []),
|
vocabulary_block=build_gpt_vocab_block(vocabulary or []),
|
||||||
|
unit_format_block=_build_unit_format_block(unit_format_hint),
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -45,7 +45,18 @@ def _fmt_idle(inc: dict, now: datetime) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def _inc_summary(inc: dict, now: datetime) -> str:
|
def _inc_summary(inc: dict, now: datetime) -> str:
|
||||||
|
# server-26#115: the model was given no title and no talkgroup, so it
|
||||||
|
# could not tell that "car alarms, Mohegan Park Ave" and "car alarms,
|
||||||
|
# Mohegan Park Avenue" on the same channel were one incident — it defaulted
|
||||||
|
# to "new". Title is the single strongest human-readable signal for "is
|
||||||
|
# this the same event"; talkgroup is what makes same-channel continuation
|
||||||
|
# obvious.
|
||||||
parts = [f"id:{inc['incident_id']}", f"type:{inc.get('type') or '?'}"]
|
parts = [f"id:{inc['incident_id']}", f"type:{inc.get('type') or '?'}"]
|
||||||
|
tgs = inc.get("talkgroup_ids") or []
|
||||||
|
if tgs:
|
||||||
|
parts.append(f"tg:[{', '.join(str(t) for t in tgs[:3])}]")
|
||||||
|
if inc.get("title"):
|
||||||
|
parts.append(f"title:{inc['title']!r}")
|
||||||
if inc.get("location"):
|
if inc.get("location"):
|
||||||
parts.append(f"loc:{inc['location']}")
|
parts.append(f"loc:{inc['location']}")
|
||||||
units = inc.get("units") or []
|
units = inc.get("units") or []
|
||||||
@@ -61,7 +72,13 @@ def _inc_summary(inc: dict, now: datetime) -> str:
|
|||||||
def _call_block(ctx: dict) -> str:
|
def _call_block(ctx: dict) -> str:
|
||||||
lines = []
|
lines = []
|
||||||
call_doc = ctx["call_doc"]
|
call_doc = ctx["call_doc"]
|
||||||
transcript = call_doc.get("transcript_corrected") or call_doc.get("transcript")
|
# The SCENE's own transcript, resolved in _build_context (server-26#102).
|
||||||
|
# Falls back to the call doc for a ctx built without a scene (tests, sweep).
|
||||||
|
transcript = (
|
||||||
|
ctx.get("scene_transcript")
|
||||||
|
or call_doc.get("transcript_corrected")
|
||||||
|
or call_doc.get("transcript")
|
||||||
|
)
|
||||||
if transcript:
|
if transcript:
|
||||||
lines.append(f"Transcript: {transcript[:700]}")
|
lines.append(f"Transcript: {transcript[:700]}")
|
||||||
if ctx["tags"]:
|
if ctx["tags"]:
|
||||||
@@ -74,19 +91,50 @@ def _call_block(ctx: dict) -> str:
|
|||||||
lines.append(f"Units: {ctx['call_units']}")
|
lines.append(f"Units: {ctx['call_units']}")
|
||||||
if ctx["call_vehicles"]:
|
if ctx["call_vehicles"]:
|
||||||
lines.append(f"Vehicles: {ctx['call_vehicles']}")
|
lines.append(f"Vehicles: {ctx['call_vehicles']}")
|
||||||
if ctx["talkgroup_name"]:
|
if ctx["talkgroup_name"] or ctx.get("talkgroup_id") is not None:
|
||||||
lines.append(f"Talkgroup: {ctx['talkgroup_name']}")
|
# Both the name and the id — _inc_summary emits numeric tg ids, so the
|
||||||
|
# id is what makes the "same talkgroup" rule in _RULES evaluable
|
||||||
|
# (server-26#115 review).
|
||||||
|
tgid = ctx.get("talkgroup_id")
|
||||||
|
name = ctx["talkgroup_name"] or "?"
|
||||||
|
lines.append(f"Talkgroup: {name}" + (f" (id {tgid})" if tgid is not None else ""))
|
||||||
return "\n".join(lines) if lines else "(no details)"
|
return "\n".join(lines) if lines else "(no details)"
|
||||||
|
|
||||||
|
|
||||||
|
def _prompt_incidents(recent: list[dict]) -> list[dict]:
|
||||||
|
"""The ≤20 candidates shown to the model, most-recently-active first.
|
||||||
|
|
||||||
|
`ctx["recent"]` is an unordered slice of a Firestore result with no
|
||||||
|
order_by, so a busy 2h window (~40 active incidents) meant the model saw
|
||||||
|
an arbitrary half of the candidates (server-26#115 review). Sorting by
|
||||||
|
updated_at desc also makes each row's `idle:` field monotonic.
|
||||||
|
"""
|
||||||
|
def _key(inc: dict):
|
||||||
|
return str(inc.get("updated_at") or inc.get("started_at") or "")
|
||||||
|
return sorted(recent, key=_key, reverse=True)[:20]
|
||||||
|
|
||||||
|
|
||||||
_SCHEMA = '{"action": "link" | "new" | "orphan", "incident_id": "<id_string or null>", "reasoning": "<one sentence>"}'
|
_SCHEMA = '{"action": "link" | "new" | "orphan", "incident_id": "<id_string or null>", "reasoning": "<one sentence>"}'
|
||||||
|
|
||||||
_RULES = """
|
_RULES = """
|
||||||
Rules:
|
Rules (this system OVER-SPLITS — a real incident routinely gets shattered into
|
||||||
- "link" only with clear positive evidence: same units, same geocoded location, or semantically identical scene on the same talkgroup within the last few minutes.
|
5-10 duplicates. A wrong link is cheap; a duplicate incident is the failure
|
||||||
- A call on a DIFFERENT talkgroup than an incident requires unit overlap or geocoded location match — topic similarity alone is not enough.
|
mode. Bias accordingly.):
|
||||||
- "new" only if the call has a clear incident_type AND describes a distinct, identifiable scene.
|
- Prefer "link" when the call plausibly continues a recent incident ON THE SAME
|
||||||
- "orphan" when in doubt — conservative is always correct.
|
TALKGROUP: same or overlapping units, the same or an adjacent location (treat
|
||||||
|
"Ave"/"Avenue", "St"/"Street", "Rd"/"Road" as identical; a house number plus
|
||||||
|
the same street is the same place), the same subject/vehicle/case number, or a
|
||||||
|
follow-up beat ("units clearing", "negative contact", "tow en route", "event
|
||||||
|
number 214-201", a status update) to an incident that is only a few minutes
|
||||||
|
idle. The bar for "link" on the same talkgroup is LOW.
|
||||||
|
- Reserve "new" for a call that clearly describes a DIFFERENT event from every
|
||||||
|
recent incident — a different place, different units, and a different subject,
|
||||||
|
not merely a different transmission about the same job.
|
||||||
|
- "orphan" a call that is not an incident at all: radio checks, roll call,
|
||||||
|
a unit marking on/off duty or 10-8/10-98, mileage/log entries, a bare
|
||||||
|
acknowledgement. Do not open a "new" incident for these.
|
||||||
|
- A call on a DIFFERENT talkgroup than an incident still requires unit overlap
|
||||||
|
or a geocoded/location match — topic similarity alone is not enough there.
|
||||||
- Do NOT link just because both calls involve police or both mention a road.
|
- Do NOT link just because both calls involve police or both mention a road.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -95,7 +143,7 @@ def _build_decide_prompt(ctx: dict) -> str:
|
|||||||
now = ctx["now"]
|
now = ctx["now"]
|
||||||
recent = ctx["recent"]
|
recent = ctx["recent"]
|
||||||
inc_block = (
|
inc_block = (
|
||||||
"\n".join(_inc_summary(inc, now) for inc in recent[:20])
|
"\n".join(_inc_summary(inc, now) for inc in _prompt_incidents(recent))
|
||||||
if recent else "(none)"
|
if recent else "(none)"
|
||||||
)
|
)
|
||||||
return (
|
return (
|
||||||
@@ -113,7 +161,7 @@ def _build_tiebreak_prompt(rules_decision: dict, llm_decision: dict, ctx: dict)
|
|||||||
now = ctx["now"]
|
now = ctx["now"]
|
||||||
recent = ctx["recent"]
|
recent = ctx["recent"]
|
||||||
inc_block = (
|
inc_block = (
|
||||||
"\n".join(_inc_summary(inc, now) for inc in recent[:20])
|
"\n".join(_inc_summary(inc, now) for inc in _prompt_incidents(recent))
|
||||||
if recent else "(none)"
|
if recent else "(none)"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -111,6 +111,8 @@ class MQTTHandler:
|
|||||||
"assigned_system_id": None,
|
"assigned_system_id": None,
|
||||||
"approval_status": "pending",
|
"approval_status": "pending",
|
||||||
"node_type": payload.get("node_type", "fixed"),
|
"node_type": payload.get("node_type", "fixed"),
|
||||||
|
"secondary_sdr_mode": payload.get("secondary_sdr_mode", "none"),
|
||||||
|
"sdr_count": payload.get("sdr_count", 1),
|
||||||
"enforce_override_timeout": payload.get("enforce_override_timeout", True),
|
"enforce_override_timeout": payload.get("enforce_override_timeout", True),
|
||||||
"is_overridden": False,
|
"is_overridden": False,
|
||||||
"override_system_id": None,
|
"override_system_id": None,
|
||||||
@@ -141,6 +143,11 @@ class MQTTHandler:
|
|||||||
updates["node_type"] = node_type
|
updates["node_type"] = node_type
|
||||||
updates["enforce_override_timeout"] = enforce_timeout
|
updates["enforce_override_timeout"] = enforce_timeout
|
||||||
|
|
||||||
|
if "secondary_sdr_mode" in payload:
|
||||||
|
updates["secondary_sdr_mode"] = payload["secondary_sdr_mode"]
|
||||||
|
if "sdr_count" in payload:
|
||||||
|
updates["sdr_count"] = payload["sdr_count"]
|
||||||
|
|
||||||
if node_type == "portable":
|
if node_type == "portable":
|
||||||
updates["is_overridden"] = False
|
updates["is_overridden"] = False
|
||||||
updates["override_system_id"] = None
|
updates["override_system_id"] = None
|
||||||
|
|||||||
@@ -20,6 +20,46 @@ from app.internal.logger import logger
|
|||||||
from app.internal import firestore as fstore
|
from app.internal import firestore as fstore
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
|
||||||
|
# server-26#131: minimum time since the real-time pipeline (routers/upload.py
|
||||||
|
# _run_intelligence_pipeline) marked intelligence_started_at before the sweep
|
||||||
|
# will touch a call, even though it already looks orphaned. STT + scene
|
||||||
|
# extraction + correlation is a multi-second-to-low-minutes chain (Whisper,
|
||||||
|
# then a Gemini call per scene); without this buffer the sweep could pick up
|
||||||
|
# a call mid-pipeline — no incident_id/corr_path written yet — and correlate
|
||||||
|
# it a second time, independently, sometimes onto a different incident than
|
||||||
|
# the real-time path lands on. That race is what #131 found (same call in
|
||||||
|
# two incidents' call_ids, ~2% of linked calls). A call with no
|
||||||
|
# intelligence_started_at at all (pre-#131 call doc, or the marker write
|
||||||
|
# itself failed) is NOT held back by this — absence isn't evidence of an
|
||||||
|
# in-flight pipeline, and #131's own bug predates this field existing.
|
||||||
|
#
|
||||||
|
# 15, not 5: neither OpenAI's Whisper client nor Gemini's call in
|
||||||
|
# llm_correlator.py sets a request timeout (server-26#153), so a hung call can
|
||||||
|
# run well past a few minutes on SDK-default retries, and this constant is a
|
||||||
|
# guess against that unbounded tail, not a measured bound. Raising it costs
|
||||||
|
# nothing on the recovery side: a call that finished processing (linked OR
|
||||||
|
# genuinely orphaned) always has corr_path set (_apply_and_log writes it even
|
||||||
|
# on the orphan action), so it's already excluded by the
|
||||||
|
# `not c.get("corr_path")` filter below and never reaches this check at all —
|
||||||
|
# this constant only ever delays calls that are still actually running.
|
||||||
|
MIN_MINUTES_SINCE_PIPELINE_START = 15
|
||||||
|
|
||||||
|
# Standard link-only retry budget before a call is tombstoned corr_path="unlinked".
|
||||||
|
MAX_SWEEP_ATTEMPTS = 3
|
||||||
|
# server-26#115 — a call the consensus LLM-orphan gate parked (llm=orphan vs
|
||||||
|
# rules=new, no substance) gets a longer budget. The gate fires before any
|
||||||
|
# incident for the job may exist, so the substantive call that would justify
|
||||||
|
# linking can land well after the standard ~6 min. Still link-only: a genuinely
|
||||||
|
# thin call must not mint an incident, and the rules creation gate would re-orphan
|
||||||
|
# it anyway.
|
||||||
|
GATED_ORPHAN_SWEEP_ATTEMPTS = 10
|
||||||
|
|
||||||
|
|
||||||
|
def _max_sweep_attempts(call: dict) -> int:
|
||||||
|
if call.get("corr_consensus") == "llm_orphan_gate":
|
||||||
|
return GATED_ORPHAN_SWEEP_ATTEMPTS
|
||||||
|
return MAX_SWEEP_ATTEMPTS
|
||||||
|
|
||||||
|
|
||||||
async def recorrelation_loop() -> None:
|
async def recorrelation_loop() -> None:
|
||||||
interval = settings.summary_interval_minutes * 60
|
interval = settings.summary_interval_minutes * 60
|
||||||
@@ -36,8 +76,22 @@ async def recorrelation_loop() -> None:
|
|||||||
logger.error(f"Re-correlation sweep failed: {e}")
|
logger.error(f"Re-correlation sweep failed: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
def _pipeline_likely_still_running(call: dict, now: datetime) -> bool:
|
||||||
|
"""server-26#131 — True when the real-time pipeline marked
|
||||||
|
intelligence_started_at recently enough that it's probably still mid-flight
|
||||||
|
(STT / scene extraction / correlation), so the sweep should not race it.
|
||||||
|
No marker at all (older call doc, or the marker write itself failed)
|
||||||
|
returns False — absence isn't evidence of an in-flight pipeline."""
|
||||||
|
started = _parse_dt(call.get("intelligence_started_at"))
|
||||||
|
if not started:
|
||||||
|
return False
|
||||||
|
age_minutes = (now - started).total_seconds() / 60
|
||||||
|
return age_minutes < MIN_MINUTES_SINCE_PIPELINE_START
|
||||||
|
|
||||||
|
|
||||||
async def _run_sweep_pass() -> None:
|
async def _run_sweep_pass() -> None:
|
||||||
cutoff = datetime.now(timezone.utc) - timedelta(minutes=settings.recorrelation_scan_minutes)
|
now = datetime.now(timezone.utc)
|
||||||
|
cutoff = now - timedelta(minutes=settings.recorrelation_scan_minutes)
|
||||||
|
|
||||||
# Server-side range query: only calls that ended within the scan window.
|
# Server-side range query: only calls that ended within the scan window.
|
||||||
# Filter incident_id=null client-side (Firestore can't query for missing fields).
|
# Filter incident_id=null client-side (Firestore can't query for missing fields).
|
||||||
@@ -46,10 +100,9 @@ async def _run_sweep_pass() -> None:
|
|||||||
("status", "==", "ended"),
|
("status", "==", "ended"),
|
||||||
("ended_at", ">=", cutoff),
|
("ended_at", ">=", cutoff),
|
||||||
])
|
])
|
||||||
# corr_path="unlinked" is written after MAX_SWEEP_ATTEMPTS failures.
|
# corr_path="unlinked" is written after the attempt budget is exhausted.
|
||||||
# Allows a few retries so a welfare-check call can link to an escalation
|
# Allows a few retries so a welfare-check call can link to an escalation
|
||||||
# incident that is created a few minutes later, without sweeping 30× forever.
|
# incident that is created a few minutes later, without sweeping 30× forever.
|
||||||
MAX_SWEEP_ATTEMPTS = 3
|
|
||||||
orphans = [
|
orphans = [
|
||||||
c for c in recent_ended
|
c for c in recent_ended
|
||||||
if not c.get("incident_ids") and not c.get("incident_id")
|
if not c.get("incident_ids") and not c.get("incident_id")
|
||||||
@@ -61,7 +114,8 @@ async def _run_sweep_pass() -> None:
|
|||||||
# the thin path minutes later and attached to whatever was most recent —
|
# the thin path minutes later and attached to whatever was most recent —
|
||||||
# a second route into the over-merge the thin fix above addresses.
|
# a second route into the over-merge the thin fix above addresses.
|
||||||
and not c.get("skip_reason")
|
and not c.get("skip_reason")
|
||||||
and c.get("corr_sweep_count", 0) < MAX_SWEEP_ATTEMPTS
|
and c.get("corr_sweep_count", 0) < _max_sweep_attempts(c)
|
||||||
|
and not _pipeline_likely_still_running(c, now)
|
||||||
]
|
]
|
||||||
|
|
||||||
if not orphans:
|
if not orphans:
|
||||||
@@ -108,6 +162,7 @@ async def _recorrelate_orphan(call: dict) -> bool:
|
|||||||
cleared_units = call.get("cleared_units") or [],
|
cleared_units = call.get("cleared_units") or [],
|
||||||
embedding = call.get("embedding"),
|
embedding = call.get("embedding"),
|
||||||
severity = call.get("severity"),
|
severity = call.get("severity"),
|
||||||
|
transcript = call.get("transcript_corrected") or call.get("transcript"),
|
||||||
reference_time = started_at, # anchor window to when the call happened
|
reference_time = started_at, # anchor window to when the call happened
|
||||||
create_if_new = False, # never create — link-only
|
create_if_new = False, # never create — link-only
|
||||||
)
|
)
|
||||||
@@ -119,12 +174,12 @@ async def _recorrelate_orphan(call: dict) -> bool:
|
|||||||
)
|
)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
# Increment the attempt counter. Once MAX_SWEEP_ATTEMPTS is reached the
|
# Increment the attempt counter. Once the budget is reached the orphan filter
|
||||||
# orphan filter above will stop picking this call up, and we write
|
# above will stop picking this call up, and we write corr_path="unlinked" as
|
||||||
# corr_path="unlinked" as a permanent tombstone.
|
# a permanent tombstone.
|
||||||
attempts = call.get("corr_sweep_count", 0) + 1
|
attempts = call.get("corr_sweep_count", 0) + 1
|
||||||
update: dict = {"corr_sweep_count": attempts}
|
update: dict = {"corr_sweep_count": attempts}
|
||||||
if attempts >= 3:
|
if attempts >= _max_sweep_attempts(call):
|
||||||
update["corr_path"] = "unlinked"
|
update["corr_path"] = "unlinked"
|
||||||
await fstore.doc_set("calls", call_id, update)
|
await fstore.doc_set("calls", call_id, update)
|
||||||
return False
|
return False
|
||||||
|
|||||||
@@ -15,6 +15,39 @@ from app.internal import firestore as fstore
|
|||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
|
||||||
|
|
||||||
|
def _scene_sort_key(scene_index: str):
|
||||||
|
"""Numeric-first sort so a >=10-scene call's entries still read in order."""
|
||||||
|
return (0, int(scene_index)) if scene_index.isdigit() else (1, scene_index)
|
||||||
|
|
||||||
|
|
||||||
|
def _scene_text_for_incident(doc: dict, incident_id: str) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
The text of `doc` (a call doc) that actually belongs to `incident_id`.
|
||||||
|
|
||||||
|
server-26#96 records, per scene, which incident_id that scene's
|
||||||
|
correlation decision resolved to (incident_correlator._apply_and_log's
|
||||||
|
`scenes.<index>.incident_id`). Use that to pick only the scene(s) of this
|
||||||
|
call that are genuinely part of this incident, joining more than one if
|
||||||
|
several scenes happened to link into the same incident.
|
||||||
|
|
||||||
|
Falls back to transcript_corrected-or-transcript when the call doc has no
|
||||||
|
`scenes` field (predates server-26#96) or — defensively — when it has one
|
||||||
|
but nothing in it names this incident_id (should not happen for a call_id
|
||||||
|
that's actually in this incident's call_ids, but silently dropping a
|
||||||
|
call's contribution to its own summary would be a worse failure mode than
|
||||||
|
falling back to the whole-call text).
|
||||||
|
"""
|
||||||
|
scenes = doc.get("scenes") or {}
|
||||||
|
matched = [
|
||||||
|
scene.get("transcript")
|
||||||
|
for _, scene in sorted(scenes.items(), key=lambda kv: _scene_sort_key(kv[0]))
|
||||||
|
if scene.get("incident_id") == incident_id and scene.get("transcript")
|
||||||
|
]
|
||||||
|
if matched:
|
||||||
|
return "\n".join(matched)
|
||||||
|
return doc.get("transcript_corrected") or doc.get("transcript")
|
||||||
|
|
||||||
|
|
||||||
async def summarizer_loop() -> None:
|
async def summarizer_loop() -> None:
|
||||||
from app.internal.feature_flags import get_flags
|
from app.internal.feature_flags import get_flags
|
||||||
interval = settings.summary_interval_minutes * 60
|
interval = settings.summary_interval_minutes * 60
|
||||||
@@ -63,12 +96,30 @@ async def _summarize_incident(inc: dict) -> None:
|
|||||||
if not call_ids:
|
if not call_ids:
|
||||||
return
|
return
|
||||||
|
|
||||||
# Fetch transcripts for all calls in this incident
|
# Fetch transcripts for all calls in this incident.
|
||||||
|
#
|
||||||
|
# server-26#114: a call links into an incident one SCENE at a time (see
|
||||||
|
# incident_correlator._apply_decision / server-26#96's `scenes` map on the
|
||||||
|
# call doc), and the same call_id can appear in more than one incident's
|
||||||
|
# call_ids — once per scene, each scene possibly landing in a different
|
||||||
|
# incident. Reading doc["transcript"] (the whole call, raw) meant an
|
||||||
|
# incident's summary was built partly on text from a DIFFERENT scene of
|
||||||
|
# that call that this incident has nothing to do with, and ignored
|
||||||
|
# transcript_corrected entirely.
|
||||||
|
#
|
||||||
|
# _scene_text_for_incident reads the specific scene(s) whose corr_debug
|
||||||
|
# recorded a link into THIS incident_id. For a call doc that predates
|
||||||
|
# this fix (no `scenes` field) it falls back to
|
||||||
|
# transcript_corrected-or-transcript — the one-liner half of #114, worth
|
||||||
|
# doing even for old-schema docs since it stops raw-transcript summaries.
|
||||||
transcripts: list[str] = []
|
transcripts: list[str] = []
|
||||||
for cid in call_ids:
|
for cid in call_ids:
|
||||||
doc = await fstore.doc_get("calls", cid)
|
doc = await fstore.doc_get("calls", cid)
|
||||||
if doc and doc.get("transcript"):
|
if not doc:
|
||||||
transcripts.append(doc["transcript"])
|
continue
|
||||||
|
text = _scene_text_for_incident(doc, incident_id)
|
||||||
|
if text:
|
||||||
|
transcripts.append(text)
|
||||||
|
|
||||||
if not transcripts:
|
if not transcripts:
|
||||||
# No transcripts yet — clear stale flag and wait for next pass
|
# No transcripts yet — clear stale flag and wait for next pass
|
||||||
|
|||||||
@@ -28,10 +28,23 @@ counties may have one talkgroup covering a single municipality, and that
|
|||||||
municipality's streets must not be buried under a county-wide list. A
|
municipality's streets must not be buried under a county-wide list. A
|
||||||
single-municipality system is the degenerate case: populate the system level and
|
single-municipality system is the degenerate case: populate the system level and
|
||||||
every talkgroup inherits it.
|
every talkgroup inherits it.
|
||||||
|
|
||||||
|
THE PROMPT'S OWN RULES ARE NOT ENFORCED (server-26#162). "Do NOT expand
|
||||||
|
ten-codes" and "NEVER add information" are instructions to the model, not
|
||||||
|
checks on its output — `correct()` used to accept `raw["corrected"]` verbatim.
|
||||||
|
Caught live: the same call came back with "10-7" rewritten to "10-13" in one
|
||||||
|
place and "10-4" in another, and "7" expanded into "ShotSpotter" — a real code
|
||||||
|
swapped for a different real code reads exactly as confident and trustworthy
|
||||||
|
as a correct one, which is worse than leaving the raw mishearing in place. The
|
||||||
|
model isn't graded on this at write time; `_code_tokens()` is a
|
||||||
|
verify-what-you-can-cheaply-check backstop, not a fix to the model's judgment:
|
||||||
|
it only catches a code-shaped token changing, not a wrong word substituted for
|
||||||
|
another equally plausible word.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
|
import re
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
@@ -96,6 +109,19 @@ def _dedupe(items: list[str]) -> list[str]:
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
# Ten-codes ("10-4"), unit/signal shorthand ("4-2"), and the digit-group
|
||||||
|
# fragments radio traffic reads out loud ("7-2-1" of a case number) all share
|
||||||
|
# this shape. The guard below does not need to know which of those a given
|
||||||
|
# token is — it only needs the SET of them to survive a "correction"
|
||||||
|
# unchanged, in order. A model rewriting "10-7" as "10-13" is not the kind of
|
||||||
|
# mishearing this pass exists to fix (server-26#162).
|
||||||
|
_CODE_TOKEN_RE = re.compile(r"\b\d{1,3}(?:-\d{1,3})+\b")
|
||||||
|
|
||||||
|
|
||||||
|
def _code_tokens(text: str) -> list[str]:
|
||||||
|
return _CODE_TOKEN_RE.findall(text or "")
|
||||||
|
|
||||||
|
|
||||||
def _talkgroup_entry(system_doc: dict, talkgroup_id: Optional[int]) -> dict:
|
def _talkgroup_entry(system_doc: dict, talkgroup_id: Optional[int]) -> dict:
|
||||||
"""The config.talkgroups[] entry for this talkgroup, or {}."""
|
"""The config.talkgroups[] entry for this talkgroup, or {}."""
|
||||||
if talkgroup_id is None:
|
if talkgroup_id is None:
|
||||||
@@ -317,6 +343,31 @@ async def correct(
|
|||||||
if verified_segments:
|
if verified_segments:
|
||||||
corrected_segments = verified_segments
|
corrected_segments = verified_segments
|
||||||
|
|
||||||
|
# server-26#162: a code-shaped token ("10-7", "4-2", a case-number
|
||||||
|
# fragment like "7-2-1") changing at all — not just going missing, any
|
||||||
|
# change — means the model touched something this pass has no business
|
||||||
|
# touching. Reject that half of the correction outright rather than trust
|
||||||
|
# a rewrite that already broke its own instructions once. Checked against
|
||||||
|
# the ORIGINAL text/segment, not each other, so a joined-text correction
|
||||||
|
# and a segment correction are judged independently, same as everywhere
|
||||||
|
# else in this function.
|
||||||
|
if corrected is not None and _code_tokens(corrected) != _code_tokens(text):
|
||||||
|
logger.warning(
|
||||||
|
f"Transcript correction for call {call_id} changed code-shaped "
|
||||||
|
f"tokens ({_code_tokens(text)} -> {_code_tokens(corrected)}) — "
|
||||||
|
f"discarding the joined correction"
|
||||||
|
)
|
||||||
|
corrected = None
|
||||||
|
if corrected_segments is not None:
|
||||||
|
for seg, orig in zip(corrected_segments, segments or []):
|
||||||
|
if _code_tokens(seg["text"]) != _code_tokens(orig.get("text", "")):
|
||||||
|
logger.warning(
|
||||||
|
f"Transcript correction for call {call_id} changed "
|
||||||
|
f"code-shaped tokens in a segment — discarding segment corrections"
|
||||||
|
)
|
||||||
|
corrected_segments = None
|
||||||
|
break
|
||||||
|
|
||||||
if corrected or corrected_segments or not_speech:
|
if corrected or corrected_segments or not_speech:
|
||||||
changed = raw.get("changed") or []
|
changed = raw.get("changed") or []
|
||||||
logger.info(
|
logger.info(
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
"""
|
||||||
|
Word error rate — server-26#163's eval harness needs a real number to compare
|
||||||
|
against, not a vibe. Standard definition: word-level Levenshtein distance
|
||||||
|
between a human-verified reference and the machine hypothesis, divided by the
|
||||||
|
reference's own word count. Case-insensitive, punctuation-insensitive — this
|
||||||
|
measures whether the right WORDS came out, not transcript formatting.
|
||||||
|
"""
|
||||||
|
import re
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
|
||||||
|
def _tokenize(text: str) -> list[str]:
|
||||||
|
return re.findall(r"[\w']+", (text or "").lower())
|
||||||
|
|
||||||
|
|
||||||
|
def word_error_rate(reference: str, hypothesis: str) -> Optional[float]:
|
||||||
|
"""
|
||||||
|
(substitutions + deletions + insertions) / len(reference words).
|
||||||
|
|
||||||
|
None when the reference has no words — WER is undefined there, not 0.0;
|
||||||
|
a caller that defaults a None to 0.0 would report a perfect score for a
|
||||||
|
call nobody actually transcribed.
|
||||||
|
"""
|
||||||
|
ref = _tokenize(reference)
|
||||||
|
hyp = _tokenize(hypothesis)
|
||||||
|
if not ref:
|
||||||
|
return None
|
||||||
|
if not hyp:
|
||||||
|
return 1.0
|
||||||
|
|
||||||
|
n, m = len(ref), len(hyp)
|
||||||
|
# Single-row DP over Levenshtein distance — O(n*m) time, O(m) space.
|
||||||
|
row = list(range(m + 1))
|
||||||
|
for i in range(1, n + 1):
|
||||||
|
prev_diag = row[0]
|
||||||
|
row[0] = i
|
||||||
|
for j in range(1, m + 1):
|
||||||
|
prev_row_j = row[j]
|
||||||
|
if ref[i - 1] == hyp[j - 1]:
|
||||||
|
row[j] = prev_diag
|
||||||
|
else:
|
||||||
|
row[j] = 1 + min(prev_diag, row[j], row[j - 1])
|
||||||
|
prev_diag = prev_row_j
|
||||||
|
return row[m] / n
|
||||||
+26
-18
@@ -17,7 +17,7 @@ from app.internal.auth import (
|
|||||||
require_node_service_or_firebase_token,
|
require_node_service_or_firebase_token,
|
||||||
)
|
)
|
||||||
from app.routers import nodes, systems, calls, upload, tokens, incidents, alerts, admin, trips, places, links, users
|
from app.routers import nodes, systems, calls, upload, tokens, incidents, alerts, admin, trips, places, links, users
|
||||||
from app.routers import enrollment, media, org, waitlist
|
from app.routers import enrollment, media, org, waitlist, telemetry
|
||||||
from app.internal import dynsec
|
from app.internal import dynsec
|
||||||
from app.internal import firestore as fstore
|
from app.internal import firestore as fstore
|
||||||
|
|
||||||
@@ -78,33 +78,40 @@ async def lifespan(app: FastAPI):
|
|||||||
|
|
||||||
app = FastAPI(title="DRB C2 Core", lifespan=lifespan)
|
app = FastAPI(title="DRB C2 Core", lifespan=lifespan)
|
||||||
|
|
||||||
# "*" plus allow_credentials=True is not the permissive-but-harmless setting it
|
# The browser needs CORS to reach this API at all: the frontend's Archive page
|
||||||
# looks like. Starlette does not refuse the combination -- it reflects the
|
# calls GET /calls/search with Authorization + Content-Type headers, which
|
||||||
# caller's Origin back and still sends Access-Control-Allow-Credentials: true,
|
# forces a preflight. Without this middleware the OPTIONS gets a bare 405 and
|
||||||
# so the effective policy becomes "any origin, with credentials", the opposite
|
# the fetch fails (#110). allow_origins is an explicit list -- never "*" in a
|
||||||
# of what a wildcard normally means. Rather than trust every deployment to
|
# deployment -- so name every host the frontend is served from in CORS_ORIGINS.
|
||||||
# remember to override CORS_ORIGINS, make the dangerous pair unrepresentable.
|
#
|
||||||
|
# allow_credentials stays False on purpose: auth here is a Bearer header, not a
|
||||||
|
# cookie, so credentialed CORS is never needed, and keeping it False is what
|
||||||
|
# lets an explicit-origin allowlist work without Starlette's "*"-only
|
||||||
|
# restriction. "*" + credentials is the dangerous pair (Starlette reflects the
|
||||||
|
# caller's Origin back WITH Access-Control-Allow-Credentials: true); this code
|
||||||
|
# cannot produce it because credentials are hard-off.
|
||||||
def cors_allows_credentials(origins: list[str]) -> bool:
|
def cors_allows_credentials(origins: list[str]) -> bool:
|
||||||
"""False when any entry is a wildcard. Extracted so it can be tested
|
"""Always False -- credentialed CORS is never enabled here (Bearer auth,
|
||||||
without re-importing this module, which drags in every router."""
|
not cookies). Kept as a named predicate so a future edit that wants to
|
||||||
return "*" not in origins
|
turn credentials on has to go through here and confront the "*" case.
|
||||||
|
A wildcard entry would additionally be refused a credentialed response."""
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
_cors_is_wildcard = not cors_allows_credentials(settings.cors_origins)
|
_cors_is_wildcard = "*" in settings.cors_origins
|
||||||
if _cors_is_wildcard:
|
if _cors_is_wildcard:
|
||||||
logger.error(
|
logger.error(
|
||||||
"CORS_ORIGINS is '*', so credentialed cross-origin requests are being "
|
"CORS_ORIGINS contains '*'. That is fine for local dev but is almost "
|
||||||
"DISABLED to avoid reflecting every caller's origin back with "
|
"certainly a misconfigured deployment -- set CORS_ORIGINS to your "
|
||||||
"Access-Control-Allow-Credentials. Set CORS_ORIGINS to your frontend "
|
"frontend origin(s), e.g. [\"https://drb.cusano.net\"]."
|
||||||
"origin(s) in production, e.g. [\"https://app.example.com\"]."
|
|
||||||
)
|
)
|
||||||
|
|
||||||
app.add_middleware(
|
app.add_middleware(
|
||||||
CORSMiddleware,
|
CORSMiddleware,
|
||||||
allow_origins=settings.cors_origins,
|
allow_origins=settings.cors_origins,
|
||||||
allow_methods=["*"],
|
allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"],
|
||||||
allow_headers=["*"],
|
allow_headers=["authorization", "content-type"],
|
||||||
allow_credentials=not _cors_is_wildcard,
|
allow_credentials=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
app.include_router(nodes.router, dependencies=[Depends(require_service_or_firebase_token)])
|
app.include_router(nodes.router, dependencies=[Depends(require_service_or_firebase_token)])
|
||||||
@@ -113,6 +120,7 @@ app.include_router(nodes.router, dependencies=[Depends(require_service_or_fi
|
|||||||
# write routes inside carry their own require_admin_token, so nodes get read
|
# write routes inside carry their own require_admin_token, so nodes get read
|
||||||
# access only.
|
# access only.
|
||||||
app.include_router(systems.router, dependencies=[Depends(require_node_service_or_firebase_token)])
|
app.include_router(systems.router, dependencies=[Depends(require_node_service_or_firebase_token)])
|
||||||
|
app.include_router(telemetry.router, dependencies=[Depends(require_node_service_or_firebase_token)])
|
||||||
app.include_router(calls.router, dependencies=[Depends(require_service_or_firebase_token)])
|
app.include_router(calls.router, dependencies=[Depends(require_service_or_firebase_token)])
|
||||||
app.include_router(tokens.router, dependencies=[Depends(require_service_or_firebase_token)])
|
app.include_router(tokens.router, dependencies=[Depends(require_service_or_firebase_token)])
|
||||||
app.include_router(incidents.router, dependencies=[Depends(require_service_or_firebase_token)])
|
app.include_router(incidents.router, dependencies=[Depends(require_service_or_firebase_token)])
|
||||||
|
|||||||
@@ -62,12 +62,43 @@ class NodeRecord(BaseModel):
|
|||||||
last_seen: Optional[datetime] = None
|
last_seen: Optional[datetime] = None
|
||||||
assigned_system_id: Optional[str] = None
|
assigned_system_id: Optional[str] = None
|
||||||
node_type: str = "fixed" # fixed or portable
|
node_type: str = "fixed" # fixed or portable
|
||||||
|
secondary_sdr_mode: str = "none" # none | adsb | ais | op25_2 — requires a second physical SDR
|
||||||
|
sdr_count: int = 1 # self-reported by the node's checkin, best-effort
|
||||||
enforce_override_timeout: bool = True
|
enforce_override_timeout: bool = True
|
||||||
is_overridden: bool = False
|
is_overridden: bool = False
|
||||||
override_system_id: Optional[str] = None
|
override_system_id: Optional[str] = None
|
||||||
override_timeout_at: Optional[datetime] = None
|
override_timeout_at: Optional[datetime] = None
|
||||||
|
|
||||||
|
|
||||||
|
class AircraftTrack(BaseModel):
|
||||||
|
"""Live ADS-B position, one doc per icao. Overwritten on every sighting —
|
||||||
|
this is a live-map snapshot, not a history (see node-26#9)."""
|
||||||
|
icao: str
|
||||||
|
org_id: Optional[str] = None
|
||||||
|
node_id: str
|
||||||
|
callsign: Optional[str] = None
|
||||||
|
lat: Optional[float] = None
|
||||||
|
lon: Optional[float] = None
|
||||||
|
altitude_ft: Optional[float] = None
|
||||||
|
ground_speed_kt: Optional[float] = None
|
||||||
|
track_deg: Optional[float] = None
|
||||||
|
last_seen: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class VesselTrack(BaseModel):
|
||||||
|
"""Live AIS position, one doc per mmsi. Same live-snapshot shape as
|
||||||
|
AircraftTrack — overwritten on every sighting (see node-26#9)."""
|
||||||
|
mmsi: str
|
||||||
|
org_id: Optional[str] = None
|
||||||
|
node_id: str
|
||||||
|
name: Optional[str] = None
|
||||||
|
lat: Optional[float] = None
|
||||||
|
lon: Optional[float] = None
|
||||||
|
speed_kt: Optional[float] = None
|
||||||
|
heading_deg: Optional[float] = None
|
||||||
|
last_seen: datetime
|
||||||
|
|
||||||
|
|
||||||
class CommandPayload(BaseModel):
|
class CommandPayload(BaseModel):
|
||||||
action: str # discord_join / discord_leave / op25_restart
|
action: str # discord_join / discord_leave / op25_restart
|
||||||
guild_id: Optional[str] = None
|
guild_id: Optional[str] = None
|
||||||
|
|||||||
@@ -97,8 +97,54 @@ async def debug_correlation(
|
|||||||
def _strip(doc: dict) -> dict:
|
def _strip(doc: dict) -> dict:
|
||||||
return {k: v for k, v in doc.items() if k != "embedding"}
|
return {k: v for k, v in doc.items() if k != "embedding"}
|
||||||
|
|
||||||
def _call_summary(call: dict) -> dict:
|
def _scene_summary(scene_index: str, scene: dict) -> dict:
|
||||||
|
"""
|
||||||
|
One scene's own correlation record, from the call doc's `scenes` map
|
||||||
|
(server-26#96). Same corr_* field names as _call_summary's flat
|
||||||
|
fields below, deliberately — a scene entry and a scene-less call
|
||||||
|
summary are interchangeable data points to the tally functions.
|
||||||
|
"""
|
||||||
|
corr_debug = scene.get("corr_debug") or {}
|
||||||
return {
|
return {
|
||||||
|
"scene_index": scene_index,
|
||||||
|
"transcript": scene.get("transcript"),
|
||||||
|
"incident_id": scene.get("incident_id"),
|
||||||
|
# server-26#139: this scene's OWN incident_type/severity, as seen
|
||||||
|
# by _call_is_substanceless at decision time — not the call doc's
|
||||||
|
# flat top-level field, which is last-scene-wins (server-26#96).
|
||||||
|
"incident_type": scene.get("incident_type"),
|
||||||
|
"severity": scene.get("severity"),
|
||||||
|
"corr_path": corr_debug.get("corr_path"),
|
||||||
|
"corr_incident_idle_min": corr_debug.get("corr_incident_idle_min"),
|
||||||
|
"corr_distance_km": corr_debug.get("corr_distance_km"),
|
||||||
|
"corr_score": corr_debug.get("corr_score"),
|
||||||
|
"corr_candidates": corr_debug.get("corr_candidates"),
|
||||||
|
"corr_shared_units": corr_debug.get("corr_shared_units"),
|
||||||
|
"corr_fit_signal": corr_debug.get("corr_fit_signal"),
|
||||||
|
"corr_matched_units": corr_debug.get("corr_matched_units"),
|
||||||
|
"corr_consensus": corr_debug.get("corr_consensus"),
|
||||||
|
"corr_llm_reasoning": corr_debug.get("corr_llm_reasoning"),
|
||||||
|
"corr_llm_action": corr_debug.get("corr_llm_action"),
|
||||||
|
"corr_rules_action": corr_debug.get("corr_rules_action"),
|
||||||
|
"corr_gate_veto": corr_debug.get("corr_gate_veto"),
|
||||||
|
}
|
||||||
|
|
||||||
|
def _call_summary(call: dict) -> dict:
|
||||||
|
# server-26#96 — per-scene records, keyed by scene index as written by
|
||||||
|
# incident_correlator._apply_and_log. Present only on calls that went
|
||||||
|
# through correlation after this fix landed; absent (None) on older
|
||||||
|
# call docs, which the tally below falls back for. Sorted numerically
|
||||||
|
# so a >=10-scene call still reads in scene order.
|
||||||
|
scenes_map = call.get("scenes") or {}
|
||||||
|
scenes = [
|
||||||
|
_scene_summary(idx, s)
|
||||||
|
for idx, s in sorted(
|
||||||
|
scenes_map.items(),
|
||||||
|
key=lambda kv: (0, int(kv[0])) if kv[0].isdigit() else (1, kv[0]),
|
||||||
|
)
|
||||||
|
] or None
|
||||||
|
return {
|
||||||
|
"scenes": scenes,
|
||||||
"call_id": call.get("call_id"),
|
"call_id": call.get("call_id"),
|
||||||
"started_at": call.get("started_at"),
|
"started_at": call.get("started_at"),
|
||||||
"ended_at": call.get("ended_at"),
|
"ended_at": call.get("ended_at"),
|
||||||
@@ -135,6 +181,19 @@ async def debug_correlation(
|
|||||||
"corr_llm_reasoning": call.get("corr_llm_reasoning"),
|
"corr_llm_reasoning": call.get("corr_llm_reasoning"),
|
||||||
"corr_llm_action": call.get("corr_llm_action"),
|
"corr_llm_action": call.get("corr_llm_action"),
|
||||||
"corr_rules_action": call.get("corr_rules_action"),
|
"corr_rules_action": call.get("corr_rules_action"),
|
||||||
|
# server-26#115 — why an llm=orphan/rules=new disagreement escalated
|
||||||
|
# to tiebreak instead of being gated (see upload.py's
|
||||||
|
# _call_is_substanceless). Present only on that disagreement shape;
|
||||||
|
# written here specifically so a live measurement window can read
|
||||||
|
# the reason instead of reconstructing it by hand from the dump.
|
||||||
|
"corr_gate_veto": call.get("corr_gate_veto"),
|
||||||
|
# server-26#127 — shadow-mode upstream chatter classifier verdict.
|
||||||
|
# Written by intelligence.extract_scenes on every transcript that
|
||||||
|
# reaches real scene extraction (not on garbage/too-short skips).
|
||||||
|
# Nothing skips extraction on this yet — it's here purely so a
|
||||||
|
# live measurement window can read the false-positive rate.
|
||||||
|
"chatter_classifier_verdict": call.get("chatter_classifier_verdict"),
|
||||||
|
"chatter_classifier_reason": call.get("chatter_classifier_reason"),
|
||||||
}
|
}
|
||||||
|
|
||||||
# ── Determine which systems have AI active ────────────────────────────────
|
# ── Determine which systems have AI active ────────────────────────────────
|
||||||
@@ -258,6 +317,21 @@ async def debug_correlation(
|
|||||||
linked = [c for inc in incident_records for c in (inc.get("calls_detail") or [])]
|
linked = [c for inc in incident_records for c in (inc.get("calls_detail") or [])]
|
||||||
call_counts = [len(inc.get("call_ids") or []) for inc in incident_records]
|
call_counts = [len(inc.get("call_ids") or []) for inc in incident_records]
|
||||||
|
|
||||||
|
def _tally_entries(call_summary: dict) -> list:
|
||||||
|
"""
|
||||||
|
server-26#96 — the unit correlation actually decided over is the
|
||||||
|
scene, not the call. A call summary carrying a `scenes` list (every
|
||||||
|
call correlated after this fix) contributes one entry per scene, each
|
||||||
|
with its own corr_path/corr_consensus/etc, instead of the single flat
|
||||||
|
record that used to blend every scene's last write together. A call
|
||||||
|
summary with no `scenes` (a call doc from before this fix) falls back
|
||||||
|
to contributing itself as one entry — identical to pre-#96 behaviour.
|
||||||
|
"""
|
||||||
|
scenes = call_summary.get("scenes")
|
||||||
|
return scenes if scenes else [call_summary]
|
||||||
|
|
||||||
|
scene_entries = [entry for c in linked for entry in _tally_entries(c)]
|
||||||
|
|
||||||
def _span_minutes(inc: dict) -> float:
|
def _span_minutes(inc: dict) -> float:
|
||||||
stamps = sorted(
|
stamps = sorted(
|
||||||
s for s in ((c.get("started_at") or "") for c in (inc.get("calls_detail") or [])) if s
|
s for s in ((c.get("started_at") or "") for c in (inc.get("calls_detail") or [])) if s
|
||||||
@@ -289,10 +363,31 @@ async def debug_correlation(
|
|||||||
"ai_systems_only": ai_systems_only,
|
"ai_systems_only": ai_systems_only,
|
||||||
"ai_enabled_system_ids": sorted(ai_systems),
|
"ai_enabled_system_ids": sorted(ai_systems),
|
||||||
"linked_call_count": len(linked),
|
"linked_call_count": len(linked),
|
||||||
"corr_path": _tally(c.get("corr_path") for c in linked),
|
# server-26#96 — tallied over scene_entries (one entry per scene of a
|
||||||
"corr_fit_signal": _tally(c.get("corr_fit_signal") for c in linked),
|
# multi-scene call, from its `scenes` map; one entry per call when it
|
||||||
"corr_consensus": _tally(c.get("corr_consensus") for c in linked),
|
# has none) rather than over `linked` directly, so a 2-scene call
|
||||||
"corr_llm_action": _tally(c.get("corr_llm_action") for c in linked),
|
# with two different corr_path values counts as two data points
|
||||||
|
# instead of one blended flat record. scene_decision_count makes that
|
||||||
|
# distinction visible next to linked_call_count.
|
||||||
|
"scene_decision_count": len(scene_entries),
|
||||||
|
"corr_path": _tally(e.get("corr_path") for e in scene_entries),
|
||||||
|
"corr_fit_signal": _tally(e.get("corr_fit_signal") for e in scene_entries),
|
||||||
|
"corr_consensus": _tally(e.get("corr_consensus") for e in scene_entries),
|
||||||
|
"corr_llm_action": _tally(e.get("corr_llm_action") for e in scene_entries),
|
||||||
|
# server-26#115 — this IS the number the escape-hatch fix exists to
|
||||||
|
# produce: why each llm=orphan/rules=new call escaped the gate.
|
||||||
|
"corr_gate_veto": _tally(e.get("corr_gate_veto") for e in scene_entries),
|
||||||
|
# server-26#127 — shadow-mode chatter classifier. The target
|
||||||
|
# population is non-events, which land as orphans or single-call
|
||||||
|
# incidents, NOT as a slice of every linked call -- tally `orphans`
|
||||||
|
# too or this undercounts the exact thing the feature measures.
|
||||||
|
"chatter_classifier_flagged": sum(
|
||||||
|
1 for c in (linked + orphans) if c.get("chatter_classifier_verdict")
|
||||||
|
),
|
||||||
|
"chatter_classifier_reason": _tally(
|
||||||
|
c.get("chatter_classifier_reason") for c in (linked + orphans)
|
||||||
|
if c.get("chatter_classifier_verdict")
|
||||||
|
),
|
||||||
# STT coverage: correlation quality is capped by this, so it belongs in
|
# STT coverage: correlation quality is capped by this, so it belongs in
|
||||||
# the same view rather than a separate investigation.
|
# the same view rather than a separate investigation.
|
||||||
"linked_calls_with_transcript": with_transcript,
|
"linked_calls_with_transcript": with_transcript,
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from typing import Optional
|
|||||||
from app.internal import firestore as fstore
|
from app.internal import firestore as fstore
|
||||||
from app.internal.auth import (
|
from app.internal.auth import (
|
||||||
require_admin_token,
|
require_admin_token,
|
||||||
|
require_firebase_token,
|
||||||
require_service_or_firebase_token,
|
require_service_or_firebase_token,
|
||||||
resolve_caller_org_id,
|
resolve_caller_org_id,
|
||||||
reprocess_limiter,
|
reprocess_limiter,
|
||||||
@@ -15,9 +16,49 @@ from app.internal.storage import gcs_uri_for_call, with_playback_url
|
|||||||
class TranscriptUpdate(BaseModel):
|
class TranscriptUpdate(BaseModel):
|
||||||
transcript: str
|
transcript: str
|
||||||
|
|
||||||
|
|
||||||
|
class EvalTranscriptUpdate(BaseModel):
|
||||||
|
text: str
|
||||||
|
|
||||||
router = APIRouter(prefix="/calls", tags=["calls"])
|
router = APIRouter(prefix="/calls", tags=["calls"])
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_ts(value: Optional[str], field: str) -> Optional[datetime]:
|
||||||
|
"""ISO string from a query param → aware datetime, or 400.
|
||||||
|
|
||||||
|
started_at is stored as a Firestore timestamp, so a cursor or range bound
|
||||||
|
passed through as the raw string compares by *type* (every string sorts
|
||||||
|
after every timestamp) rather than by time — a string cursor made "Load
|
||||||
|
more" return the first page again.
|
||||||
|
"""
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||||
|
except ValueError:
|
||||||
|
raise HTTPException(400, f"{field} is not an ISO-8601 timestamp.")
|
||||||
|
return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _next_cursor(rows: list[dict], matches: list[dict], page: list[dict], window: int) -> Optional[str]:
|
||||||
|
"""Where the next page of a bounded-window scan starts.
|
||||||
|
|
||||||
|
More matches than fit on the page → resume right after the last row
|
||||||
|
returned, or every match between it and the end of the window is skipped
|
||||||
|
(a 200-row window shown 50 at a time lost 150 calls per "Load more").
|
||||||
|
Otherwise resume after the last row SCANNED, not the last match — a page
|
||||||
|
whose last match sits early in the window would re-scan everything after
|
||||||
|
it and loop forever on a sparse filter. A short window is the end.
|
||||||
|
"""
|
||||||
|
if len(matches) > len(page):
|
||||||
|
last = page[-1].get("started_at")
|
||||||
|
elif len(rows) == window:
|
||||||
|
last = rows[-1].get("started_at")
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
return last.isoformat() if hasattr(last, "isoformat") else last
|
||||||
|
|
||||||
|
|
||||||
@router.get("")
|
@router.get("")
|
||||||
async def list_calls(
|
async def list_calls(
|
||||||
node_id: Optional[str] = Query(None),
|
node_id: Optional[str] = Query(None),
|
||||||
@@ -50,7 +91,9 @@ async def search_calls(
|
|||||||
link: str = Query("any", pattern="^(any|orphan|linked)$"),
|
link: str = Query("any", pattern="^(any|orphan|linked)$"),
|
||||||
transcript: str = Query("any", pattern="^(any|yes|no)$"),
|
transcript: str = Query("any", pattern="^(any|yes|no)$"),
|
||||||
q: Optional[str] = Query(None, description="case-insensitive substring of the transcript"),
|
q: Optional[str] = Query(None, description="case-insensitive substring of the transcript"),
|
||||||
decoded: dict = Depends(require_admin_token),
|
date_from: Optional[str] = Query(None, description="ISO timestamp, inclusive lower bound on started_at"),
|
||||||
|
date_to: Optional[str] = Query(None, description="ISO timestamp, inclusive upper bound on started_at"),
|
||||||
|
decoded: dict = Depends(require_firebase_token),
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Paged, filterable call archive — the backend for the /calls page.
|
Paged, filterable call archive — the backend for the /calls page.
|
||||||
@@ -68,6 +111,11 @@ async def search_calls(
|
|||||||
|
|
||||||
`window_exhausted` says the scan hit its cap before filling the page, so an
|
`window_exhausted` says the scan hit its cap before filling the page, so an
|
||||||
empty result means "not in this window", not "none exist".
|
empty result means "not in this window", not "none exist".
|
||||||
|
|
||||||
|
Open to every org member (viewer included), not just admins: the Firestore
|
||||||
|
rules already let any member read every call doc in their org
|
||||||
|
(firestore.rules `calls` → docInMyOrg), so this route exposes nothing a
|
||||||
|
viewer's browser couldn't already read directly.
|
||||||
"""
|
"""
|
||||||
org_id = await resolve_caller_org_id(decoded)
|
org_id = await resolve_caller_org_id(decoded)
|
||||||
if org_id is None:
|
if org_id is None:
|
||||||
@@ -78,13 +126,24 @@ async def search_calls(
|
|||||||
if not org_id:
|
if not org_id:
|
||||||
raise HTTPException(403, "No organization scope for this caller.")
|
raise HTTPException(403, "No organization scope for this caller.")
|
||||||
|
|
||||||
|
cursor_dt = _parse_ts(cursor, "cursor")
|
||||||
|
from_dt = _parse_ts(date_from, "date_from")
|
||||||
|
to_dt = _parse_ts(date_to, "date_to")
|
||||||
|
|
||||||
|
# A range on the ordered field rides the same org_id/started_at index.
|
||||||
|
conditions: list[tuple[str, str, object]] = [("org_id", "==", org_id)]
|
||||||
|
if from_dt:
|
||||||
|
conditions.append(("started_at", ">=", from_dt))
|
||||||
|
if to_dt:
|
||||||
|
conditions.append(("started_at", "<=", to_dt))
|
||||||
|
|
||||||
window = max(limit * 10, 200)
|
window = max(limit * 10, 200)
|
||||||
rows = await fstore.collection_where(
|
rows = await fstore.collection_where(
|
||||||
"calls",
|
"calls",
|
||||||
[("org_id", "==", org_id)],
|
conditions,
|
||||||
order_by=[("started_at", "DESCENDING")],
|
order_by=[("started_at", "DESCENDING")],
|
||||||
limit_to=window,
|
limit_to=window,
|
||||||
start_after={"started_at": cursor} if cursor else None,
|
start_after={"started_at": cursor_dt} if cursor_dt else None,
|
||||||
)
|
)
|
||||||
|
|
||||||
needle = (q or "").strip().lower()
|
needle = (q or "").strip().lower()
|
||||||
@@ -113,13 +172,7 @@ async def search_calls(
|
|||||||
matches = [c for c in rows if _keep(c)]
|
matches = [c for c in rows if _keep(c)]
|
||||||
page = matches[:limit]
|
page = matches[:limit]
|
||||||
|
|
||||||
# Cursor advances over the SCANNED window, not the filtered page — otherwise
|
next_cursor = _next_cursor(rows, matches, page, window)
|
||||||
# a page whose last match sits early in the window would re-scan everything
|
|
||||||
# after it on the next request and loop forever on a sparse filter.
|
|
||||||
next_cursor = None
|
|
||||||
if len(rows) == window:
|
|
||||||
last_scanned = rows[-1].get("started_at")
|
|
||||||
next_cursor = last_scanned.isoformat() if hasattr(last_scanned, "isoformat") else last_scanned
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"calls": [with_playback_url(c) for c in page],
|
"calls": [with_playback_url(c) for c in page],
|
||||||
@@ -130,6 +183,107 @@ async def search_calls(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/eval-queue")
|
||||||
|
async def eval_queue(
|
||||||
|
limit: int = Query(5, ge=1, le=20),
|
||||||
|
cursor: Optional[str] = Query(None, description="started_at of the last row of the previous page"),
|
||||||
|
decoded: dict = Depends(require_admin_token),
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
A batch of calls that have a machine transcript but no human-verified one
|
||||||
|
yet — the backend for the STT eval page (server-26#163).
|
||||||
|
|
||||||
|
Deliberately separate from `PATCH /{call_id}/transcript`: that route is a
|
||||||
|
PRODUCTION correction — it re-runs extraction, unlinks incidents, and
|
||||||
|
feeds the vocabulary learner. An eval annotation must never trigger any
|
||||||
|
of that; it only exists to measure the pipeline, not to change what it
|
||||||
|
already decided. `eval_transcript` lives next to `transcript`/
|
||||||
|
`transcript_corrected` on the call doc and nothing downstream reads it.
|
||||||
|
|
||||||
|
Same bounded-window-scan-plus-cursor shape as `/search`, for the same
|
||||||
|
reason: no composite index exists for "eval_transcript is unset", and one
|
||||||
|
scan ordered by started_at is already trusted here. Paging through with
|
||||||
|
the returned cursor is how "however many, over time" actually works —
|
||||||
|
each call is where the last session left off, not a fresh random sample.
|
||||||
|
"""
|
||||||
|
org_id = await resolve_caller_org_id(decoded)
|
||||||
|
if org_id is None:
|
||||||
|
org_id = decoded.get("org_id")
|
||||||
|
if not org_id:
|
||||||
|
raise HTTPException(403, "No organization scope for this caller.")
|
||||||
|
|
||||||
|
cursor_dt = _parse_ts(cursor, "cursor")
|
||||||
|
window = max(limit * 20, 300)
|
||||||
|
rows = await fstore.collection_where(
|
||||||
|
"calls",
|
||||||
|
[("org_id", "==", org_id)],
|
||||||
|
order_by=[("started_at", "DESCENDING")],
|
||||||
|
limit_to=window,
|
||||||
|
start_after={"started_at": cursor_dt} if cursor_dt else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _eligible(c: dict) -> bool:
|
||||||
|
text = c.get("transcript_corrected") or c.get("transcript") or ""
|
||||||
|
return bool(text) and not c.get("eval_transcript")
|
||||||
|
|
||||||
|
matches = [c for c in rows if _eligible(c)]
|
||||||
|
page = matches[:limit]
|
||||||
|
|
||||||
|
next_cursor = _next_cursor(rows, matches, page, window)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"calls": [with_playback_url(c) for c in page],
|
||||||
|
"next_cursor": next_cursor,
|
||||||
|
"scanned": len(rows),
|
||||||
|
"matched": len(matches),
|
||||||
|
"window_exhausted": len(rows) == window,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/eval-stats")
|
||||||
|
async def eval_stats(decoded: dict = Depends(require_admin_token)):
|
||||||
|
"""
|
||||||
|
How many calls have a human-verified transcript, and the WER of the raw
|
||||||
|
and corrected machine transcripts against them (server-26#163).
|
||||||
|
|
||||||
|
Whole-collection scan, matching `GET /calls` (list_calls above) rather
|
||||||
|
than the bounded-window pattern the paged routes use: the eval set this
|
||||||
|
is measuring is built a few calls at a time and expected to stay small
|
||||||
|
(tens to hundreds), so a full scan filtered in Python is the honest
|
||||||
|
answer rather than a windowed guess that could miss eval'd calls sitting
|
||||||
|
outside a recency window.
|
||||||
|
"""
|
||||||
|
from app.internal.wer import word_error_rate
|
||||||
|
|
||||||
|
org_id = await resolve_caller_org_id(decoded)
|
||||||
|
filters = {"org_id": org_id} if org_id is not None else {}
|
||||||
|
calls = await fstore.collection_list("calls", **filters)
|
||||||
|
|
||||||
|
raw_wers: list[float] = []
|
||||||
|
corrected_wers: list[float] = []
|
||||||
|
for c in calls:
|
||||||
|
ref = c.get("eval_transcript")
|
||||||
|
if not ref:
|
||||||
|
continue
|
||||||
|
raw = c.get("transcript") or ""
|
||||||
|
corrected = c.get("transcript_corrected") or raw
|
||||||
|
raw_wer = word_error_rate(ref, raw)
|
||||||
|
corrected_wer = word_error_rate(ref, corrected)
|
||||||
|
if raw_wer is not None:
|
||||||
|
raw_wers.append(raw_wer)
|
||||||
|
if corrected_wer is not None:
|
||||||
|
corrected_wers.append(corrected_wer)
|
||||||
|
|
||||||
|
def _avg(xs: list[float]) -> Optional[float]:
|
||||||
|
return round(sum(xs) / len(xs), 4) if xs else None
|
||||||
|
|
||||||
|
return {
|
||||||
|
"eval_count": len(raw_wers),
|
||||||
|
"raw_wer": _avg(raw_wers),
|
||||||
|
"corrected_wer": _avg(corrected_wers),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{call_id}")
|
@router.get("/{call_id}")
|
||||||
async def get_call(call_id: str, decoded: dict = Depends(require_service_or_firebase_token)):
|
async def get_call(call_id: str, decoded: dict = Depends(require_service_or_firebase_token)):
|
||||||
call = await fstore.doc_get("calls", call_id)
|
call = await fstore.doc_get("calls", call_id)
|
||||||
@@ -260,6 +414,15 @@ async def patch_transcript(
|
|||||||
"vehicles": [],
|
"vehicles": [],
|
||||||
"embedding": None,
|
"embedding": None,
|
||||||
})
|
})
|
||||||
|
# server-26#96/#114 review: doc_set(merge=True) can only ADD/overwrite keys
|
||||||
|
# in a nested map, never remove one, so the fields above get cleared but a
|
||||||
|
# prior `scenes` map would survive re-extraction forever. A call corrected
|
||||||
|
# from 3 scenes down to 1 would keep scenes.1/scenes.2 with pre-correction
|
||||||
|
# transcripts and incident_ids -- corrupting the exact per-scene tally #96
|
||||||
|
# exists to make trustworthy, and re-feeding stale text into #114's
|
||||||
|
# summarizer fix if a stale scene's incident_id still names a real
|
||||||
|
# incident. Must be a real delete, not a merge over an empty map.
|
||||||
|
await fstore.doc_update("calls", call_id, {"scenes": fstore.DELETE_FIELD})
|
||||||
|
|
||||||
# Unlink from ALL current incidents so re-correlation starts clean.
|
# Unlink from ALL current incidents so re-correlation starts clean.
|
||||||
# Handles both old single incident_id and new incident_ids list.
|
# Handles both old single incident_id and new incident_ids list.
|
||||||
@@ -304,3 +467,29 @@ async def patch_transcript(
|
|||||||
preserve_transcript_correction=True,
|
preserve_transcript_correction=True,
|
||||||
)
|
)
|
||||||
return {"ok": True, "call_id": call_id}
|
return {"ok": True, "call_id": call_id}
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/{call_id}/eval-transcript")
|
||||||
|
async def put_eval_transcript(
|
||||||
|
call_id: str,
|
||||||
|
body: EvalTranscriptUpdate,
|
||||||
|
decoded: dict = Depends(require_admin_token),
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Record a human-verified reference transcript for the STT eval harness
|
||||||
|
(server-26#163). Pure data capture — unlike `PATCH /{call_id}/transcript`
|
||||||
|
above, this never touches `transcript`/`transcript_corrected`, never
|
||||||
|
re-runs extraction, never unlinks incidents, and never feeds the
|
||||||
|
vocabulary learner. It exists to MEASURE the pipeline's output, not to
|
||||||
|
change it; the two must not share a code path.
|
||||||
|
"""
|
||||||
|
call = await fstore.doc_get("calls", call_id)
|
||||||
|
if not call:
|
||||||
|
raise HTTPException(404, f"Call '{call_id}' not found.")
|
||||||
|
|
||||||
|
await fstore.doc_set("calls", call_id, {
|
||||||
|
"eval_transcript": body.text,
|
||||||
|
"eval_transcript_by": decoded.get("email") or decoded.get("uid"),
|
||||||
|
"eval_transcript_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
})
|
||||||
|
return {"ok": True, "call_id": call_id}
|
||||||
|
|||||||
@@ -195,6 +195,7 @@ async def assign_system(
|
|||||||
class NodeUpdateBody(BaseModel):
|
class NodeUpdateBody(BaseModel):
|
||||||
node_type: Optional[str] = None
|
node_type: Optional[str] = None
|
||||||
enforce_override_timeout: Optional[bool] = None
|
enforce_override_timeout: Optional[bool] = None
|
||||||
|
secondary_sdr_mode: Optional[str] = None # none | adsb | ais | op25_2
|
||||||
|
|
||||||
|
|
||||||
@router.patch("/{node_id}")
|
@router.patch("/{node_id}")
|
||||||
@@ -227,6 +228,8 @@ async def update_node(
|
|||||||
}
|
}
|
||||||
if updated_node.get("ppm_override") is not None:
|
if updated_node.get("ppm_override") is not None:
|
||||||
push_payload["ppm_override"] = updated_node["ppm_override"]
|
push_payload["ppm_override"] = updated_node["ppm_override"]
|
||||||
|
if updated_node.get("secondary_sdr_mode") is not None:
|
||||||
|
push_payload["secondary_sdr_mode"] = updated_node["secondary_sdr_mode"]
|
||||||
mqtt_handler.push_config(node_id, push_payload)
|
mqtt_handler.push_config(node_id, push_payload)
|
||||||
|
|
||||||
return {"ok": True}
|
return {"ok": True}
|
||||||
|
|||||||
@@ -24,6 +24,10 @@ class TenCodesBody(BaseModel):
|
|||||||
ten_codes: Dict[str, str]
|
ten_codes: Dict[str, str]
|
||||||
|
|
||||||
|
|
||||||
|
class UnitFormatBody(BaseModel):
|
||||||
|
unit_format_hint: str
|
||||||
|
|
||||||
|
|
||||||
class PendingTermBody(BaseModel):
|
class PendingTermBody(BaseModel):
|
||||||
talkgroup_id: int
|
talkgroup_id: int
|
||||||
term: str
|
term: str
|
||||||
@@ -155,6 +159,38 @@ async def update_ten_codes(
|
|||||||
return {"ok": True, "ten_codes": body.ten_codes}
|
return {"ok": True, "ten_codes": body.ten_codes}
|
||||||
|
|
||||||
|
|
||||||
|
# ── Unit ID format hint ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
@router.get("/{system_id}/unit-format")
|
||||||
|
async def get_unit_format(system_id: str):
|
||||||
|
"""Return the unit-ID format hint for a system."""
|
||||||
|
system = await fstore.doc_get("systems", system_id)
|
||||||
|
if not system:
|
||||||
|
raise HTTPException(404, f"System '{system_id}' not found.")
|
||||||
|
return {"unit_format_hint": system.get("unit_format_hint") or ""}
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/{system_id}/unit-format")
|
||||||
|
async def update_unit_format(
|
||||||
|
system_id: str,
|
||||||
|
body: UnitFormatBody,
|
||||||
|
_: dict = Depends(require_admin_token),
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Set the free-text unit-ID format hint fed into intelligence.py's
|
||||||
|
extraction prompt (server-26#<pending>). Departments have no shared unit
|
||||||
|
ID convention — e.g. "5-David"/bare "David" vs "SAM-1"/"airport-3" — and
|
||||||
|
the extraction prompt has no way to recognise a format it hasn't been
|
||||||
|
told about. Own route for the same reason ten-codes has one: not carried
|
||||||
|
by the systems form, so folding it into PUT /{id} would wipe it.
|
||||||
|
"""
|
||||||
|
existing = await fstore.doc_get("systems", system_id)
|
||||||
|
if not existing:
|
||||||
|
raise HTTPException(404, f"System '{system_id}' not found.")
|
||||||
|
await fstore.doc_update("systems", system_id, {"unit_format_hint": body.unit_format_hint})
|
||||||
|
return {"ok": True, "unit_format_hint": body.unit_format_hint}
|
||||||
|
|
||||||
|
|
||||||
# ── Area context ──────────────────────────────────────────────────────────────
|
# ── Area context ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@router.get("/{system_id}/area-context")
|
@router.get("/{system_id}/area-context")
|
||||||
|
|||||||
@@ -0,0 +1,125 @@
|
|||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import List, Optional
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
from app.internal import firestore as fstore
|
||||||
|
from app.internal.auth import require_node_service_or_firebase_token
|
||||||
|
from app.internal.logger import logger
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/telemetry", tags=["telemetry"])
|
||||||
|
|
||||||
|
|
||||||
|
class AircraftReport(BaseModel):
|
||||||
|
icao: str
|
||||||
|
callsign: Optional[str] = None
|
||||||
|
lat: Optional[float] = None
|
||||||
|
lon: Optional[float] = None
|
||||||
|
altitude_ft: Optional[float] = None
|
||||||
|
ground_speed_kt: Optional[float] = None
|
||||||
|
track_deg: Optional[float] = None
|
||||||
|
|
||||||
|
|
||||||
|
class AdsbUploadBody(BaseModel):
|
||||||
|
aircraft: List[AircraftReport]
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/adsb")
|
||||||
|
async def upload_adsb(
|
||||||
|
body: AdsbUploadBody,
|
||||||
|
decoded: dict = Depends(require_node_service_or_firebase_token),
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Node-initiated: a second-SDR ADS-B decoder (node-26#9) periodically posts
|
||||||
|
its current aircraft snapshot here. One doc per icao, last-seen-wins —
|
||||||
|
this is a live-map overlay, not a flight history.
|
||||||
|
"""
|
||||||
|
node_id = decoded.get("node_id")
|
||||||
|
if not node_id:
|
||||||
|
raise HTTPException(400, "This endpoint requires node identity, not a service/admin token.")
|
||||||
|
|
||||||
|
node = await fstore.doc_get_cached("nodes", node_id)
|
||||||
|
org_id = node.get("org_id") if node else None
|
||||||
|
now = datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
writes = []
|
||||||
|
for ac in body.aircraft:
|
||||||
|
if not ac.icao:
|
||||||
|
continue
|
||||||
|
doc = {
|
||||||
|
"icao": ac.icao,
|
||||||
|
"node_id": node_id,
|
||||||
|
"callsign": ac.callsign,
|
||||||
|
"lat": ac.lat,
|
||||||
|
"lon": ac.lon,
|
||||||
|
"altitude_ft": ac.altitude_ft,
|
||||||
|
"ground_speed_kt": ac.ground_speed_kt,
|
||||||
|
"track_deg": ac.track_deg,
|
||||||
|
"last_seen": now,
|
||||||
|
}
|
||||||
|
if org_id:
|
||||||
|
doc["org_id"] = org_id
|
||||||
|
writes.append(("aircraft", ac.icao, doc))
|
||||||
|
|
||||||
|
for collection, doc_id, doc in writes:
|
||||||
|
try:
|
||||||
|
await fstore.doc_set(collection, doc_id, doc, merge=True)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Failed to upsert {collection}/{doc_id} from node {node_id}: {e}")
|
||||||
|
|
||||||
|
return {"ok": True, "count": len(writes)}
|
||||||
|
|
||||||
|
|
||||||
|
class VesselReport(BaseModel):
|
||||||
|
mmsi: str
|
||||||
|
name: Optional[str] = None
|
||||||
|
lat: Optional[float] = None
|
||||||
|
lon: Optional[float] = None
|
||||||
|
speed_kt: Optional[float] = None
|
||||||
|
heading_deg: Optional[float] = None
|
||||||
|
|
||||||
|
|
||||||
|
class AisUploadBody(BaseModel):
|
||||||
|
vessels: List[VesselReport]
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/ais")
|
||||||
|
async def upload_ais(
|
||||||
|
body: AisUploadBody,
|
||||||
|
decoded: dict = Depends(require_node_service_or_firebase_token),
|
||||||
|
):
|
||||||
|
"""Same shape as /telemetry/adsb, one doc per mmsi in `vessels`."""
|
||||||
|
node_id = decoded.get("node_id")
|
||||||
|
if not node_id:
|
||||||
|
raise HTTPException(400, "This endpoint requires node identity, not a service/admin token.")
|
||||||
|
|
||||||
|
node = await fstore.doc_get_cached("nodes", node_id)
|
||||||
|
org_id = node.get("org_id") if node else None
|
||||||
|
now = datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
writes = []
|
||||||
|
for v in body.vessels:
|
||||||
|
if not v.mmsi:
|
||||||
|
continue
|
||||||
|
doc = {
|
||||||
|
"mmsi": v.mmsi,
|
||||||
|
"node_id": node_id,
|
||||||
|
"name": v.name,
|
||||||
|
"lat": v.lat,
|
||||||
|
"lon": v.lon,
|
||||||
|
"speed_kt": v.speed_kt,
|
||||||
|
"heading_deg": v.heading_deg,
|
||||||
|
"last_seen": now,
|
||||||
|
}
|
||||||
|
if org_id:
|
||||||
|
doc["org_id"] = org_id
|
||||||
|
writes.append(("vessels", v.mmsi, doc))
|
||||||
|
|
||||||
|
for collection, doc_id, doc in writes:
|
||||||
|
try:
|
||||||
|
await fstore.doc_set(collection, doc_id, doc, merge=True)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Failed to upsert {collection}/{doc_id} from node {node_id}: {e}")
|
||||||
|
|
||||||
|
return {"ok": True, "count": len(writes)}
|
||||||
@@ -100,6 +100,98 @@ async def upload_call_audio(
|
|||||||
return {"url": gcs_uri}
|
return {"url": gcs_uri}
|
||||||
|
|
||||||
|
|
||||||
|
# server-26#115 — the consensus LLM-orphan gate only fires when the call is
|
||||||
|
# genuinely substanceless. The earlier version tested `rules_decision["corr_debug"]`
|
||||||
|
# for a "positive signal", but corr_debug is EMPTY at preview time for
|
||||||
|
# action=="new" (corr_path:"new" is written at APPLY time), so that test was
|
||||||
|
# always False and the gate dropped real events — a major "extinguishing fire",
|
||||||
|
# geocoded calls, pursuit updates. The substance test now runs against `ctx`,
|
||||||
|
# which is fully populated at preview time.
|
||||||
|
|
||||||
|
|
||||||
|
def _recent_incident_on_same_talkgroup(ctx: dict) -> bool:
|
||||||
|
"""
|
||||||
|
True when a recent incident is running on this call's own system +
|
||||||
|
talkgroup, within `settings.tg_dispatch_thin_idle_minutes` (5 min) —
|
||||||
|
applied uniformly regardless of the talkgroup's name (server-26#134).
|
||||||
|
Covers "unit dispatched, thin ack 10-30s later": the ack has no
|
||||||
|
substance of its own but plainly belongs to the job just opened.
|
||||||
|
|
||||||
|
Reads ctx["recent"] (the rules engine's own candidate list — no extra
|
||||||
|
Firestore read). That list is status=="active" incidents only, so an
|
||||||
|
already-resolved or capacity-capped same-talkgroup incident won't be
|
||||||
|
seen here even if chronologically recent (server-26#115, unresolved —
|
||||||
|
would need a dedicated non-status-filtered query).
|
||||||
|
|
||||||
|
Whether this limitation explains the 2/24 unexplained gate misses in the
|
||||||
|
window #3 measurement is UNANSWERED, not confirmed either way — a prior
|
||||||
|
pass here claimed a "confirmed explanation" for both that turned out to
|
||||||
|
be self-contradictory. Read `corr_gate_veto` (written to corr_debug on
|
||||||
|
every escalation of this exact disagreement shape — see the caller) in
|
||||||
|
the next measurement window instead of guessing from the raw dump again.
|
||||||
|
# TODO(server-26#115): add a talkgroup-scoped incident lookup (any
|
||||||
|
# status, no capacity filter) if a future measurement window pins a real
|
||||||
|
# gate miss on a resolved/capped same-talkgroup incident.
|
||||||
|
"""
|
||||||
|
from app.internal.incident_correlator import _idle_gate_minutes
|
||||||
|
|
||||||
|
tg_id = ctx.get("talkgroup_id")
|
||||||
|
system_id = ctx.get("system_id")
|
||||||
|
if tg_id is None or not system_id:
|
||||||
|
return False
|
||||||
|
tg_str = str(tg_id)
|
||||||
|
now = ctx.get("now") or datetime.now(timezone.utc)
|
||||||
|
idle_limit = settings.tg_dispatch_thin_idle_minutes
|
||||||
|
for inc in ctx.get("recent") or []:
|
||||||
|
if system_id not in (inc.get("system_ids") or []):
|
||||||
|
continue
|
||||||
|
if tg_str not in (inc.get("talkgroup_ids") or []):
|
||||||
|
continue
|
||||||
|
if _idle_gate_minutes(inc, now) <= idle_limit:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _call_is_substanceless(ctx: dict) -> tuple[bool, Optional[str]]:
|
||||||
|
"""
|
||||||
|
True when the call carries nothing that marks it as a real event:
|
||||||
|
• no resolved incident_type and not a reassignment, AND
|
||||||
|
• severity is not moderate/major, AND
|
||||||
|
• no vehicle, geocode or tag (incident_correlator.has_event_substance —
|
||||||
|
the same predicate the incident-creation gate uses), AND
|
||||||
|
• no recent incident already running on the same talkgroup.
|
||||||
|
Only then may the LLM-orphan gate drop the call without a tiebreak.
|
||||||
|
|
||||||
|
Returns (substanceless, veto_reason). veto_reason names whichever
|
||||||
|
condition kept the tiebreak alive ("type" | "reassignment" | "severity" |
|
||||||
|
"substance" | "recent_tg"), or None when the call is substanceless. The
|
||||||
|
caller writes this into corr_debug on the escalation path so a live
|
||||||
|
measurement window can see *why* each llm=orphan/rules=new call escaped
|
||||||
|
the gate instead of inferring it after the fact from the raw dump —
|
||||||
|
exactly the guesswork that produced a wrong "confirmed explanation" for
|
||||||
|
2 window-#3 misses on the first pass of this fix.
|
||||||
|
"""
|
||||||
|
from app.internal import incident_correlator
|
||||||
|
|
||||||
|
# The incident-creation gate skips the has_event_substance check entirely
|
||||||
|
# when a type resolved (incident_correlator._run_decision ~:1397), so a
|
||||||
|
# typed call — fire/medical/etc. — opens an incident on substance we do not
|
||||||
|
# re-check here. reassignment=True is dispatch pulling a unit onto a NEW
|
||||||
|
# job (units are blanked at :296 for exactly that reason): the strongest
|
||||||
|
# new-incident signal in the pipeline. Either one means "keep the tiebreak".
|
||||||
|
if ctx.get("incident_type"):
|
||||||
|
return False, "type"
|
||||||
|
if ctx.get("reassignment"):
|
||||||
|
return False, "reassignment"
|
||||||
|
if (ctx.get("call_severity") or "routine") in ("moderate", "major"):
|
||||||
|
return False, "severity"
|
||||||
|
if incident_correlator.has_event_substance(ctx):
|
||||||
|
return False, "substance"
|
||||||
|
if _recent_incident_on_same_talkgroup(ctx):
|
||||||
|
return False, "recent_tg"
|
||||||
|
return True, None
|
||||||
|
|
||||||
|
|
||||||
async def _correlate_with_consensus(
|
async def _correlate_with_consensus(
|
||||||
call_id: str,
|
call_id: str,
|
||||||
node_id: str,
|
node_id: str,
|
||||||
@@ -116,6 +208,8 @@ async def _correlate_with_consensus(
|
|||||||
reassignment: bool = False,
|
reassignment: bool = False,
|
||||||
embedding: Optional[list] = None,
|
embedding: Optional[list] = None,
|
||||||
severity: Optional[str] = None,
|
severity: Optional[str] = None,
|
||||||
|
transcript: Optional[str] = None,
|
||||||
|
scene_index: int = 0,
|
||||||
) -> Optional[str]:
|
) -> Optional[str]:
|
||||||
"""
|
"""
|
||||||
Consensus correlator: runs the rules engine and the cheap LLM in sequence.
|
Consensus correlator: runs the rules engine and the cheap LLM in sequence.
|
||||||
@@ -124,6 +218,11 @@ async def _correlate_with_consensus(
|
|||||||
|
|
||||||
Falls back to rules-only when GEMINI_API_KEY is absent, the call is
|
Falls back to rules-only when GEMINI_API_KEY is absent, the call is
|
||||||
content-free (thin), or any LLM call fails.
|
content-free (thin), or any LLM call fails.
|
||||||
|
|
||||||
|
``scene_index`` (server-26#96) — which scene of the call this is, from the
|
||||||
|
caller's ``enumerate(scenes)`` loop. Threaded through so the call doc's
|
||||||
|
per-scene ``scenes`` map records this scene's own corr_debug/transcript
|
||||||
|
instead of colliding with every other scene's write on the flat fields.
|
||||||
"""
|
"""
|
||||||
from app.internal import incident_correlator, llm_correlator
|
from app.internal import incident_correlator, llm_correlator
|
||||||
|
|
||||||
@@ -133,7 +232,8 @@ async def _correlate_with_consensus(
|
|||||||
tags=tags, incident_type=incident_type, location=location,
|
tags=tags, incident_type=incident_type, location=location,
|
||||||
location_coords=location_coords, units=units, vehicles=vehicles,
|
location_coords=location_coords, units=units, vehicles=vehicles,
|
||||||
cleared_units=cleared_units, reassignment=reassignment,
|
cleared_units=cleared_units, reassignment=reassignment,
|
||||||
embedding=embedding, severity=severity,
|
embedding=embedding, severity=severity, transcript=transcript,
|
||||||
|
scene_index=scene_index,
|
||||||
)
|
)
|
||||||
ctx = preview["ctx"]
|
ctx = preview["ctx"]
|
||||||
rules_decision = preview["decision"]
|
rules_decision = preview["decision"]
|
||||||
@@ -150,6 +250,37 @@ async def _correlate_with_consensus(
|
|||||||
rules_decision["corr_debug"]["corr_llm_reasoning"] = llm_decision.get("reasoning", "")
|
rules_decision["corr_debug"]["corr_llm_reasoning"] = llm_decision.get("reasoning", "")
|
||||||
return await incident_correlator.apply_correlation(preview)
|
return await incident_correlator.apply_correlation(preview)
|
||||||
|
|
||||||
|
# server-26#115 — LLM-orphan gate.
|
||||||
|
# When the cheap LLM says `orphan`, the rules engine says `new`, and the call
|
||||||
|
# is genuinely substanceless (routine severity, no vehicle/geocode/tag, and
|
||||||
|
# no incident already running on this talkgroup), resolve to `orphan` and DO
|
||||||
|
# NOT pay for the smart tiebreaker. A bare rules `new` there means only
|
||||||
|
# "nothing to link to" — trivially true for radio housekeeping (check-ins,
|
||||||
|
# roll call, 10-8/10-98) — and the tiebreaker rubber-stamped it ~21/21 of the
|
||||||
|
# time on exactly this disagreement (CORRELATION_REVIEW_0907b.md). Any real
|
||||||
|
# signal (severity, coords, tags, a live same-talkgroup incident) still
|
||||||
|
# escalates, so an event the LLM misreads as orphan is not lost.
|
||||||
|
is_orphan_vs_new = llm_decision["action"] == "orphan" and rules_decision["action"] == "new"
|
||||||
|
substanceless, gate_veto_reason = _call_is_substanceless(ctx) if is_orphan_vs_new else (False, None)
|
||||||
|
if is_orphan_vs_new and substanceless:
|
||||||
|
logger.info(
|
||||||
|
f"Consensus gate for call {call_id}: llm=orphan vs rules=new and call "
|
||||||
|
f"is substanceless — resolving orphan, skipping tiebreak"
|
||||||
|
)
|
||||||
|
gated = {
|
||||||
|
"action": "orphan",
|
||||||
|
"matched_incident": None,
|
||||||
|
"incident_type": None,
|
||||||
|
"corr_debug": dict(rules_decision.get("corr_debug") or {}),
|
||||||
|
}
|
||||||
|
gated["corr_debug"].update({
|
||||||
|
"corr_consensus": "llm_orphan_gate",
|
||||||
|
"corr_rules_action": rules_decision["action"],
|
||||||
|
"corr_llm_action": llm_decision["action"],
|
||||||
|
"corr_llm_reasoning": llm_decision.get("reasoning", ""),
|
||||||
|
})
|
||||||
|
return await incident_correlator.apply_correlation({"decision": gated, "ctx": ctx})
|
||||||
|
|
||||||
# Disagree — escalate to the smarter tiebreaker.
|
# Disagree — escalate to the smarter tiebreaker.
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Consensus disagreement for call {call_id}: "
|
f"Consensus disagreement for call {call_id}: "
|
||||||
@@ -159,6 +290,12 @@ async def _correlate_with_consensus(
|
|||||||
final["corr_debug"]["corr_consensus"] = "tiebreak"
|
final["corr_debug"]["corr_consensus"] = "tiebreak"
|
||||||
final["corr_debug"]["corr_rules_action"] = rules_decision["action"]
|
final["corr_debug"]["corr_rules_action"] = rules_decision["action"]
|
||||||
final["corr_debug"]["corr_llm_action"] = llm_decision["action"]
|
final["corr_debug"]["corr_llm_action"] = llm_decision["action"]
|
||||||
|
if is_orphan_vs_new:
|
||||||
|
# server-26#115 — record *why* the llm=orphan/rules=new gate stood
|
||||||
|
# down instead of leaving a future measurement window to guess it
|
||||||
|
# from the raw dump (which produced a wrong "confirmed explanation"
|
||||||
|
# for 2/24 misses the first time around).
|
||||||
|
final["corr_debug"]["corr_gate_veto"] = gate_veto_reason
|
||||||
return await incident_correlator.apply_correlation({"decision": final, "ctx": ctx})
|
return await incident_correlator.apply_correlation({"decision": final, "ctx": ctx})
|
||||||
|
|
||||||
|
|
||||||
@@ -204,7 +341,10 @@ async def _run_extraction_pipeline(
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Step 3: Correlate each scene to an incident independently.
|
# Step 3: Correlate each scene to an incident independently.
|
||||||
for scene in scenes:
|
# server-26#96: scene_index is threaded through so each scene's
|
||||||
|
# corr_debug/transcript lands in its own entry of the call doc's
|
||||||
|
# `scenes` map instead of clobbering every other scene's write.
|
||||||
|
for scene_index, scene in enumerate(scenes):
|
||||||
all_tags.extend(scene["tags"])
|
all_tags.extend(scene["tags"])
|
||||||
# When dispatch is pulling a unit to a NEW call (reassignment), suppress unit
|
# When dispatch is pulling a unit to a NEW call (reassignment), suppress unit
|
||||||
# overlap so the new scene doesn't chain into the unit's previous incident.
|
# overlap so the new scene doesn't chain into the unit's previous incident.
|
||||||
@@ -226,6 +366,8 @@ async def _run_extraction_pipeline(
|
|||||||
reassignment=is_reassignment,
|
reassignment=is_reassignment,
|
||||||
embedding=scene.get("embedding"),
|
embedding=scene.get("embedding"),
|
||||||
severity=scene.get("severity"),
|
severity=scene.get("severity"),
|
||||||
|
transcript=scene.get("transcript"),
|
||||||
|
scene_index=scene_index,
|
||||||
)
|
)
|
||||||
if incident_id and incident_id not in incident_ids:
|
if incident_id and incident_id not in incident_ids:
|
||||||
incident_ids.append(incident_id)
|
incident_ids.append(incident_id)
|
||||||
@@ -271,6 +413,25 @@ async def _run_intelligence_pipeline(
|
|||||||
"""
|
"""
|
||||||
from app.internal import transcription, intelligence, incident_correlator, alerter, talkgroups
|
from app.internal import transcription, intelligence, incident_correlator, alerter, talkgroups
|
||||||
|
|
||||||
|
# server-26#131: mark that real-time processing has started for this call
|
||||||
|
# BEFORE any of the slow steps below (STT, scene extraction, correlation).
|
||||||
|
# The re-correlation sweep (internal/recorrelation_sweep.py) scans for
|
||||||
|
# calls that still look orphaned within a wide window (recorrelation_scan_
|
||||||
|
# minutes, default 60) — with no guard here, a call whose real-time
|
||||||
|
# pipeline is still mid-flight (still transcribing, still waiting on a
|
||||||
|
# Gemini call) has no incident_id/corr_path written yet, so the sweep's
|
||||||
|
# orphan filter can't tell "never processed" from "processing right now"
|
||||||
|
# and correlates it a second time, independently, sometimes landing on a
|
||||||
|
# different incident than the real-time path — the exact duplicate-link
|
||||||
|
# bug #131 found (same call in two incidents' call_ids, ~2% of linked
|
||||||
|
# calls). Best-effort: a write failure here must not abort the pipeline.
|
||||||
|
try:
|
||||||
|
await fstore.doc_set("calls", call_id, {
|
||||||
|
"intelligence_started_at": datetime.now(timezone.utc).isoformat()
|
||||||
|
})
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Could not mark intelligence_started_at for call {call_id}: {e}")
|
||||||
|
|
||||||
# The node only sends talkgroup_name when OP25 had it in the loaded tags
|
# The node only sends talkgroup_name when OP25 had it in the loaded tags
|
||||||
# file, so it arrives empty for exactly the talkgroups C2 can name from the
|
# file, so it arrives empty for exactly the talkgroups C2 can name from the
|
||||||
# system config. Resolve it once, here, at the single funnel both /upload
|
# system config. Resolve it once, here, at the single funnel both /upload
|
||||||
@@ -323,7 +484,10 @@ async def _run_intelligence_pipeline(
|
|||||||
incident_ids: list[str] = []
|
incident_ids: list[str] = []
|
||||||
all_tags: list[str] = []
|
all_tags: list[str] = []
|
||||||
if _flag("correlation_enabled"):
|
if _flag("correlation_enabled"):
|
||||||
for scene in scenes:
|
# server-26#96: scene_index is threaded through so each scene's
|
||||||
|
# corr_debug/transcript lands in its own entry of the call doc's
|
||||||
|
# `scenes` map instead of clobbering every other scene's write.
|
||||||
|
for scene_index, scene in enumerate(scenes):
|
||||||
all_tags.extend(scene["tags"])
|
all_tags.extend(scene["tags"])
|
||||||
is_reassignment = bool(scene.get("reassignment"))
|
is_reassignment = bool(scene.get("reassignment"))
|
||||||
corr_units = [] if is_reassignment else scene.get("units")
|
corr_units = [] if is_reassignment else scene.get("units")
|
||||||
@@ -343,6 +507,8 @@ async def _run_intelligence_pipeline(
|
|||||||
reassignment=is_reassignment,
|
reassignment=is_reassignment,
|
||||||
embedding=scene.get("embedding"),
|
embedding=scene.get("embedding"),
|
||||||
severity=scene.get("severity"),
|
severity=scene.get("severity"),
|
||||||
|
transcript=scene.get("transcript"),
|
||||||
|
scene_index=scene_index,
|
||||||
)
|
)
|
||||||
if incident_id and incident_id not in incident_ids:
|
if incident_id and incident_id not in incident_ids:
|
||||||
incident_ids.append(incident_id)
|
incident_ids.append(incident_id)
|
||||||
@@ -356,11 +522,18 @@ async def _run_intelligence_pipeline(
|
|||||||
|
|
||||||
# Correlator also runs for calls with no scenes (unclassified) to attempt
|
# Correlator also runs for calls with no scenes (unclassified) to attempt
|
||||||
# talkgroup-based linking even when no transcript could be produced.
|
# talkgroup-based linking even when no transcript could be produced.
|
||||||
# Skip when extraction flagged the call — garbage or too-short transcripts
|
# transcript_too_short (<=5 words: "10-8", "show me clear", a unit
|
||||||
# carry no signal and would only attach spuriously via the thin path.
|
# check-in) still carries a real transcript and talkgroup — exactly the
|
||||||
|
# brief follow-up/clearance traffic an incident needs, and the thin-path
|
||||||
|
# merge below already requires a same-talkgroup, recently-active
|
||||||
|
# incident before attaching anything, same guard already trusted for
|
||||||
|
# no-transcript calls. Previously excluded here, so these calls never
|
||||||
|
# attached to anything at all. garbage_transcript (Whisper
|
||||||
|
# hallucination) has no real content behind it and stays excluded.
|
||||||
if not scenes:
|
if not scenes:
|
||||||
_call_doc = await fstore.doc_get("calls", call_id)
|
_call_doc = await fstore.doc_get("calls", call_id)
|
||||||
if not (_call_doc or {}).get("skip_reason"):
|
skip_reason = (_call_doc or {}).get("skip_reason")
|
||||||
|
if not skip_reason or skip_reason == "transcript_too_short":
|
||||||
incident_id = await _correlate_with_consensus(
|
incident_id = await _correlate_with_consensus(
|
||||||
call_id=call_id,
|
call_id=call_id,
|
||||||
node_id=node_id,
|
node_id=node_id,
|
||||||
|
|||||||
@@ -34,6 +34,10 @@ except ModuleNotFoundError:
|
|||||||
# into dicts that tests compare against, and a MagicMock compares unequal
|
# into dicts that tests compare against, and a MagicMock compares unequal
|
||||||
# to itself across attribute accesses.
|
# to itself across attribute accesses.
|
||||||
_fs.SERVER_TIMESTAMP = "__SERVER_TIMESTAMP__"
|
_fs.SERVER_TIMESTAMP = "__SERVER_TIMESTAMP__"
|
||||||
|
# Same reasoning as SERVER_TIMESTAMP above: a distinct sentinel, not a
|
||||||
|
# MagicMock, so `fstore.DELETE_FIELD is fs.DELETE_FIELD` and dict/`is`
|
||||||
|
# comparisons against it in tests (server-26#96/#114, PR #132) behave.
|
||||||
|
_fs.DELETE_FIELD = "__DELETE_FIELD__"
|
||||||
|
|
||||||
_auth = ModuleType("firebase_admin.auth")
|
_auth = ModuleType("firebase_admin.auth")
|
||||||
_auth.verify_id_token = MagicMock()
|
_auth.verify_id_token = MagicMock()
|
||||||
|
|||||||
@@ -89,3 +89,75 @@ async def test_debug_correlation_llm_fields_absent_when_rules_only():
|
|||||||
assert detail["corr_consensus"] == "rules_only"
|
assert detail["corr_consensus"] == "rules_only"
|
||||||
assert detail["corr_llm_reasoning"] is None
|
assert detail["corr_llm_reasoning"] is None
|
||||||
assert detail["corr_llm_action"] is None
|
assert detail["corr_llm_action"] is None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# server-26#96 — the summary tally must count per-scene decisions, not the
|
||||||
|
# one blended flat record a multi-scene call used to leave behind.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_debug_correlation_exposes_scenes_and_tallies_each_as_its_own_datapoint():
|
||||||
|
"""A 2-scene call: the flat fields still show last-scene-wins (unchanged
|
||||||
|
behaviour for old readers), but the summary tally must see two distinct
|
||||||
|
corr_path/corr_consensus data points, not one blend."""
|
||||||
|
call = {
|
||||||
|
"call_id": "call-1",
|
||||||
|
# Flat fields — last scene wins, kept as-is for backward compat.
|
||||||
|
"corr_path": "slow",
|
||||||
|
"corr_consensus": "tiebreak",
|
||||||
|
"scenes": {
|
||||||
|
"0": {
|
||||||
|
"transcript": "scene zero",
|
||||||
|
"incident_id": "inc-1",
|
||||||
|
"corr_debug": {"corr_path": "new", "corr_consensus": "agreed"},
|
||||||
|
},
|
||||||
|
"1": {
|
||||||
|
"transcript": "scene one",
|
||||||
|
"incident_id": "inc-1",
|
||||||
|
"corr_debug": {"corr_path": "slow", "corr_consensus": "tiebreak"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
result = await _run([_incident(["call-1"])], {"call-1": call})
|
||||||
|
|
||||||
|
detail = result["incidents"][0]["calls_detail"][0]
|
||||||
|
assert detail["corr_path"] == "slow" # flat field: last scene wins
|
||||||
|
assert len(detail["scenes"]) == 2
|
||||||
|
assert detail["scenes"][0]["corr_path"] == "new"
|
||||||
|
assert detail["scenes"][1]["corr_path"] == "slow"
|
||||||
|
|
||||||
|
summary = result["summary"]
|
||||||
|
assert summary["linked_call_count"] == 1 # still one CALL
|
||||||
|
assert summary["scene_decision_count"] == 2 # but two DECISIONS
|
||||||
|
assert summary["corr_path"] == {"new": 1, "slow": 1}
|
||||||
|
assert summary["corr_consensus"] == {"agreed": 1, "tiebreak": 1}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_debug_correlation_tally_falls_back_for_single_scene_call():
|
||||||
|
"""A plain single-scene call has no `scenes` field at all — the tally
|
||||||
|
must fall back to its flat fields as one data point, same as pre-#96."""
|
||||||
|
call = {"call_id": "call-2", "corr_path": "fast/single", "corr_consensus": "rules_only"}
|
||||||
|
result = await _run([_incident(["call-2"])], {"call-2": call})
|
||||||
|
|
||||||
|
detail = result["incidents"][0]["calls_detail"][0]
|
||||||
|
assert detail["scenes"] is None
|
||||||
|
|
||||||
|
summary = result["summary"]
|
||||||
|
assert summary["linked_call_count"] == 1
|
||||||
|
assert summary["scene_decision_count"] == 1
|
||||||
|
assert summary["corr_path"] == {"fast/single": 1}
|
||||||
|
assert summary["corr_consensus"] == {"rules_only": 1}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_debug_correlation_tally_handles_old_schema_call_with_no_scenes_field():
|
||||||
|
"""A call doc written before server-26#96 has never heard of `scenes` —
|
||||||
|
must behave identically to the single-scene case, not error."""
|
||||||
|
old_call = {"call_id": "call-3", "corr_path": "cross-tg", "corr_consensus": "agreed"}
|
||||||
|
result = await _run([_incident(["call-3"])], {"call-3": old_call})
|
||||||
|
|
||||||
|
summary = result["summary"]
|
||||||
|
assert summary["scene_decision_count"] == 1
|
||||||
|
assert summary["corr_path"] == {"cross-tg": 1}
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
"""calls._parse_ts — cursor/date bounds must reach Firestore as datetimes.
|
||||||
|
|
||||||
|
A raw ISO string compared against a timestamp field sorts by type, not time,
|
||||||
|
which made the Archive's "Load more" return the first page again.
|
||||||
|
"""
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi import HTTPException
|
||||||
|
|
||||||
|
from app.routers.calls import _next_cursor, _parse_ts
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_is_none():
|
||||||
|
assert _parse_ts(None, "cursor") is None
|
||||||
|
assert _parse_ts("", "cursor") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_z_suffix_parses_as_utc():
|
||||||
|
assert _parse_ts("2026-09-20T12:00:00Z", "date_from") == datetime(2026, 9, 20, 12, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def test_naive_is_assumed_utc():
|
||||||
|
assert _parse_ts("2026-09-20T12:00:00", "date_to").tzinfo == timezone.utc
|
||||||
|
|
||||||
|
|
||||||
|
def test_round_trips_isoformat_cursor():
|
||||||
|
dt = datetime(2026, 9, 20, 12, 30, 5, 123456, tzinfo=timezone.utc)
|
||||||
|
assert _parse_ts(dt.isoformat(), "cursor") == dt
|
||||||
|
|
||||||
|
|
||||||
|
def test_garbage_is_400():
|
||||||
|
with pytest.raises(HTTPException) as exc:
|
||||||
|
_parse_ts("yesterday", "date_from")
|
||||||
|
assert exc.value.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
# ── _next_cursor ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _rows(n):
|
||||||
|
return [{"started_at": datetime(2026, 9, 20, 12, i // 60, i % 60, tzinfo=timezone.utc)} for i in range(n)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_cursor_resumes_after_last_returned_row_when_matches_overflow():
|
||||||
|
rows = _rows(200)
|
||||||
|
page = rows[:50]
|
||||||
|
assert _next_cursor(rows, rows, page, 200) == page[-1]["started_at"].isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def test_cursor_resumes_after_window_when_page_holds_every_match():
|
||||||
|
rows = _rows(200)
|
||||||
|
matches = rows[:3]
|
||||||
|
assert _next_cursor(rows, matches, matches, 200) == rows[-1]["started_at"].isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def test_short_window_is_the_end():
|
||||||
|
rows = _rows(20)
|
||||||
|
assert _next_cursor(rows, rows[:5], rows[:5], 200) is None
|
||||||
@@ -0,0 +1,144 @@
|
|||||||
|
"""
|
||||||
|
server-26#127 — upstream dispatch-vs-chatter classifier, shadow mode.
|
||||||
|
|
||||||
|
Fixtures are real transcripts, not invented ones: pulled from
|
||||||
|
`corr_dump_9-7_0437am.json`, `corr_dump_9-7_pm.json`, `corr_dump_9-12.json`
|
||||||
|
and the hand-labeled examples in `CORRELATION_REVIEW_0907b.md` /
|
||||||
|
`CORRELATION_REVIEW_0912.md`. The "must classify False" set specifically
|
||||||
|
includes every transcript those review docs flagged as dangerous to drop —
|
||||||
|
a false positive here is a real event silently losing its scene once this
|
||||||
|
classifier ever goes live, which is a much worse failure than a missed
|
||||||
|
chatter call staying in the existing (already-working) pipeline.
|
||||||
|
"""
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.internal.chatter_classifier import classify_chatter
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Must classify as chatter
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
CHATTER_EXAMPLES = [
|
||||||
|
# Bare acknowledgements / unit check-ins (CORRELATION_REVIEW_0907b.md)
|
||||||
|
("114 Paul.\n114 Paul, Metro Central.\n10-4.", "bare_acknowledgement"),
|
||||||
|
("Affirmative, in charge of 10-8. 10-8, 10-4.", "bare_acknowledgement"),
|
||||||
|
("6-8, you can show me 98. 10-4.", "bare_acknowledgement"),
|
||||||
|
("10-4, 10-4 Central, 98. 10-4, 98.", "bare_acknowledgement"),
|
||||||
|
("7 for Post 1 and 2, 98. Affirm.", "bare_acknowledgement"),
|
||||||
|
("11-Victor to Central. 11-Victor. 72-Holland, 1-5. Central.", "bare_acknowledgement"),
|
||||||
|
# Roll call (CORRELATION_REVIEW_0907b.md / _0912.md)
|
||||||
|
("Post 4, Ossining. And to volunteer patrol, stand by for roll call.", "roll_call"),
|
||||||
|
("Headquarters to all cars, stand by for roll call.", "roll_call"),
|
||||||
|
("All Troop NYC Patrols, stand by for roll call.", "roll_call"),
|
||||||
|
(
|
||||||
|
"Car 100, roll call.\nHenry 1.\nHenry 1.\nSam 1.\nSam 1.\n45 Baker.\n"
|
||||||
|
"45 Baker.\n11 Adam.\nAdam.\n11 Baker.\nBaker.\nStaff 1.\n1.\nStaff 2.",
|
||||||
|
"roll_call",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"Headquarters, all cars on a roll call. Baker 1? Baker 1. Henry 1? "
|
||||||
|
"Henry 1. Sam 2? Sam 2. 11 Adam? 11. 11 Baker? 11 Baker.",
|
||||||
|
"roll_call",
|
||||||
|
),
|
||||||
|
("Because all cars came out for roll call.", "roll_call"),
|
||||||
|
("10-1. KL Cars, that concludes roll call, time is 3-31.", "roll_call"),
|
||||||
|
# Minimal single-word / bare-code transmissions (orphan pool, all 3 dumps)
|
||||||
|
("10-4.", "bare_acknowledgement"),
|
||||||
|
("Roger.", "bare_acknowledgement"),
|
||||||
|
("Clear.", "bare_acknowledgement"),
|
||||||
|
("Affirmative.", "bare_acknowledgement"),
|
||||||
|
("Received.", "bare_acknowledgement"),
|
||||||
|
("10-8, clear. 10-4.", "bare_acknowledgement"),
|
||||||
|
("Post 4, 10-8. 10-4.", "bare_acknowledgement"),
|
||||||
|
("Central to 6 Henry.", "bare_acknowledgement"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("transcript,expected_reason", CHATTER_EXAMPLES)
|
||||||
|
def test_classifies_chatter(transcript, expected_reason):
|
||||||
|
is_chatter, reason = classify_chatter(transcript)
|
||||||
|
assert is_chatter is True
|
||||||
|
assert reason == expected_reason
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Must NOT classify as chatter — real events, including every transcript the
|
||||||
|
# review docs specifically named as dangerous to drop.
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
REAL_EVENT_EXAMPLES = [
|
||||||
|
# The major "extinguishing fire" call (severity=major, tags=[extinguishing-fire])
|
||||||
|
("Dispatch, this is 7-4, extinguishing fire.", "extinguishing_fire"),
|
||||||
|
# Geocoded 911-hangup call (has location_coords)
|
||||||
|
(
|
||||||
|
"7, Charlie. Charlie, check and advise, we've got a call for service "
|
||||||
|
"coming over, it's going to be a 9-1-1 hangout, no voice contact. "
|
||||||
|
"Looks like it was an automated message saying it's the Doral Hat Company.",
|
||||||
|
"geocoded_911_hangup",
|
||||||
|
),
|
||||||
|
# Pursuit updates (severity=major, tags include pursuit / low-speed-pursuit)
|
||||||
|
("I'm aware of that one. It's a low-speed pursuit. It's refusing to pull over.", "low_speed_pursuit"),
|
||||||
|
(
|
||||||
|
"1. Headquarters to 5-charlie. I'm going to say the last thing to anyone.\n"
|
||||||
|
"2. Info, Sgt. Repeat.\n"
|
||||||
|
"3. The SP is on a pursuit southbound on I-684. It's approaching the airport.\n"
|
||||||
|
"4. Okay, thank you.\n5. 23-59.",
|
||||||
|
"pursuit_i684",
|
||||||
|
),
|
||||||
|
# "6 Alpha ... Pelham Station" subject check (CORRELATION_REVIEW_0907b.md's
|
||||||
|
# own "genuinely distinct events" list) — looks like a bare check-in but
|
||||||
|
# dispatches a unit to a specific location.
|
||||||
|
(
|
||||||
|
"6 Alpha, this is Central. 7 Alpha here.\n"
|
||||||
|
"6 Alpha, can you show me on scene at Pelham Station? Stand by.",
|
||||||
|
"pelham_station_subject_check",
|
||||||
|
),
|
||||||
|
# Property-retrieval call (tags=[property-retrieval])
|
||||||
|
(
|
||||||
|
"Property was retrieved with a 911. Can I get a phone number? 10-4. "
|
||||||
|
"Phone number is 214792. 214792.",
|
||||||
|
"property_retrieval",
|
||||||
|
),
|
||||||
|
# Subject check south of Maronex Station (tags=[subject-check])
|
||||||
|
(
|
||||||
|
"Proceed. Show me on a subject south of Maronex Station. Can I get a "
|
||||||
|
"15 check by New York client ID?",
|
||||||
|
"maronex_subject_check",
|
||||||
|
),
|
||||||
|
# Trespassing at milepost 13.7 (tags=[trespassing])
|
||||||
|
(
|
||||||
|
"Can you just 10-5 that job? You came over real muffled.\n"
|
||||||
|
"10-4, there's going to be a trespass on the tracks.\n"
|
||||||
|
"Train 8755 reports two juveniles, one male, one female, both wearing "
|
||||||
|
"white shirts, track three side, at milepost 13.7.",
|
||||||
|
"trespass_milepost_13_7",
|
||||||
|
),
|
||||||
|
# MVA (severity=moderate, tags=[traffic-accident])
|
||||||
|
("1. Train patrol 9.\n2. MVA 4, how close is it?\n3. 10-4.", "mva"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("transcript,label", REAL_EVENT_EXAMPLES, ids=[l for _, l in REAL_EVENT_EXAMPLES])
|
||||||
|
def test_does_not_classify_real_events_as_chatter(transcript, label):
|
||||||
|
is_chatter, reason = classify_chatter(transcript)
|
||||||
|
assert is_chatter is False, f"{label}: false positive, reason={reason!r}"
|
||||||
|
assert reason is None
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Edge cases
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_empty_transcript_not_chatter():
|
||||||
|
assert classify_chatter("") == (False, None)
|
||||||
|
assert classify_chatter(None) == (False, None)
|
||||||
|
assert classify_chatter(" ") == (False, None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_unrecognized_content_defaults_to_not_chatter():
|
||||||
|
# Anything containing real descriptive words the classifier doesn't
|
||||||
|
# recognize must fall through to (False, None), not guess.
|
||||||
|
is_chatter, reason = classify_chatter("Shots fired, officer down, requesting backup immediately.")
|
||||||
|
assert is_chatter is False
|
||||||
|
assert reason is None
|
||||||
@@ -0,0 +1,430 @@
|
|||||||
|
"""
|
||||||
|
server-26#115 — two consensus-quality fixes.
|
||||||
|
|
||||||
|
Fix 1 (routers/upload.py): when the cheap LLM says `orphan`, the rules engine
|
||||||
|
says `new`, and the call is genuinely SUBSTANCELESS (routine severity, no
|
||||||
|
vehicle/geocode/tag, and no incident already running on the same talkgroup),
|
||||||
|
resolve to `orphan` and DO NOT pay for the smart tiebreaker. Radio housekeeping
|
||||||
|
(unit check-ins, roll call, 10-8/10-98) was being promoted to incidents because
|
||||||
|
the tiebreaker rubber-stamped the rules `new` ~21/21 of the time
|
||||||
|
(CORRELATION_REVIEW_0907b.md).
|
||||||
|
|
||||||
|
The substance test runs against `ctx` (fully populated at preview time), NOT
|
||||||
|
against `rules_decision["corr_debug"]` — that dict is EMPTY at preview time for
|
||||||
|
action=="new" (corr_path:"new" is written at APPLY time), so the first version of
|
||||||
|
this gate fired on real events (a `major` "extinguishing fire", geocoded calls,
|
||||||
|
pursuit updates).
|
||||||
|
|
||||||
|
Fix 2 (incident_correlator.py): the `location` correlation path linked on a bare
|
||||||
|
sub-`location_proximity_km` (0.5 km) distance alone, taking whichever incident
|
||||||
|
came first in an unsorted `recent`. In a dense village two unrelated events
|
||||||
|
routinely geocode that close. A `location` link now needs unit overlap with the
|
||||||
|
candidate OR a distance under a tighter bar, and picks the NEAREST qualifying
|
||||||
|
candidate. A unit-overlap location link is tagged `location_unit_overlap` so it
|
||||||
|
does not merge into the fast path's bucket in the admin fit-signal histogram.
|
||||||
|
"""
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.routers import upload
|
||||||
|
from app.internal.incident_correlator import _run_decision, has_event_substance
|
||||||
|
|
||||||
|
NOW = datetime(2026, 9, 7, 21, 30, 0, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Fix 1 — the LLM-orphan gate in _correlate_with_consensus
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _preview(action, corr_debug=None, ctx=None):
|
||||||
|
base_ctx = {"call_id": "call-1"}
|
||||||
|
if ctx:
|
||||||
|
base_ctx.update(ctx)
|
||||||
|
return {
|
||||||
|
"decision": {
|
||||||
|
"action": action,
|
||||||
|
"matched_incident": None,
|
||||||
|
"incident_type": "other" if action == "new" else None,
|
||||||
|
"corr_debug": {} if corr_debug is None else dict(corr_debug),
|
||||||
|
},
|
||||||
|
"ctx": base_ctx,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _llm(action, reasoning="—"):
|
||||||
|
md = {"incident_id": "inc-1"} if action == "link" else None
|
||||||
|
return {"action": action, "matched_incident": md, "reasoning": reasoning}
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_consensus(preview, llm_decision):
|
||||||
|
tiebreak_result = {
|
||||||
|
"action": "new", "matched_incident": None, "incident_type": "other",
|
||||||
|
"corr_debug": {}, "reasoning": "tb",
|
||||||
|
}
|
||||||
|
with patch("app.internal.incident_correlator.preview_correlation",
|
||||||
|
new=AsyncMock(return_value=preview)), \
|
||||||
|
patch("app.internal.incident_correlator.apply_correlation",
|
||||||
|
new=AsyncMock(return_value="incident-x")) as m_apply, \
|
||||||
|
patch("app.internal.llm_correlator.decide",
|
||||||
|
new=AsyncMock(return_value=llm_decision)), \
|
||||||
|
patch("app.internal.llm_correlator.tiebreak",
|
||||||
|
new=AsyncMock(return_value=tiebreak_result)) as m_tiebreak:
|
||||||
|
await upload._correlate_with_consensus(
|
||||||
|
call_id="call-1", node_id="n1", system_id="sys-1",
|
||||||
|
talkgroup_id=9048, talkgroup_name="Dispatch", tags=[],
|
||||||
|
incident_type=None, location=None, location_coords=None,
|
||||||
|
)
|
||||||
|
return m_apply, m_tiebreak
|
||||||
|
|
||||||
|
|
||||||
|
async def test_substanceless_no_recent_same_tg_incident_gates_without_tiebreak():
|
||||||
|
m_apply, m_tiebreak = await _run_consensus(
|
||||||
|
_preview("new", {}), _llm("orphan", "unit check-in, not an incident"),
|
||||||
|
)
|
||||||
|
m_tiebreak.assert_not_called()
|
||||||
|
m_apply.assert_called_once()
|
||||||
|
gated = m_apply.call_args[0][0]["decision"]
|
||||||
|
assert gated["action"] == "orphan"
|
||||||
|
dbg = gated["corr_debug"]
|
||||||
|
assert dbg["corr_consensus"] == "llm_orphan_gate"
|
||||||
|
assert dbg["corr_consensus"] != "tiebreak"
|
||||||
|
assert dbg["corr_rules_action"] == "new"
|
||||||
|
assert dbg["corr_llm_action"] == "orphan"
|
||||||
|
assert dbg["corr_llm_reasoning"] == "unit check-in, not an incident"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("severity", ["moderate", "major"])
|
||||||
|
async def test_moderate_or_major_severity_is_not_gated(severity):
|
||||||
|
m_apply, m_tiebreak = await _run_consensus(
|
||||||
|
_preview("new", {}, ctx={"call_severity": severity}), _llm("orphan"),
|
||||||
|
)
|
||||||
|
m_tiebreak.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_routine_severity_alone_still_gates():
|
||||||
|
m_apply, m_tiebreak = await _run_consensus(
|
||||||
|
_preview("new", {}, ctx={"call_severity": "routine"}), _llm("orphan"),
|
||||||
|
)
|
||||||
|
m_tiebreak.assert_not_called()
|
||||||
|
assert m_apply.call_args[0][0]["decision"]["action"] == "orphan"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_call_with_coords_is_not_gated():
|
||||||
|
m_apply, m_tiebreak = await _run_consensus(
|
||||||
|
_preview("new", {}, ctx={"coords": {"lat": 41.15, "lng": -73.86}}),
|
||||||
|
_llm("orphan"),
|
||||||
|
)
|
||||||
|
m_tiebreak.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_call_with_tags_is_not_gated():
|
||||||
|
m_apply, m_tiebreak = await _run_consensus(
|
||||||
|
_preview("new", {}, ctx={"tags": ["structure-fire"]}), _llm("orphan"),
|
||||||
|
)
|
||||||
|
m_tiebreak.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_call_with_vehicles_is_not_gated():
|
||||||
|
m_apply, m_tiebreak = await _run_consensus(
|
||||||
|
_preview("new", {}, ctx={"call_vehicles": ["red sedan"]}), _llm("orphan"),
|
||||||
|
)
|
||||||
|
m_tiebreak.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_call_with_resolved_incident_type_is_not_gated():
|
||||||
|
# The creation gate skips has_event_substance when a type resolved, so a
|
||||||
|
# typed call (fire/medical/…) opens an incident on substance the gate does
|
||||||
|
# not re-check — it must keep the tiebreak, not be dropped.
|
||||||
|
m_apply, m_tiebreak = await _run_consensus(
|
||||||
|
_preview("new", {}, ctx={"incident_type": "fire"}), _llm("orphan"),
|
||||||
|
)
|
||||||
|
m_tiebreak.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_reassignment_call_is_not_gated():
|
||||||
|
# reassignment=True is dispatch pulling a unit onto a NEW job (units are
|
||||||
|
# blanked for exactly that reason) — the strongest new-incident signal.
|
||||||
|
m_apply, m_tiebreak = await _run_consensus(
|
||||||
|
_preview("new", {}, ctx={"reassignment": True}), _llm("orphan"),
|
||||||
|
)
|
||||||
|
m_tiebreak.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_recent_incident_on_same_talkgroup_is_not_gated():
|
||||||
|
ctx = {
|
||||||
|
"system_id": "sys-1",
|
||||||
|
"talkgroup_id": 9048,
|
||||||
|
"talkgroup_name": "Dispatch",
|
||||||
|
"now": NOW,
|
||||||
|
"recent": [{
|
||||||
|
"incident_id": "inc-live",
|
||||||
|
"system_ids": ["sys-1"],
|
||||||
|
"talkgroup_ids": ["9048"],
|
||||||
|
"updated_at": (NOW - timedelta(minutes=1)).isoformat(),
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
m_apply, m_tiebreak = await _run_consensus(
|
||||||
|
_preview("new", {}, ctx=ctx), _llm("orphan"),
|
||||||
|
)
|
||||||
|
m_tiebreak.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
# server-26#115 window #3 (CORRELATION_REVIEW_0912.md): the escape hatch used
|
||||||
|
# to treat ANY same-talkgroup incident inside the 2h correlation_window_hours
|
||||||
|
# as "recent", which on a busy dispatch channel (3-13 incidents/2h) was
|
||||||
|
# satisfied almost unconditionally — the gate fired 0/24 times against its own
|
||||||
|
# target shape. It now only counts an incident as recent within
|
||||||
|
# settings.tg_dispatch_thin_idle_minutes (5 min), applied uniformly regardless
|
||||||
|
# of the talkgroup's name (owner correction, 2026-09-13 — see
|
||||||
|
# test_channel_name_does_not_affect_the_window below for why the dichotomy
|
||||||
|
# this originally had with incident_correlator's fast/thin idle selection was
|
||||||
|
# removed here).
|
||||||
|
|
||||||
|
async def test_recent_same_tg_incident_inside_new_short_window_still_escapes_gate():
|
||||||
|
ctx = {
|
||||||
|
"system_id": "sys-1",
|
||||||
|
"talkgroup_id": 9048,
|
||||||
|
"talkgroup_name": "Dispatch",
|
||||||
|
"now": NOW,
|
||||||
|
"recent": [{
|
||||||
|
"incident_id": "inc-live",
|
||||||
|
"system_ids": ["sys-1"],
|
||||||
|
"talkgroup_ids": ["9048"],
|
||||||
|
# 3 min ago — inside tg_dispatch_thin_idle_minutes (5).
|
||||||
|
"updated_at": (NOW - timedelta(minutes=3)).isoformat(),
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
m_apply, m_tiebreak = await _run_consensus(
|
||||||
|
_preview("new", {}, ctx=ctx), _llm("orphan"),
|
||||||
|
)
|
||||||
|
m_tiebreak.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_recent_same_tg_incident_older_than_short_window_now_gates():
|
||||||
|
# Regression test for the fix: 8 minutes is past the 5-minute bound but
|
||||||
|
# still inside the OLD 2-hour correlation_window_hours lookback. Before
|
||||||
|
# the fix this escaped the gate on any channel; after the fix it gates.
|
||||||
|
ctx = {
|
||||||
|
"system_id": "sys-1",
|
||||||
|
"talkgroup_id": 9048,
|
||||||
|
"talkgroup_name": "Dispatch",
|
||||||
|
"now": NOW,
|
||||||
|
"recent": [{
|
||||||
|
"incident_id": "inc-stale",
|
||||||
|
"system_ids": ["sys-1"],
|
||||||
|
"talkgroup_ids": ["9048"],
|
||||||
|
"updated_at": (NOW - timedelta(minutes=8)).isoformat(),
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
m_apply, m_tiebreak = await _run_consensus(
|
||||||
|
_preview("new", {}, ctx=ctx), _llm("orphan"),
|
||||||
|
)
|
||||||
|
m_tiebreak.assert_not_called()
|
||||||
|
assert m_apply.call_args[0][0]["decision"]["action"] == "orphan"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_channel_name_does_not_affect_the_window():
|
||||||
|
# Owner correction, 2026-09-13 (direct scanning experience): a talkgroup
|
||||||
|
# named "tac"/"tactical" only sees materially different traffic during a
|
||||||
|
# real incident, and that's rare -- the bulk of traffic on any monitored
|
||||||
|
# channel, including high-risk stops and pursuits, runs on the main
|
||||||
|
# channel regardless of what it's named. An earlier version of this used
|
||||||
|
# a longer 15-minute window on anything not literally named "dispatch"/
|
||||||
|
# "patched"/"primary" (mirroring incident_correlator's fast/thin idle
|
||||||
|
# selection); that meant a busy single-channel department not literally
|
||||||
|
# named "dispatch" silently got the more permissive window and could
|
||||||
|
# reproduce #115's original bug. Same 8-minute age as the dispatch-named
|
||||||
|
# test above, but on a channel named "Tac 3" -- must gate identically,
|
||||||
|
# not escape into a longer window just because of the name.
|
||||||
|
ctx = {
|
||||||
|
"system_id": "sys-1",
|
||||||
|
"talkgroup_id": 383,
|
||||||
|
"talkgroup_name": "Tac 3",
|
||||||
|
"now": NOW,
|
||||||
|
"recent": [{
|
||||||
|
"incident_id": "inc-tac",
|
||||||
|
"system_ids": ["sys-1"],
|
||||||
|
"talkgroup_ids": ["383"],
|
||||||
|
"updated_at": (NOW - timedelta(minutes=8)).isoformat(),
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
m_apply, m_tiebreak = await _run_consensus(
|
||||||
|
_preview("new", {}, ctx=ctx), _llm("orphan"),
|
||||||
|
)
|
||||||
|
m_tiebreak.assert_not_called()
|
||||||
|
assert m_apply.call_args[0][0]["decision"]["action"] == "orphan"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_gate_veto_reason_is_recorded_on_the_escalation_path():
|
||||||
|
# server-26#115: a live measurement window must be able to see *why* an
|
||||||
|
# llm=orphan/rules=new call escaped the gate without guessing from the raw
|
||||||
|
# dump (which produced a wrong "confirmed explanation" for 2 window-#3
|
||||||
|
# misses the first time). corr_gate_veto names the surviving condition.
|
||||||
|
ctx = {"call_severity": "major"}
|
||||||
|
m_apply, m_tiebreak = await _run_consensus(
|
||||||
|
_preview("new", {}, ctx=ctx), _llm("orphan"),
|
||||||
|
)
|
||||||
|
m_tiebreak.assert_called_once()
|
||||||
|
final = m_apply.call_args[0][0]["decision"]
|
||||||
|
assert final["corr_debug"]["corr_gate_veto"] == "severity"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_gate_veto_reason_is_absent_when_the_disagreement_is_not_orphan_vs_new():
|
||||||
|
# corr_gate_veto is only meaningful for the llm=orphan/rules=new shape the
|
||||||
|
# gate targets — it must not appear (or be misleadingly None-vs-absent) on
|
||||||
|
# an unrelated disagreement shape.
|
||||||
|
m_apply, m_tiebreak = await _run_consensus(
|
||||||
|
_preview("link", {}), _llm("orphan"),
|
||||||
|
)
|
||||||
|
m_tiebreak.assert_called_once()
|
||||||
|
final = m_apply.call_args[0][0]["decision"]
|
||||||
|
assert "corr_gate_veto" not in final["corr_debug"]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_recent_incident_on_a_different_talkgroup_still_gates():
|
||||||
|
ctx = {
|
||||||
|
"system_id": "sys-1",
|
||||||
|
"talkgroup_id": 9048,
|
||||||
|
"recent": [{
|
||||||
|
"incident_id": "inc-other",
|
||||||
|
"system_ids": ["sys-1"],
|
||||||
|
"talkgroup_ids": ["1200"],
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
m_apply, m_tiebreak = await _run_consensus(
|
||||||
|
_preview("new", {}, ctx=ctx), _llm("orphan"),
|
||||||
|
)
|
||||||
|
m_tiebreak.assert_not_called()
|
||||||
|
assert m_apply.call_args[0][0]["decision"]["action"] == "orphan"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_llm_link_vs_rules_new_still_escalates():
|
||||||
|
m_apply, m_tiebreak = await _run_consensus(_preview("new", {}), _llm("link", "same job"))
|
||||||
|
m_tiebreak.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_llm_orphan_vs_rules_link_still_escalates():
|
||||||
|
# Not the gate condition (gate needs rules=="new"); must fall through.
|
||||||
|
m_apply, m_tiebreak = await _run_consensus(_preview("link", {}), _llm("orphan"))
|
||||||
|
m_tiebreak.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_has_event_substance_predicate():
|
||||||
|
assert has_event_substance({"coords": {"lat": 1, "lng": 2}})
|
||||||
|
assert has_event_substance({"tags": ["fire"]})
|
||||||
|
assert has_event_substance({"call_vehicles": ["sedan"]})
|
||||||
|
assert not has_event_substance({})
|
||||||
|
assert not has_event_substance({"coords": None, "tags": [], "call_vehicles": []})
|
||||||
|
# units and location are NOT substance — nearly every transmission has them.
|
||||||
|
assert not has_event_substance({"call_units": ["7-Adam"], "location": "Main St"})
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Fix 2 — tighten corr_path=location
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
CALL_COORDS = {"lat": 41.150000, "lng": -73.860000}
|
||||||
|
# ~0.39 km north of the call — inside location_proximity_km (0.5) but well
|
||||||
|
# outside the tight bar (_LOCATION_TIGHT_PROXIMITY_KM, 0.2).
|
||||||
|
FAR_INC_COORDS = {"lat": 41.153500, "lng": -73.860000}
|
||||||
|
# ~0.13 km north of the call — inside the tight bar.
|
||||||
|
NEAR_INC_COORDS = {"lat": 41.151200, "lng": -73.860000}
|
||||||
|
# ~0.28 km north — inside the 0.5 radius, outside the 0.2 tight bar; used as a
|
||||||
|
# second candidate that must lose the nearest-wins sort to NEAR_INC_COORDS.
|
||||||
|
MID_INC_COORDS = {"lat": 41.152500, "lng": -73.860000}
|
||||||
|
|
||||||
|
|
||||||
|
def _inc(incident_id, coords, units):
|
||||||
|
return {
|
||||||
|
"incident_id": incident_id,
|
||||||
|
"system_ids": ["sys-1"],
|
||||||
|
"talkgroup_ids": ["100"], # different TGID → fast path is a no-op
|
||||||
|
"location_coords": coords,
|
||||||
|
"units": units,
|
||||||
|
"tags": [],
|
||||||
|
"type": "police",
|
||||||
|
"updated_at": (NOW - timedelta(minutes=6)).isoformat(),
|
||||||
|
"started_at": (NOW - timedelta(minutes=20)).isoformat(),
|
||||||
|
"status": "active",
|
||||||
|
"call_ids": ["c0"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _loc_ctx(*, incidents, call_units):
|
||||||
|
return {
|
||||||
|
"call_id": "call-loc",
|
||||||
|
"all_active": list(incidents),
|
||||||
|
"recent": list(incidents),
|
||||||
|
"call_doc": {},
|
||||||
|
"call_embedding": None,
|
||||||
|
"call_units": call_units,
|
||||||
|
"call_vehicles": [],
|
||||||
|
"call_cleared": [],
|
||||||
|
"call_severity": "routine",
|
||||||
|
"coords": CALL_COORDS,
|
||||||
|
"is_thin_call": False,
|
||||||
|
"now": NOW,
|
||||||
|
"system_id": "sys-1",
|
||||||
|
"talkgroup_id": 999, # not in inc.talkgroup_ids
|
||||||
|
"talkgroup_name": "Tactical",
|
||||||
|
"tags": [],
|
||||||
|
"incident_type": "police",
|
||||||
|
"location": "Main St",
|
||||||
|
"location_coords": CALL_COORDS,
|
||||||
|
"reassignment": True, # suppress the unit-continuity path
|
||||||
|
"create_if_new": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_location_path_in_radius_but_no_unit_overlap_no_tight_proximity_does_not_link(caplog):
|
||||||
|
ctx = _loc_ctx(
|
||||||
|
incidents=[_inc("inc-loc", FAR_INC_COORDS, ["7-Adam"])],
|
||||||
|
call_units=["3-Boy"],
|
||||||
|
)
|
||||||
|
with caplog.at_level("INFO", logger="drb-c2-core"):
|
||||||
|
decision = _run_decision(ctx)
|
||||||
|
# Reaches, and is rejected by, the new guard (not an earlier path).
|
||||||
|
assert "location-path skipped" in caplog.text
|
||||||
|
assert decision["action"] != "link"
|
||||||
|
assert (decision.get("corr_debug") or {}).get("corr_path") != "location"
|
||||||
|
|
||||||
|
|
||||||
|
def test_location_path_links_on_unit_overlap_with_distinct_fit_signal():
|
||||||
|
ctx = _loc_ctx(
|
||||||
|
incidents=[_inc("inc-loc", FAR_INC_COORDS, ["5-Adam"])],
|
||||||
|
call_units=["5-Adam"],
|
||||||
|
)
|
||||||
|
decision = _run_decision(ctx)
|
||||||
|
assert decision["action"] == "link"
|
||||||
|
assert decision["corr_debug"]["corr_path"] == "location"
|
||||||
|
# NOT "unit_overlap" — that value belongs to the fast path's histogram bucket.
|
||||||
|
assert decision["corr_debug"]["corr_fit_signal"] == "location_unit_overlap"
|
||||||
|
|
||||||
|
|
||||||
|
def test_location_path_links_on_tight_proximity_without_unit_overlap():
|
||||||
|
ctx = _loc_ctx(
|
||||||
|
incidents=[_inc("inc-loc", NEAR_INC_COORDS, ["7-Adam"])],
|
||||||
|
call_units=["3-Boy"],
|
||||||
|
)
|
||||||
|
decision = _run_decision(ctx)
|
||||||
|
assert decision["action"] == "link"
|
||||||
|
assert decision["corr_debug"]["corr_path"] == "location"
|
||||||
|
assert decision["corr_debug"]["corr_fit_signal"] == "location_proximity"
|
||||||
|
|
||||||
|
|
||||||
|
def test_location_path_picks_nearest_in_radius_candidate():
|
||||||
|
# `recent` order puts the farther tight-proximity incident first; the guard
|
||||||
|
# must still select the nearest one.
|
||||||
|
ctx = _loc_ctx(
|
||||||
|
incidents=[
|
||||||
|
_inc("inc-mid", MID_INC_COORDS, ["3-Boy"]), # ~0.28 km, tight-fail
|
||||||
|
_inc("inc-near", NEAR_INC_COORDS, ["3-Boy"]), # ~0.13 km, tight-pass
|
||||||
|
],
|
||||||
|
call_units=["3-Boy"],
|
||||||
|
)
|
||||||
|
decision = _run_decision(ctx)
|
||||||
|
assert decision["action"] == "link"
|
||||||
|
assert decision["matched_incident"]["incident_id"] == "inc-near"
|
||||||
|
assert decision["corr_debug"]["corr_path"] == "location"
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
"""
|
||||||
|
server-26#115 — the tiebreaker manufactured incidents because it was blind to
|
||||||
|
what would tell it two incidents are one.
|
||||||
|
|
||||||
|
Two low-risk supports for the reframed prompt:
|
||||||
|
1. `_extract_road_ids` collapses street-type synonyms, so "Mohegan Park Ave"
|
||||||
|
and "Mohegan Park Avenue" share a road id (they were splitting one
|
||||||
|
car-alarm incident into two).
|
||||||
|
2. `_inc_summary` now carries the incident title and talkgroup, the two
|
||||||
|
signals the model needs to recognise a same-channel continuation.
|
||||||
|
"""
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from app.internal.incident_correlator import (
|
||||||
|
_extract_road_ids, _location_mentions_road_overlap,
|
||||||
|
)
|
||||||
|
from app.internal.llm_correlator import _inc_summary, _prompt_incidents
|
||||||
|
|
||||||
|
NOW = datetime(2026, 9, 7, 8, 0, 0, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def test_avenue_and_ave_are_the_same_road_id():
|
||||||
|
assert _extract_road_ids("Mohegan Park Avenue") == _extract_road_ids("Mohegan Park Ave")
|
||||||
|
assert _extract_road_ids("191 Broadway Street") == _extract_road_ids("191 Broadway St")
|
||||||
|
assert _extract_road_ids("North State Road") == _extract_road_ids("North State Rd")
|
||||||
|
|
||||||
|
|
||||||
|
def test_road_overlap_matches_across_the_synonym():
|
||||||
|
assert _location_mentions_road_overlap("multiple car alarms Mohegan Park Avenue",
|
||||||
|
["patrol to Mohegan Park Ave"]) is True
|
||||||
|
# still discriminates genuinely different streets
|
||||||
|
assert _location_mentions_road_overlap("Oak Avenue", ["Elm Avenue"]) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_inc_summary_carries_title_and_talkgroup():
|
||||||
|
s = _inc_summary({
|
||||||
|
"incident_id": "abc123",
|
||||||
|
"type": "police",
|
||||||
|
"talkgroup_ids": [9560],
|
||||||
|
"title": "Nuisance Alarm at Mohegan Park Ave",
|
||||||
|
"location": "Mohegan Park Ave",
|
||||||
|
"units": ["Headquarters"],
|
||||||
|
"tags": ["car-alarm"],
|
||||||
|
"updated_at": NOW.isoformat(),
|
||||||
|
}, NOW)
|
||||||
|
assert "title:'Nuisance Alarm at Mohegan Park Ave'" in s
|
||||||
|
assert "tg:[9560]" in s
|
||||||
|
assert "id:abc123" in s
|
||||||
|
|
||||||
|
|
||||||
|
def test_inc_summary_omits_missing_optional_fields():
|
||||||
|
s = _inc_summary({"incident_id": "x", "updated_at": NOW.isoformat()}, NOW)
|
||||||
|
assert "title:" not in s and "tg:" not in s and "loc:" not in s
|
||||||
|
assert s.startswith("id:x")
|
||||||
|
|
||||||
|
|
||||||
|
def test_prompt_incidents_is_most_recently_active_first_and_capped():
|
||||||
|
recent = [
|
||||||
|
{"incident_id": f"i{n}", "updated_at": f"2026-09-07T0{n}:00:00+00:00"}
|
||||||
|
for n in range(1, 8)
|
||||||
|
]
|
||||||
|
ordered = _prompt_incidents(recent)
|
||||||
|
assert [i["incident_id"] for i in ordered] == ["i7", "i6", "i5", "i4", "i3", "i2", "i1"]
|
||||||
|
assert len(_prompt_incidents(recent * 5)) == 20
|
||||||
|
# falls back to started_at when updated_at is absent, and never raises
|
||||||
|
assert _prompt_incidents([{"incident_id": "a", "started_at": NOW.isoformat()},
|
||||||
|
{"incident_id": "b"}])[0]["incident_id"] == "a"
|
||||||
@@ -31,8 +31,10 @@ from app.internal.incident_correlator import (
|
|||||||
|
|
||||||
NOW = datetime(2026, 8, 20, 7, 0, 0, tzinfo=timezone.utc)
|
NOW = datetime(2026, 8, 20, 7, 0, 0, tzinfo=timezone.utc)
|
||||||
|
|
||||||
# TG 383 from the dump: "Ch 1 (Patched with 155.310)". _DISPATCH_TG_RE matches
|
# TG 383 from the dump: "Ch 1 (Patched with 155.310)", a shared dispatch
|
||||||
# "patched", so this is a shared dispatch backbone carrying the whole department.
|
# backbone carrying the whole department. Kept as two distinct fixture names
|
||||||
|
# for readability even though the channel's name no longer affects behavior
|
||||||
|
# (server-26#134).
|
||||||
DISPATCH_TG = "Ch 1 (Patched with 155.310)"
|
DISPATCH_TG = "Ch 1 (Patched with 155.310)"
|
||||||
TACTICAL_TG = "Fireground 2"
|
TACTICAL_TG = "Fireground 2"
|
||||||
|
|
||||||
@@ -140,12 +142,9 @@ def test_thin_call_with_no_overlap_does_not_attach_on_a_dispatch_channel():
|
|||||||
assert _run_decision(_ctx(all_active=[inc], recent=[inc]))["action"] == "orphan"
|
assert _run_decision(_ctx(all_active=[inc], recent=[inc]))["action"] == "orphan"
|
||||||
|
|
||||||
|
|
||||||
def test_thin_call_with_no_overlap_does_not_attach_on_a_tactical_channel():
|
def test_thin_call_with_no_overlap_does_not_attach_on_a_tactical_named_channel():
|
||||||
"""
|
"""A channel's name no longer changes anything (server-26#134) — same
|
||||||
The widest version of the bug: non-dispatch talkgroups skipped the tiering
|
assertion as the dispatch-named case above, different fixture name."""
|
||||||
entirely and used the whole 90-minute fast-path window with no
|
|
||||||
single-candidate requirement, so ANY thin call joined whatever was newest.
|
|
||||||
"""
|
|
||||||
inc = _incident(idle_minutes=40)
|
inc = _incident(idle_minutes=40)
|
||||||
decision = _run_decision(_ctx(
|
decision = _run_decision(_ctx(
|
||||||
all_active=[inc], recent=[inc], talkgroup_name=TACTICAL_TG,
|
all_active=[inc], recent=[inc], talkgroup_name=TACTICAL_TG,
|
||||||
@@ -153,9 +152,18 @@ def test_thin_call_with_no_overlap_does_not_attach_on_a_tactical_channel():
|
|||||||
assert decision["action"] == "orphan"
|
assert decision["action"] == "orphan"
|
||||||
|
|
||||||
|
|
||||||
def test_tactical_thin_call_still_attaches_inside_its_own_window():
|
def test_tactical_named_channel_uses_the_dispatch_window_now():
|
||||||
"""Bounded, not removed — a "10-4" on a working channel is still context."""
|
"""server-26#134: 14 min was inside the old 15-min tactical window; now
|
||||||
inc = _incident(idle_minutes=settings.tg_thin_idle_minutes - 1)
|
every channel uses the 5-min window regardless of name."""
|
||||||
|
inc = _incident(idle_minutes=14)
|
||||||
|
decision = _run_decision(_ctx(
|
||||||
|
all_active=[inc], recent=[inc], talkgroup_name=TACTICAL_TG,
|
||||||
|
))
|
||||||
|
assert decision["action"] == "orphan"
|
||||||
|
|
||||||
|
|
||||||
|
def test_tactical_named_channel_still_attaches_inside_the_dispatch_window():
|
||||||
|
inc = _incident(idle_minutes=settings.tg_dispatch_thin_idle_minutes - 1)
|
||||||
decision = _run_decision(_ctx(
|
decision = _run_decision(_ctx(
|
||||||
all_active=[inc], recent=[inc], talkgroup_name=TACTICAL_TG,
|
all_active=[inc], recent=[inc], talkgroup_name=TACTICAL_TG,
|
||||||
))
|
))
|
||||||
@@ -172,6 +180,55 @@ def test_tactical_thin_call_is_ambiguous_with_two_candidates():
|
|||||||
assert decision["action"] == "orphan"
|
assert decision["action"] == "orphan"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# server-26#158: srcaddr identity beats recency guesswork for thin calls
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_thin_call_srcaddr_match_resolves_tier2_ambiguity():
|
||||||
|
"""
|
||||||
|
Same fixture as test_tactical_thin_call_is_ambiguous_with_two_candidates —
|
||||||
|
two candidates, tier-2 window, no unit ID parsed (transcript_too_short
|
||||||
|
skipped GPT). Without srcaddr this orphans. With it, the radio that sent
|
||||||
|
the call already touched inc-b, so that's the thread — not a guess.
|
||||||
|
"""
|
||||||
|
a = _incident(idle_minutes=3.0, incident_id="inc-a", srcaddrs=["9001"])
|
||||||
|
b = _incident(idle_minutes=4.0, incident_id="inc-b", srcaddrs=["9002"])
|
||||||
|
decision = _run_decision(_ctx(
|
||||||
|
all_active=[a, b], recent=[a, b], talkgroup_name=TACTICAL_TG,
|
||||||
|
call_srcaddr="9002",
|
||||||
|
))
|
||||||
|
assert decision["action"] == "link"
|
||||||
|
assert decision["matched_incident"]["incident_id"] == "inc-b"
|
||||||
|
assert decision["corr_debug"]["corr_fit_signal"] == "thin_srcaddr_match"
|
||||||
|
|
||||||
|
|
||||||
|
def test_thin_call_srcaddr_match_overrides_recency_in_tier1():
|
||||||
|
"""
|
||||||
|
Both candidates are inside the 30s conversational window, where recency
|
||||||
|
alone would pick inc-a (more recently updated) even though the radio that
|
||||||
|
sent this call has only ever touched inc-b — the exact busy-channel,
|
||||||
|
two-concurrent-incidents misattach server-26#158 was filed for.
|
||||||
|
"""
|
||||||
|
a = _incident(idle_minutes=0.1, incident_id="inc-a", srcaddrs=["9001"])
|
||||||
|
b = _incident(idle_minutes=0.2, incident_id="inc-b", srcaddrs=["9002"])
|
||||||
|
decision = _run_decision(_ctx(
|
||||||
|
all_active=[a, b], recent=[a, b], call_srcaddr="9002",
|
||||||
|
))
|
||||||
|
assert decision["action"] == "link"
|
||||||
|
assert decision["matched_incident"]["incident_id"] == "inc-b"
|
||||||
|
|
||||||
|
|
||||||
|
def test_thin_call_with_no_srcaddr_match_falls_back_to_recency():
|
||||||
|
"""A radio ID that matches nothing on this talkgroup behaves exactly as
|
||||||
|
before — no regression for the ordinary case."""
|
||||||
|
a = _incident(idle_minutes=0.1, incident_id="inc-a", srcaddrs=["9001"])
|
||||||
|
decision = _run_decision(_ctx(
|
||||||
|
all_active=[a], recent=[a], call_srcaddr="unrelated-radio",
|
||||||
|
))
|
||||||
|
assert decision["action"] == "link"
|
||||||
|
assert decision["corr_debug"]["corr_fit_signal"] == "thin_recency"
|
||||||
|
|
||||||
|
|
||||||
def test_call_with_unit_overlap_does_attach():
|
def test_call_with_unit_overlap_does_attach():
|
||||||
"""
|
"""
|
||||||
Positive control: real evidence still links. Carrying units also means the
|
Positive control: real evidence still links. Carrying units also means the
|
||||||
@@ -224,47 +281,27 @@ def test_back_dated_thin_call_does_not_sail_through_the_recency_gate():
|
|||||||
|
|
||||||
def test_back_dated_call_does_not_bypass_the_content_divergence_veto(monkeypatch):
|
def test_back_dated_call_does_not_bypass_the_content_divergence_veto(monkeypatch):
|
||||||
"""
|
"""
|
||||||
Same `9d376ffe` failure mode, but exercised directly against
|
Same `9d376ffe` failure mode, exercised directly against
|
||||||
`_call_fits_incident` on a dispatch channel: unit overlap plus a
|
`_call_fits_incident`: unit overlap plus a back-dated call (incident
|
||||||
back-dated call (incident updated 45 minutes AFTER the call's own
|
updated 45 minutes AFTER the call's own `started_at`, which the sweep
|
||||||
`started_at`, which the sweep passes as `now`) used to make the signed
|
passes as `now`) used to make the signed idle -45, so `idle_min >= 15`
|
||||||
idle -45, so `idle_min >= 15` read False and the content-divergence
|
read False and the content-divergence veto never ran — unit overlap
|
||||||
veto never ran — unit overlap alone forced the merge regardless of
|
alone forced the merge regardless of what the call was actually about.
|
||||||
what the call was actually about. With the gate fixed to compare
|
With the gate fixed to compare distance, idle_min is 45 (>= 15), the
|
||||||
distance, idle_min is 45 (>= 15), the veto runs, and a divergent
|
veto runs, and a divergent embedding (patched below so the assertion
|
||||||
embedding (patched below so the assertion doesn't depend on numpy
|
doesn't depend on numpy being installed in this environment) fails it.
|
||||||
being installed in this environment) fails it.
|
|
||||||
"""
|
"""
|
||||||
monkeypatch.setattr(correlator_mod, "_cosine_similarity", lambda a, b: 0.0)
|
monkeypatch.setattr(correlator_mod, "_cosine_similarity", lambda a, b: 0.0)
|
||||||
inc = _incident(idle_minutes=-45, units=["6-Adam"])
|
inc = _incident(idle_minutes=-45, units=["6-Adam"])
|
||||||
inc["embedding"] = [1.0, 0.0]
|
inc["embedding"] = [1.0, 0.0]
|
||||||
fits, signal = _call_fits_incident(
|
fits, signal = _call_fits_incident(
|
||||||
inc, call_units=["6-Adam"], call_vehicles=[], call_coords=None,
|
inc, call_units=["6-Adam"], call_vehicles=[], call_coords=None,
|
||||||
proximity_km=settings.location_proximity_km, is_dispatch=True,
|
proximity_km=settings.location_proximity_km,
|
||||||
call_embedding=[0.0, 1.0], now=NOW,
|
call_embedding=[0.0, 1.0], now=NOW,
|
||||||
)
|
)
|
||||||
assert (fits, signal) == (False, "content_divergence")
|
assert (fits, signal) == (False, "content_divergence")
|
||||||
|
|
||||||
|
|
||||||
def test_back_dated_call_on_tactical_channel_does_not_get_the_default():
|
|
||||||
"""
|
|
||||||
Tactical-channel counterpart: no unit/vehicle/location signal, so the
|
|
||||||
function falls through to step 4's `idle_min < 20.0` default. A
|
|
||||||
back-dated call (incident updated 45 minutes after the call's own
|
|
||||||
started_at) used to read idle_min as -45, which is always < 20.0, so
|
|
||||||
`tactical_default` fired unconditionally no matter how stale the
|
|
||||||
incident actually was relative to this call. Fixed, idle_min is the
|
|
||||||
45-minute distance, which is not < 20.0.
|
|
||||||
"""
|
|
||||||
inc = _incident(idle_minutes=-45)
|
|
||||||
fits, signal = _call_fits_incident(
|
|
||||||
inc, call_units=[], call_vehicles=[], call_coords=None,
|
|
||||||
proximity_km=settings.location_proximity_km, is_dispatch=False,
|
|
||||||
call_embedding=None, now=NOW,
|
|
||||||
)
|
|
||||||
assert (fits, signal) == (False, "tactical_idle")
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# 4. Hard caps — path-independent, because pairwise fit tests can't see shape
|
# 4. Hard caps — path-independent, because pairwise fit tests can't see shape
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -0,0 +1,198 @@
|
|||||||
|
"""
|
||||||
|
server-26#<pending> — pattern B clearance: a unit accepting a NEW dispatch
|
||||||
|
("dispatch: are you able to clear and take a run at X / unit: 10-4") carries
|
||||||
|
no self-reported clearance language intelligence.py's cleared_units
|
||||||
|
extraction looks for (that only catches pattern A, "Unit 7, 10-8"). Before
|
||||||
|
this fix, reassignment=True only ever suppressed the unit from re-linking to
|
||||||
|
their prior incident (upload.py's corr_units=[] on reassignment) — nothing
|
||||||
|
ever released them from it, so it sat "active" until the 90-minute idle
|
||||||
|
sweep timed it out instead of being marked cleared by a real event.
|
||||||
|
|
||||||
|
`_release_reassigned_units` closes that gap: when a scene is a reassignment,
|
||||||
|
scan the OTHER active incidents for unit overlap and release the unit there,
|
||||||
|
using the same units_active/units_cleared merge (`_apply_unit_clearance`)
|
||||||
|
that explicit 10-8 extraction already used via `_update_incident`.
|
||||||
|
"""
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.internal.incident_correlator import (
|
||||||
|
_apply_unit_clearance, _release_reassigned_units,
|
||||||
|
)
|
||||||
|
|
||||||
|
NOW = datetime(2026, 9, 20, 12, 0, 0, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _incident(incident_id="inc-1", units_active=None, units_cleared=None,
|
||||||
|
system_ids=("sys-1",), **overrides):
|
||||||
|
inc = {
|
||||||
|
"incident_id": incident_id,
|
||||||
|
"system_ids": list(system_ids),
|
||||||
|
"units_active": list(units_active or []),
|
||||||
|
"units_cleared": list(units_cleared or []),
|
||||||
|
"status": "active",
|
||||||
|
"updated_at": (NOW - timedelta(minutes=5)).isoformat(),
|
||||||
|
}
|
||||||
|
inc.update(overrides)
|
||||||
|
return inc
|
||||||
|
|
||||||
|
|
||||||
|
def _ctx(call_units, all_active, system_id="sys-1", now=NOW):
|
||||||
|
return {"call_units": call_units, "all_active": all_active, "system_id": system_id, "now": now}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# _apply_unit_clearance — pure merge logic
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_clearance_moves_unit_from_active_to_cleared():
|
||||||
|
inc = _incident(units_active=["6-3"], units_cleared=[])
|
||||||
|
active, cleared, resolved = _apply_unit_clearance(inc, ["6-3"])
|
||||||
|
assert active == []
|
||||||
|
assert cleared == ["6-3"]
|
||||||
|
assert resolved is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_clearance_leaves_other_active_units_alone():
|
||||||
|
inc = _incident(units_active=["6-3", "6-7"], units_cleared=[])
|
||||||
|
active, cleared, resolved = _apply_unit_clearance(inc, ["6-3"])
|
||||||
|
assert active == ["6-7"]
|
||||||
|
assert cleared == ["6-3"]
|
||||||
|
assert resolved is False # 6-7 still active
|
||||||
|
|
||||||
|
|
||||||
|
def test_clearing_a_unit_not_tracked_as_active_is_a_noop_for_active_list():
|
||||||
|
inc = _incident(units_active=["6-7"], units_cleared=[])
|
||||||
|
active, cleared, resolved = _apply_unit_clearance(inc, ["ghost-unit"])
|
||||||
|
assert active == ["6-7"]
|
||||||
|
assert cleared == ["ghost-unit"]
|
||||||
|
assert resolved is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_units_ever_tracked_does_not_auto_resolve():
|
||||||
|
# An incident that never had a unit signal at all — clearing nothing
|
||||||
|
# must not manufacture a resolve.
|
||||||
|
inc = _incident(units_active=[], units_cleared=[])
|
||||||
|
active, cleared, resolved = _apply_unit_clearance(inc, [])
|
||||||
|
assert resolved is False
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# _release_reassigned_units — reassignment releases the unit from its
|
||||||
|
# PRIOR incident, scoped correctly, without touching that incident's calls
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_reassignment_clears_unit_from_prior_incident():
|
||||||
|
prior = _incident(incident_id="inc-prior", units_active=["6-3", "6-7"])
|
||||||
|
ctx = _ctx(call_units=["6-3"], all_active=[prior])
|
||||||
|
|
||||||
|
doc_sets = []
|
||||||
|
async def fake_doc_set(collection, doc_id, data, merge=True):
|
||||||
|
doc_sets.append((collection, doc_id, data))
|
||||||
|
|
||||||
|
with patch("app.internal.incident_correlator.fstore") as mock_fstore:
|
||||||
|
mock_fstore.doc_set = fake_doc_set
|
||||||
|
await _release_reassigned_units(ctx, exclude_incident_id="inc-new")
|
||||||
|
|
||||||
|
assert len(doc_sets) == 1
|
||||||
|
collection, doc_id, data = doc_sets[0]
|
||||||
|
assert collection == "incidents" and doc_id == "inc-prior"
|
||||||
|
assert data["units_active"] == ["6-7"]
|
||||||
|
assert data["units_cleared"] == ["6-3"]
|
||||||
|
assert "status" not in data # 6-7 still active — not auto-resolved
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_reassignment_auto_resolves_when_last_unit_clears():
|
||||||
|
prior = _incident(incident_id="inc-prior", units_active=["6-3"])
|
||||||
|
ctx = _ctx(call_units=["6-3"], all_active=[prior])
|
||||||
|
|
||||||
|
doc_sets = []
|
||||||
|
async def fake_doc_set(collection, doc_id, data, merge=True):
|
||||||
|
doc_sets.append((collection, doc_id, data))
|
||||||
|
async def fake_doc_get(collection, doc_id):
|
||||||
|
return None # no parent — maybe_resolve_parent exits immediately
|
||||||
|
|
||||||
|
with patch("app.internal.incident_correlator.fstore") as mock_fstore:
|
||||||
|
mock_fstore.doc_set = fake_doc_set
|
||||||
|
mock_fstore.doc_get = fake_doc_get
|
||||||
|
await _release_reassigned_units(ctx, exclude_incident_id=None)
|
||||||
|
|
||||||
|
collection, doc_id, data = doc_sets[0]
|
||||||
|
assert data["status"] == "resolved"
|
||||||
|
assert "resolved_at" in data
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_reassignment_never_touches_the_calls_own_incident():
|
||||||
|
# The call's own decision (link/new) already handled its own incident —
|
||||||
|
# excluding it here prevents double-writing or self-clearing on it.
|
||||||
|
same = _incident(incident_id="inc-new", units_active=["6-3"])
|
||||||
|
ctx = _ctx(call_units=["6-3"], all_active=[same])
|
||||||
|
|
||||||
|
doc_sets = []
|
||||||
|
async def fake_doc_set(collection, doc_id, data, merge=True):
|
||||||
|
doc_sets.append((collection, doc_id, data))
|
||||||
|
|
||||||
|
with patch("app.internal.incident_correlator.fstore") as mock_fstore:
|
||||||
|
mock_fstore.doc_set = fake_doc_set
|
||||||
|
await _release_reassigned_units(ctx, exclude_incident_id="inc-new")
|
||||||
|
|
||||||
|
assert doc_sets == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_reassignment_does_not_cross_systems():
|
||||||
|
other_system = _incident(incident_id="inc-other-sys", units_active=["6-3"], system_ids=("sys-2",))
|
||||||
|
ctx = _ctx(call_units=["6-3"], all_active=[other_system], system_id="sys-1")
|
||||||
|
|
||||||
|
doc_sets = []
|
||||||
|
async def fake_doc_set(collection, doc_id, data, merge=True):
|
||||||
|
doc_sets.append((collection, doc_id, data))
|
||||||
|
|
||||||
|
with patch("app.internal.incident_correlator.fstore") as mock_fstore:
|
||||||
|
mock_fstore.doc_set = fake_doc_set
|
||||||
|
await _release_reassigned_units(ctx, exclude_incident_id=None)
|
||||||
|
|
||||||
|
assert doc_sets == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_reassignment_with_no_call_units_is_a_noop():
|
||||||
|
prior = _incident(incident_id="inc-prior", units_active=["6-3"])
|
||||||
|
ctx = _ctx(call_units=[], all_active=[prior])
|
||||||
|
|
||||||
|
doc_sets = []
|
||||||
|
async def fake_doc_set(collection, doc_id, data, merge=True):
|
||||||
|
doc_sets.append((collection, doc_id, data))
|
||||||
|
|
||||||
|
with patch("app.internal.incident_correlator.fstore") as mock_fstore:
|
||||||
|
mock_fstore.doc_set = fake_doc_set
|
||||||
|
await _release_reassigned_units(ctx, exclude_incident_id=None)
|
||||||
|
|
||||||
|
assert doc_sets == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_reassignment_matches_units_by_normalized_key():
|
||||||
|
# "5-David" vs "5David" — same unit, different transcription — must
|
||||||
|
# still match via the existing _normalize_unit key, not exact string eq.
|
||||||
|
prior = _incident(incident_id="inc-prior", units_active=["5-David"])
|
||||||
|
ctx = _ctx(call_units=["5 David"], all_active=[prior])
|
||||||
|
|
||||||
|
doc_sets = []
|
||||||
|
async def fake_doc_set(collection, doc_id, data, merge=True):
|
||||||
|
doc_sets.append((collection, doc_id, data))
|
||||||
|
async def fake_doc_get(collection, doc_id):
|
||||||
|
return None # no parent — maybe_resolve_parent exits immediately
|
||||||
|
|
||||||
|
with patch("app.internal.incident_correlator.fstore") as mock_fstore:
|
||||||
|
mock_fstore.doc_set = fake_doc_set
|
||||||
|
mock_fstore.doc_get = fake_doc_get
|
||||||
|
await _release_reassigned_units(ctx, exclude_incident_id=None)
|
||||||
|
|
||||||
|
assert len(doc_sets) == 1
|
||||||
|
assert doc_sets[0][2]["units_cleared"] == ["5-David"]
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
"""
|
||||||
|
End-to-end CORS wiring for the one browser-facing REST surface.
|
||||||
|
|
||||||
|
The frontend's Archive page calls GET /calls/search with Authorization +
|
||||||
|
Content-Type headers, which forces the browser to send a CORS preflight
|
||||||
|
first. Before #110 that OPTIONS got a bare 405 with no Access-Control-*
|
||||||
|
headers and the fetch failed with "TypeError: Failed to fetch". These
|
||||||
|
tests drive the real app through TestClient so a regression in the
|
||||||
|
middleware wiring (not just the helper) is caught.
|
||||||
|
|
||||||
|
TestClient is NOT used as a context manager on purpose: that would run the
|
||||||
|
lifespan (mqtt_handler.connect(), the sweeper loops, dynsec bootstrap),
|
||||||
|
none of which is needed here -- CORSMiddleware answers a preflight before
|
||||||
|
routing or dependencies run.
|
||||||
|
"""
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
from app.main import app
|
||||||
|
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
ALLOWED_ORIGIN = "https://drb.cusano.net"
|
||||||
|
DISALLOWED_ORIGIN = "https://evil.example.com"
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_allowed_origin_matches_the_deployed_frontend():
|
||||||
|
# The frontend is served on the bare domain (infra Caddyfile.j2), so the
|
||||||
|
# default must allow exactly that origin without any env override.
|
||||||
|
assert ALLOWED_ORIGIN in settings.cors_origins
|
||||||
|
|
||||||
|
|
||||||
|
def test_preflight_for_calls_search_is_allowed():
|
||||||
|
resp = client.options(
|
||||||
|
"/calls/search",
|
||||||
|
headers={
|
||||||
|
"Origin": ALLOWED_ORIGIN,
|
||||||
|
"Access-Control-Request-Method": "GET",
|
||||||
|
"Access-Control-Request-Headers": "authorization,content-type",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert resp.headers.get("access-control-allow-origin") == ALLOWED_ORIGIN
|
||||||
|
allow_methods = resp.headers.get("access-control-allow-methods", "").upper()
|
||||||
|
assert "GET" in allow_methods
|
||||||
|
# Bearer auth, not cookies -- credentials must never be advertised.
|
||||||
|
assert "access-control-allow-credentials" not in resp.headers
|
||||||
|
|
||||||
|
|
||||||
|
def test_preflight_from_disallowed_origin_gets_no_allow_origin():
|
||||||
|
resp = client.options(
|
||||||
|
"/calls/search",
|
||||||
|
headers={
|
||||||
|
"Origin": DISALLOWED_ORIGIN,
|
||||||
|
"Access-Control-Request-Method": "GET",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert resp.headers.get("access-control-allow-origin") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_simple_get_from_allowed_origin_is_annotated():
|
||||||
|
# Even a non-preflight GET must carry Access-Control-Allow-Origin or the
|
||||||
|
# browser hides the response body from the page.
|
||||||
|
resp = client.get("/health", headers={"Origin": ALLOWED_ORIGIN})
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert resp.headers.get("access-control-allow-origin") == ALLOWED_ORIGIN
|
||||||
@@ -5,8 +5,9 @@ Starlette does not reject `allow_origins=["*"]` combined with
|
|||||||
`allow_credentials=True`. It reflects the caller's Origin back in
|
`allow_credentials=True`. It reflects the caller's Origin back in
|
||||||
Access-Control-Allow-Origin and still sends
|
Access-Control-Allow-Origin and still sends
|
||||||
Access-Control-Allow-Credentials: true, so the effective policy is the
|
Access-Control-Allow-Credentials: true, so the effective policy is the
|
||||||
opposite of what a wildcard usually means. main.py defuses that by turning
|
opposite of what a wildcard usually means. main.py never enables
|
||||||
credentials off whenever it sees a wildcard; these tests hold it to that.
|
credentials at all (auth is a Bearer header, not a cookie), which makes
|
||||||
|
that pair unrepresentable; these tests hold it to that.
|
||||||
|
|
||||||
The policy lives in a pure function so it can be exercised directly --
|
The policy lives in a pure function so it can be exercised directly --
|
||||||
reloading app.main to vary settings drags every router back through import
|
reloading app.main to vary settings drags every router back through import
|
||||||
@@ -28,11 +29,11 @@ def test_wildcard_among_real_origins_still_disables_credentials():
|
|||||||
assert cors_allows_credentials(["https://app.example.com", "*"]) is False
|
assert cors_allows_credentials(["https://app.example.com", "*"]) is False
|
||||||
|
|
||||||
|
|
||||||
def test_named_origins_keep_credentials():
|
def test_credentials_never_enabled_even_for_named_origins():
|
||||||
# Naming your origins is how you ask for credentialed requests, so a
|
# Auth here is a Bearer header, not a cookie, so credentialed CORS is
|
||||||
# correctly configured deployment must not be penalised.
|
# never needed. The predicate is hard-off regardless of the origin list.
|
||||||
assert cors_allows_credentials(["https://app.example.com"]) is True
|
assert cors_allows_credentials(["https://app.example.com"]) is False
|
||||||
assert cors_allows_credentials([]) is True
|
assert cors_allows_credentials([]) is False
|
||||||
|
|
||||||
|
|
||||||
def test_the_app_actually_mounted_that_policy():
|
def test_the_app_actually_mounted_that_policy():
|
||||||
@@ -42,6 +43,7 @@ def test_the_app_actually_mounted_that_policy():
|
|||||||
(mw.kwargs for mw in app.user_middleware if mw.cls is CORSMiddleware), None
|
(mw.kwargs for mw in app.user_middleware if mw.cls is CORSMiddleware), None
|
||||||
)
|
)
|
||||||
assert opts is not None, "CORSMiddleware is not mounted at all"
|
assert opts is not None, "CORSMiddleware is not mounted at all"
|
||||||
|
assert opts["allow_credentials"] is False
|
||||||
assert opts["allow_credentials"] is cors_allows_credentials(settings.cors_origins)
|
assert opts["allow_credentials"] is cors_allows_credentials(settings.cors_origins)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,145 @@
|
|||||||
|
"""
|
||||||
|
server-26#163 — the STT eval harness: word_error_rate() and the three routes
|
||||||
|
that back the /admin "STT Eval" tab.
|
||||||
|
|
||||||
|
Load-bearing property, checked directly: eval annotation must never touch
|
||||||
|
`transcript`/`transcript_corrected`, re-run extraction, or unlink incidents —
|
||||||
|
that's PATCH /{call_id}/transcript's job, a production correction with real
|
||||||
|
side effects. This is pure measurement and must stay pure.
|
||||||
|
"""
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
from app.internal.wer import word_error_rate
|
||||||
|
from app.main import app
|
||||||
|
from app.internal.auth import require_admin_token, require_service_or_firebase_token
|
||||||
|
from app.routers import calls
|
||||||
|
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
ADMIN = {"role": "admin", "org_id": "org-A"}
|
||||||
|
|
||||||
|
|
||||||
|
def _override(decoded: dict):
|
||||||
|
# calls.router carries its own router-level require_service_or_firebase_token
|
||||||
|
# (app/main.py) ON TOP OF each admin route's own require_admin_token — both
|
||||||
|
# have to be overridden or the router-level one 401s before the route's own
|
||||||
|
# dependency is ever evaluated.
|
||||||
|
app.dependency_overrides[require_admin_token] = lambda: decoded
|
||||||
|
app.dependency_overrides[require_service_or_firebase_token] = lambda: decoded
|
||||||
|
|
||||||
|
|
||||||
|
def teardown_function():
|
||||||
|
app.dependency_overrides.pop(require_admin_token, None)
|
||||||
|
app.dependency_overrides.pop(require_service_or_firebase_token, None)
|
||||||
|
|
||||||
|
|
||||||
|
# ── word_error_rate ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_identical_transcripts_are_zero_wer():
|
||||||
|
assert word_error_rate("K on the 600, I'm on Jackson Avenue.",
|
||||||
|
"K on the 600, I'm on Jackson Avenue.") == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_case_and_punctuation_are_ignored():
|
||||||
|
assert word_error_rate("Home Street and Forest Ave!", "home street and forest ave") == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_one_substitution_out_of_three_words():
|
||||||
|
assert word_error_rate("the cat sat", "the cat sit") == pytest.approx(1 / 3)
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_reference_is_undefined_not_zero():
|
||||||
|
"""A call nobody transcribed must not score as a perfect match."""
|
||||||
|
assert word_error_rate("", "anything") is None
|
||||||
|
assert word_error_rate(None, "anything") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_hypothesis_against_real_reference_is_total_loss():
|
||||||
|
assert word_error_rate("home street and forest ave", "") == 1.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_insertion_counts_against_the_hypothesis():
|
||||||
|
# reference 3 words, hypothesis adds 2 extra -> 2 insertions / 3 ref words
|
||||||
|
assert word_error_rate("show me clear", "show me clear right now") == pytest.approx(2 / 3)
|
||||||
|
|
||||||
|
|
||||||
|
# ── GET /calls/eval-queue ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _call(call_id, transcript="a real transcript here", corrected=None, eval_transcript=None, org_id="org-A"):
|
||||||
|
return {
|
||||||
|
"call_id": call_id, "org_id": org_id, "started_at": "2026-09-21T00:00:00+00:00",
|
||||||
|
"transcript": transcript, "transcript_corrected": corrected, "eval_transcript": eval_transcript,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_eval_queue_skips_already_evaluated_and_transcript_less_calls():
|
||||||
|
rows = [
|
||||||
|
_call("c1", eval_transcript="already done"),
|
||||||
|
_call("c2", transcript=None),
|
||||||
|
_call("c3"),
|
||||||
|
]
|
||||||
|
_override(ADMIN)
|
||||||
|
with patch.object(calls.fstore, "collection_where", AsyncMock(return_value=rows)):
|
||||||
|
resp = client.get("/calls/eval-queue")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = resp.json()
|
||||||
|
assert [c["call_id"] for c in body["calls"]] == ["c3"]
|
||||||
|
assert body["matched"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_eval_queue_requires_an_org_scope():
|
||||||
|
_override({"role": "admin"}) # platform admin, no org claim
|
||||||
|
resp = client.get("/calls/eval-queue")
|
||||||
|
assert resp.status_code == 403
|
||||||
|
|
||||||
|
|
||||||
|
# ── GET /calls/eval-stats ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_eval_stats_averages_wer_across_evaluated_calls_only():
|
||||||
|
rows = [
|
||||||
|
_call("c1", transcript="the cat sat", corrected="the cat sat", eval_transcript="the cat sat"), # 0.0 / 0.0
|
||||||
|
_call("c2", transcript="the cat sit", corrected="the cat sat", eval_transcript="the cat sat"), # raw 1/3, corrected 0.0
|
||||||
|
_call("c3", eval_transcript=None), # excluded entirely
|
||||||
|
]
|
||||||
|
_override(ADMIN)
|
||||||
|
with patch.object(calls.fstore, "collection_list", AsyncMock(return_value=rows)):
|
||||||
|
resp = client.get("/calls/eval-stats")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = resp.json()
|
||||||
|
assert body["eval_count"] == 2
|
||||||
|
assert body["raw_wer"] == pytest.approx((0.0 + 1 / 3) / 2, abs=1e-4)
|
||||||
|
assert body["corrected_wer"] == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_eval_stats_with_nothing_evaluated_yet_reports_none_not_zero():
|
||||||
|
_override(ADMIN)
|
||||||
|
with patch.object(calls.fstore, "collection_list", AsyncMock(return_value=[_call("c1")])):
|
||||||
|
resp = client.get("/calls/eval-stats")
|
||||||
|
body = resp.json()
|
||||||
|
assert body == {"eval_count": 0, "raw_wer": None, "corrected_wer": None}
|
||||||
|
|
||||||
|
|
||||||
|
# ── PUT /{call_id}/eval-transcript ──────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_put_eval_transcript_writes_only_eval_fields():
|
||||||
|
_override(ADMIN)
|
||||||
|
existing = _call("c1", transcript="raw text", corrected="corrected text")
|
||||||
|
with patch.object(calls.fstore, "doc_get", AsyncMock(return_value=existing)), \
|
||||||
|
patch.object(calls.fstore, "doc_set", AsyncMock()) as mock_set:
|
||||||
|
resp = client.put("/calls/c1/eval-transcript", json={"text": "the verified ground truth"})
|
||||||
|
assert resp.status_code == 200
|
||||||
|
(collection, doc_id, doc), _ = mock_set.await_args
|
||||||
|
assert collection == "calls" and doc_id == "c1"
|
||||||
|
assert doc["eval_transcript"] == "the verified ground truth"
|
||||||
|
assert doc["eval_transcript_at"]
|
||||||
|
assert "transcript" not in doc and "transcript_corrected" not in doc
|
||||||
|
|
||||||
|
|
||||||
|
def test_put_eval_transcript_404s_on_missing_call():
|
||||||
|
_override(ADMIN)
|
||||||
|
with patch.object(calls.fstore, "doc_get", AsyncMock(return_value=None)):
|
||||||
|
resp = client.put("/calls/nope/eval-transcript", json={"text": "x"})
|
||||||
|
assert resp.status_code == 404
|
||||||
@@ -0,0 +1,185 @@
|
|||||||
|
"""
|
||||||
|
server-26#159: a citywide/patched feed can be received far from its own
|
||||||
|
coverage area — "New York City - NYPD Citywide 2 Patch" was ~56km from the
|
||||||
|
receiving node, well past geocode_max_km (40km). Real, correctly-geocoded
|
||||||
|
addresses on that talkgroup were rejected by intelligence._geocode_location's
|
||||||
|
node-distance sanity check every time, so location_coords never populated for
|
||||||
|
the whole system: location_proximity correlation was permanently dead there,
|
||||||
|
and the same real event reported at two nearby addresses two minutes apart
|
||||||
|
became two separate incidents instead of one.
|
||||||
|
|
||||||
|
`trust_named_region` fixes this narrowly: the node-distance check is a proxy
|
||||||
|
for "is this plausible" that only makes sense when the node's own position is
|
||||||
|
the best guess we have at the area. It must not apply when the query already
|
||||||
|
names a different region on its own terms (operator-set area_context, or a
|
||||||
|
municipality parsed straight from the talkgroup's own name) — and it must
|
||||||
|
never touch the anchor path, whose own radius is always authoritative.
|
||||||
|
"""
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import app.internal.intelligence as intel
|
||||||
|
from app.config import settings
|
||||||
|
|
||||||
|
|
||||||
|
def _maps_result(lat: float, lng: float, location_type: str = "ROOFTOP"):
|
||||||
|
payload = {
|
||||||
|
"status": "OK",
|
||||||
|
"results": [{
|
||||||
|
"geometry": {
|
||||||
|
"location": {"lat": lat, "lng": lng},
|
||||||
|
"location_type": location_type,
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
|
||||||
|
class _Resp:
|
||||||
|
def raise_for_status(self): pass
|
||||||
|
def json(self): return payload
|
||||||
|
|
||||||
|
class _Client:
|
||||||
|
async def __aenter__(self): return self
|
||||||
|
async def __aexit__(self, *a): return False
|
||||||
|
async def get(self, *a, **k): return _Resp()
|
||||||
|
|
||||||
|
return patch("httpx.AsyncClient", lambda *a, **k: _Client())
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _api_key():
|
||||||
|
# intelligence.py imports settings locally per-function (`from app.config
|
||||||
|
# import settings`), which binds the same cached singleton — patching the
|
||||||
|
# module-level object here reaches it, but `intel.settings` itself does
|
||||||
|
# not exist as an attribute.
|
||||||
|
with patch.object(settings, "google_maps_api_key", "test-key"):
|
||||||
|
yield
|
||||||
|
|
||||||
|
|
||||||
|
# Node at (0, 0); result at (1, 0) is ~111km away — well past the 40km default.
|
||||||
|
NODE_LAT, NODE_LON = 0.0, 0.0
|
||||||
|
FAR_LAT, FAR_LNG = 1.0, 0.0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_named_region_geocode_accepted_beyond_node_distance():
|
||||||
|
with _maps_result(FAR_LAT, FAR_LNG):
|
||||||
|
coords = await intel._geocode_location(
|
||||||
|
"1108 Jackson Avenue, New York City - NYPD Citywide 2 Patch",
|
||||||
|
node_lat=NODE_LAT, node_lon=NODE_LON,
|
||||||
|
trust_named_region=True,
|
||||||
|
)
|
||||||
|
assert coords == {"lat": FAR_LAT, "lng": FAR_LNG}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_local_geocode_still_rejected_beyond_node_distance_without_named_region():
|
||||||
|
"""Regression guard: a bare street name with no named region still uses
|
||||||
|
the node as its only plausibility check, exactly as before this fix."""
|
||||||
|
with _maps_result(FAR_LAT, FAR_LNG):
|
||||||
|
coords = await intel._geocode_location(
|
||||||
|
"Main Street",
|
||||||
|
node_lat=NODE_LAT, node_lon=NODE_LON,
|
||||||
|
trust_named_region=False,
|
||||||
|
)
|
||||||
|
assert coords is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_anchor_path_ignores_trust_named_region():
|
||||||
|
"""The anchor's own radius is always authoritative — trust_named_region
|
||||||
|
is only a statement about the node fallback, never a way to widen an
|
||||||
|
anchor that was itself deliberately sized to discriminate."""
|
||||||
|
anchor = {"lat": NODE_LAT, "lng": NODE_LON, "radius_km": 10.0}
|
||||||
|
with _maps_result(FAR_LAT, FAR_LNG):
|
||||||
|
coords = await intel._geocode_location(
|
||||||
|
"1108 Jackson Avenue, New York City - NYPD Citywide 2 Patch",
|
||||||
|
node_lat=NODE_LAT, node_lon=NODE_LON,
|
||||||
|
anchor=anchor, trust_named_region=True,
|
||||||
|
)
|
||||||
|
assert coords is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_named_region_geocode_within_node_distance_is_unaffected():
|
||||||
|
"""A close result is accepted the same way regardless of the flag."""
|
||||||
|
near_lat, near_lng = 0.05, 0.0 # ~5.5km from the node
|
||||||
|
with _maps_result(near_lat, near_lng):
|
||||||
|
coords = await intel._geocode_location(
|
||||||
|
"Main Street, Ossining, New York",
|
||||||
|
node_lat=NODE_LAT, node_lon=NODE_LON,
|
||||||
|
trust_named_region=True,
|
||||||
|
)
|
||||||
|
assert coords == {"lat": near_lat, "lng": near_lng}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_imprecise_result_still_rejected_regardless_of_trust():
|
||||||
|
"""trust_named_region relaxes the distance check only — the location_type
|
||||||
|
precision filter (server-26#37) still applies unconditionally."""
|
||||||
|
with _maps_result(FAR_LAT, FAR_LNG, location_type="APPROXIMATE"):
|
||||||
|
coords = await intel._geocode_location(
|
||||||
|
"1108 Jackson Avenue, New York City - NYPD Citywide 2 Patch",
|
||||||
|
node_lat=NODE_LAT, node_lon=NODE_LON,
|
||||||
|
trust_named_region=True,
|
||||||
|
)
|
||||||
|
assert coords is None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# _location_query_parts — pure query assembly, no HTTP involved
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_operator_configured_area_wins_and_is_named_region():
|
||||||
|
parts, named = intel._location_query_parts(
|
||||||
|
"High Street", {"municipality": "Yorktown", "state": "New York"},
|
||||||
|
"Tac 1", node_state="New York", node_county="Westchester",
|
||||||
|
)
|
||||||
|
assert parts == ["High Street", "Yorktown", "New York"]
|
||||||
|
assert named is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_local_talkgroup_name_gets_node_state_but_not_node_county():
|
||||||
|
"""
|
||||||
|
"Ossining PD" is genuinely local to the node, so appending the node's own
|
||||||
|
state is correct. Its COUNTY is dropped even here — server-26#159's fix
|
||||||
|
applies uniformly once a municipality is derived, since there is no way
|
||||||
|
to tell "local" and "distant-but-node-adjacent" apart from the string
|
||||||
|
alone, and the county was never necessary for a bare municipality name
|
||||||
|
that already disambiguates via the state.
|
||||||
|
"""
|
||||||
|
parts, named = intel._location_query_parts(
|
||||||
|
"High Street", {}, "Ossining PD",
|
||||||
|
node_state="New York", node_county="Westchester",
|
||||||
|
)
|
||||||
|
assert parts == ["High Street", "Ossining", "New York"]
|
||||||
|
assert named is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_citywide_patched_feed_does_not_get_the_nodes_county_grafted_on():
|
||||||
|
"""
|
||||||
|
server-26#159's actual production case: the talkgroup names its own
|
||||||
|
(distant) region, so the node's county (Westchester, ~56km away) must not
|
||||||
|
be appended — it would make the query self-contradictory ("...New York
|
||||||
|
City..., Westchester, New York") and risks degrading the geocode result's
|
||||||
|
precision independently of the distance check this issue also fixes.
|
||||||
|
"""
|
||||||
|
parts, named = intel._location_query_parts(
|
||||||
|
"1108 Jackson Avenue", {}, "New York City - NYPD Citywide 2 Patch",
|
||||||
|
node_state="New York", node_county="Westchester",
|
||||||
|
)
|
||||||
|
assert "Westchester" not in parts
|
||||||
|
assert parts == ["1108 Jackson Avenue", "New York City - NYPD Citywide 2 Patch", "New York"]
|
||||||
|
assert named is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_uninformative_talkgroup_name_falls_back_to_node_county_and_state():
|
||||||
|
"""A tactical channel or bare code gives _municipality_from_tg nothing —
|
||||||
|
the only remaining evidence really is where the node sits, so the
|
||||||
|
original node-county-and-state fallback is preserved for this case."""
|
||||||
|
parts, named = intel._location_query_parts(
|
||||||
|
"High Street", {}, "Tac 1",
|
||||||
|
node_state="New York", node_county="Westchester",
|
||||||
|
)
|
||||||
|
assert parts == ["High Street", "Westchester", "New York"]
|
||||||
|
assert named is False
|
||||||
@@ -304,6 +304,39 @@ async def test_a_scene_is_judged_on_its_own_embedding_and_severity():
|
|||||||
assert ctx["call_severity"] == "major"
|
assert ctx["call_severity"] == "major"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_the_llm_tier_reads_the_scene_transcript_not_the_whole_call():
|
||||||
|
"""
|
||||||
|
server-26#102. intelligence.py writes only the primary scene's corrected
|
||||||
|
text to calls/{id}. _call_block (the LLM correlation prompt) must reason
|
||||||
|
over the SCENE being correlated, not a whole-call transcript that also
|
||||||
|
contains the other scenes. _build_context threads the scene's text in;
|
||||||
|
with no scene text it falls back to the call doc (sweep / single-scene).
|
||||||
|
"""
|
||||||
|
with patch("app.internal.incident_correlator.fstore") as mock_fstore:
|
||||||
|
mock_fstore.doc_get = AsyncMock(return_value={
|
||||||
|
"transcript": "scene one about a fire. scene two about a traffic stop.",
|
||||||
|
})
|
||||||
|
mock_fstore.collection_list = AsyncMock(return_value=[])
|
||||||
|
scene = await _build_context(
|
||||||
|
call_id="call-1", units=None, vehicles=None, cleared_units=None,
|
||||||
|
location_coords=None, reference_time=NOW,
|
||||||
|
system_id="sys-1", talkgroup_id=383, talkgroup_name=DISPATCH_TG,
|
||||||
|
tags=[], incident_type="police", location=None,
|
||||||
|
reassignment=False, create_if_new=True,
|
||||||
|
transcript="scene two about a traffic stop.",
|
||||||
|
)
|
||||||
|
fallback = await _build_context(
|
||||||
|
call_id="call-1", units=None, vehicles=None, cleared_units=None,
|
||||||
|
location_coords=None, reference_time=NOW,
|
||||||
|
system_id="sys-1", talkgroup_id=383, talkgroup_name=DISPATCH_TG,
|
||||||
|
tags=[], incident_type="police", location=None,
|
||||||
|
reassignment=False, create_if_new=True,
|
||||||
|
)
|
||||||
|
assert scene["scene_transcript"] == "scene two about a traffic stop."
|
||||||
|
assert fallback["scene_transcript"] == "scene one about a fire. scene two about a traffic stop."
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_a_bare_number_never_becomes_an_incident_location_or_title():
|
async def test_a_bare_number_never_becomes_an_incident_location_or_title():
|
||||||
inc = await _create(tags=["flames"], location="49", coords=None,
|
inc = await _create(tags=["flames"], location="49", coords=None,
|
||||||
|
|||||||
@@ -0,0 +1,54 @@
|
|||||||
|
"""
|
||||||
|
server-26#<pending> — no per-system unit-ID format awareness existed anywhere
|
||||||
|
in the pipeline (vocabulary_learner's "known local terms" is a flat glossary,
|
||||||
|
not a structured format). Departments use incompatible unit ID conventions
|
||||||
|
(Yorktown: "5-David", sometimes spoken as bare "David"; County:
|
||||||
|
"SAM-1"/"airport-3"/"parks-4", a location word + number) and the extraction
|
||||||
|
prompt had no way to be told which one a given system uses. This pins the
|
||||||
|
prompt-block builder and the template wiring that carries it.
|
||||||
|
"""
|
||||||
|
from app.internal.intelligence import (
|
||||||
|
_PROMPT_TEMPLATE, _build_unit_format_block, _build_ten_codes_block,
|
||||||
|
_build_transcript_block,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_hint_produces_no_block():
|
||||||
|
assert _build_unit_format_block(None) == ""
|
||||||
|
assert _build_unit_format_block("") == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_hint_is_labelled_and_fed_to_the_model_verbatim():
|
||||||
|
block = _build_unit_format_block(
|
||||||
|
"Yorktown: <district>-<phonetic name>, e.g. 5-David. Sometimes spoken as just the name alone."
|
||||||
|
)
|
||||||
|
assert "unit ID format" in block
|
||||||
|
assert "5-David" in block
|
||||||
|
|
||||||
|
|
||||||
|
def test_prompt_template_renders_with_all_blocks_including_empty_unit_format():
|
||||||
|
# Regression guard: a missing placeholder in .format() raises KeyError at
|
||||||
|
# request time, not import time — this is the cheapest way to catch that
|
||||||
|
# before it reaches a live call.
|
||||||
|
rendered = _PROMPT_TEMPLATE.format(
|
||||||
|
transcript_block=_build_transcript_block("1. Test.", None),
|
||||||
|
talkgroup_name="Test TG",
|
||||||
|
system_id="sys-1",
|
||||||
|
ten_codes_block=_build_ten_codes_block({}),
|
||||||
|
vocabulary_block="",
|
||||||
|
unit_format_block=_build_unit_format_block(""),
|
||||||
|
)
|
||||||
|
assert "Test TG" in rendered
|
||||||
|
assert "1. Test." in rendered
|
||||||
|
|
||||||
|
|
||||||
|
def test_prompt_template_renders_with_a_populated_unit_format_block():
|
||||||
|
rendered = _PROMPT_TEMPLATE.format(
|
||||||
|
transcript_block=_build_transcript_block("1. Test.", None),
|
||||||
|
talkgroup_name="Test TG",
|
||||||
|
system_id="sys-1",
|
||||||
|
ten_codes_block=_build_ten_codes_block({}),
|
||||||
|
vocabulary_block="",
|
||||||
|
unit_format_block=_build_unit_format_block("County: <location>-<number>, e.g. SAM-1, airport-3."),
|
||||||
|
)
|
||||||
|
assert "SAM-1" in rendered
|
||||||
@@ -0,0 +1,197 @@
|
|||||||
|
"""
|
||||||
|
server-26#96 — every scene of a multi-scene call writes corr_debug onto the
|
||||||
|
SAME call doc via incident_correlator._apply_and_log, last-scene-wins. The
|
||||||
|
fix additionally nests each scene's corr_debug/transcript/incident_id under
|
||||||
|
scenes.<scene_index> on the call doc, keyed so Firestore's
|
||||||
|
`set(merge=True)` (a recursive merge of nested map fields — this is the
|
||||||
|
behaviour these tests assume and pin) lands each scene in its own map entry
|
||||||
|
instead of colliding.
|
||||||
|
|
||||||
|
Firestore itself isn't available in this sandbox (see tests/conftest.py), so
|
||||||
|
`_fake_doc_set` below implements that documented recursive-merge semantics by
|
||||||
|
hand and is used as the fstore stand-in — these tests both exercise
|
||||||
|
_apply_and_log's write shape AND pin the merge behaviour it depends on.
|
||||||
|
"""
|
||||||
|
import pytest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from app.internal import incident_correlator
|
||||||
|
|
||||||
|
|
||||||
|
def _merge(dst: dict, src: dict) -> None:
|
||||||
|
"""Firestore DocumentReference.set(data, merge=True) semantics: nested
|
||||||
|
map fields are merged recursively by key, not replaced wholesale."""
|
||||||
|
for k, v in src.items():
|
||||||
|
if isinstance(v, dict) and isinstance(dst.get(k), dict):
|
||||||
|
_merge(dst[k], v)
|
||||||
|
else:
|
||||||
|
dst[k] = v
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_multiscene_call_lands_each_scene_distinctly_and_flat_fields_last_write_wins():
|
||||||
|
docs: dict[tuple, dict] = {}
|
||||||
|
|
||||||
|
async def fake_doc_set(collection, doc_id, data, merge=True):
|
||||||
|
docs.setdefault((collection, doc_id), {})
|
||||||
|
_merge(docs[(collection, doc_id)], data)
|
||||||
|
|
||||||
|
decision0 = {
|
||||||
|
"action": "orphan", "matched_incident": None, "incident_type": None,
|
||||||
|
"corr_debug": {"corr_path": "new", "corr_consensus": "agreed"},
|
||||||
|
}
|
||||||
|
ctx0 = {"call_id": "call-1", "scene_index": 0, "scene_transcript": "scene zero text"}
|
||||||
|
|
||||||
|
decision1 = {
|
||||||
|
"action": "orphan", "matched_incident": None, "incident_type": None,
|
||||||
|
"corr_debug": {"corr_path": "slow", "corr_consensus": "tiebreak"},
|
||||||
|
}
|
||||||
|
ctx1 = {"call_id": "call-1", "scene_index": 1, "scene_transcript": "scene one text"}
|
||||||
|
|
||||||
|
with patch.object(incident_correlator, "fstore") as mock_fstore:
|
||||||
|
mock_fstore.doc_set = fake_doc_set
|
||||||
|
await incident_correlator._apply_and_log(decision0, ctx0)
|
||||||
|
await incident_correlator._apply_and_log(decision1, ctx1)
|
||||||
|
|
||||||
|
doc = docs[("calls", "call-1")]
|
||||||
|
|
||||||
|
# Flat top-level fields: unchanged behaviour, last scene's write wins —
|
||||||
|
# the safe backward-compatible default for any reader that doesn't yet
|
||||||
|
# know about `scenes`.
|
||||||
|
assert doc["corr_path"] == "slow"
|
||||||
|
assert doc["corr_consensus"] == "tiebreak"
|
||||||
|
|
||||||
|
# New `scenes` map: both scenes present, distinct, uncorrupted by the
|
||||||
|
# second write.
|
||||||
|
assert set(doc["scenes"].keys()) == {"0", "1"}
|
||||||
|
assert doc["scenes"]["0"]["corr_debug"]["corr_path"] == "new"
|
||||||
|
assert doc["scenes"]["0"]["corr_debug"]["corr_consensus"] == "agreed"
|
||||||
|
assert doc["scenes"]["0"]["transcript"] == "scene zero text"
|
||||||
|
assert doc["scenes"]["1"]["corr_debug"]["corr_path"] == "slow"
|
||||||
|
assert doc["scenes"]["1"]["corr_debug"]["corr_consensus"] == "tiebreak"
|
||||||
|
assert doc["scenes"]["1"]["transcript"] == "scene one text"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_scene_entry_records_which_incident_it_resolved_to():
|
||||||
|
"""summarizer.py (#114) needs this to pick the right scene per incident."""
|
||||||
|
docs: dict[tuple, dict] = {}
|
||||||
|
|
||||||
|
async def fake_doc_set(collection, doc_id, data, merge=True):
|
||||||
|
docs.setdefault((collection, doc_id), {})
|
||||||
|
_merge(docs[(collection, doc_id)], data)
|
||||||
|
|
||||||
|
with patch.object(incident_correlator, "fstore") as mock_fstore, \
|
||||||
|
patch.object(incident_correlator, "_apply_decision", return_value="inc-42"):
|
||||||
|
mock_fstore.doc_set = fake_doc_set
|
||||||
|
decision = {
|
||||||
|
"action": "new", "matched_incident": None, "incident_type": "fire",
|
||||||
|
"corr_debug": {"corr_path": "new"},
|
||||||
|
}
|
||||||
|
ctx = {"call_id": "call-2", "scene_index": 0, "scene_transcript": "structure fire"}
|
||||||
|
incident_id = await incident_correlator._apply_and_log(decision, ctx)
|
||||||
|
|
||||||
|
assert incident_id == "inc-42"
|
||||||
|
assert docs[("calls", "call-2")]["scenes"]["0"]["incident_id"] == "inc-42"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_single_scene_call_still_gets_a_scenes_map_equivalent_to_flat_fields():
|
||||||
|
"""scene_index defaults to 0 for every caller with no scene concept, so a
|
||||||
|
plain single-scene call is one entry in `scenes` — equivalent to reading
|
||||||
|
the flat fields, not a behaviour change for that population."""
|
||||||
|
docs: dict[tuple, dict] = {}
|
||||||
|
|
||||||
|
async def fake_doc_set(collection, doc_id, data, merge=True):
|
||||||
|
docs.setdefault((collection, doc_id), {})
|
||||||
|
_merge(docs[(collection, doc_id)], data)
|
||||||
|
|
||||||
|
decision = {
|
||||||
|
"action": "orphan", "matched_incident": None, "incident_type": None,
|
||||||
|
"corr_debug": {"corr_path": "fast/thin", "corr_consensus": "rules_only"},
|
||||||
|
}
|
||||||
|
ctx = {"call_id": "call-3", "scene_transcript": "10-4"} # no scene_index key at all
|
||||||
|
|
||||||
|
with patch.object(incident_correlator, "fstore") as mock_fstore:
|
||||||
|
mock_fstore.doc_set = fake_doc_set
|
||||||
|
await incident_correlator._apply_and_log(decision, ctx)
|
||||||
|
|
||||||
|
doc = docs[("calls", "call-3")]
|
||||||
|
assert doc["corr_path"] == "fast/thin"
|
||||||
|
assert doc["scenes"] == {
|
||||||
|
"0": {
|
||||||
|
"transcript": "10-4",
|
||||||
|
"incident_id": None,
|
||||||
|
"corr_debug": {"corr_path": "fast/thin", "corr_consensus": "rules_only"},
|
||||||
|
"incident_type": None,
|
||||||
|
"severity": None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_scene_entry_captures_its_own_incident_type_not_a_sibling_scenes():
|
||||||
|
"""
|
||||||
|
server-26#139: _call_is_substanceless's "type" veto reads ctx["incident_type"]
|
||||||
|
at decision time, but that value was never persisted per-scene — only the
|
||||||
|
last-scene-wins flat field, which #138's dump analysis couldn't
|
||||||
|
distinguish from cross-scene contamination. Pins _apply_and_log's write
|
||||||
|
side: each scene's own scenes.<n> entry carries its own incident_type/
|
||||||
|
severity, distinct from any other scene on the same call. Does NOT cover
|
||||||
|
whether the ctx handed to _call_is_substanceless is the same object that
|
||||||
|
reaches here — that linkage is pinned by test_consensus_gate.py and
|
||||||
|
test_incident_identity.py, not this file.
|
||||||
|
"""
|
||||||
|
docs: dict[tuple, dict] = {}
|
||||||
|
|
||||||
|
async def fake_doc_set(collection, doc_id, data, merge=True):
|
||||||
|
docs.setdefault((collection, doc_id), {})
|
||||||
|
_merge(docs[(collection, doc_id)], data)
|
||||||
|
|
||||||
|
decision0 = {
|
||||||
|
"action": "orphan", "matched_incident": None, "incident_type": None,
|
||||||
|
"corr_debug": {"corr_path": "new", "corr_consensus": "tiebreak", "corr_gate_veto": "type"},
|
||||||
|
}
|
||||||
|
ctx0 = {
|
||||||
|
"call_id": "call-5", "scene_index": 0, "scene_transcript": "10-4, clear",
|
||||||
|
"incident_type": "traffic-stop", "call_severity": "routine",
|
||||||
|
}
|
||||||
|
|
||||||
|
decision1 = {
|
||||||
|
"action": "orphan", "matched_incident": None, "incident_type": None,
|
||||||
|
"corr_debug": {"corr_path": "new", "corr_consensus": "agreed"},
|
||||||
|
}
|
||||||
|
ctx1 = {
|
||||||
|
"call_id": "call-5", "scene_index": 1, "scene_transcript": "roll call",
|
||||||
|
"incident_type": None, "call_severity": "moderate",
|
||||||
|
}
|
||||||
|
|
||||||
|
with patch.object(incident_correlator, "fstore") as mock_fstore:
|
||||||
|
mock_fstore.doc_set = fake_doc_set
|
||||||
|
await incident_correlator._apply_and_log(decision0, ctx0)
|
||||||
|
await incident_correlator._apply_and_log(decision1, ctx1)
|
||||||
|
|
||||||
|
doc = docs[("calls", "call-5")]
|
||||||
|
scenes = doc["scenes"]
|
||||||
|
assert scenes["0"]["incident_type"] == "traffic-stop"
|
||||||
|
assert scenes["0"]["severity"] == "routine"
|
||||||
|
assert scenes["1"]["incident_type"] is None
|
||||||
|
assert scenes["1"]["severity"] == "moderate"
|
||||||
|
# _apply_and_log only ever flat-merges corr_debug's own keys (:1460) — a
|
||||||
|
# future corr_debug["incident_type"] would silently clobber
|
||||||
|
# intelligence.py's flat field, so this is asserted, not just commented.
|
||||||
|
assert "incident_type" not in doc
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_empty_corr_debug_writes_nothing_same_as_before():
|
||||||
|
"""Preserve the pre-#96 short-circuit: no corr_debug means no write at
|
||||||
|
all, flat or nested."""
|
||||||
|
with patch.object(incident_correlator, "fstore") as mock_fstore, \
|
||||||
|
patch.object(incident_correlator, "_apply_decision", return_value=None):
|
||||||
|
mock_fstore.doc_set = None # would raise TypeError if ever called
|
||||||
|
decision = {"action": "orphan", "matched_incident": None, "incident_type": None, "corr_debug": {}}
|
||||||
|
ctx = {"call_id": "call-4", "scene_index": 0, "scene_transcript": "x"}
|
||||||
|
result = await incident_correlator._apply_and_log(decision, ctx)
|
||||||
|
|
||||||
|
assert result is None
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
"""
|
||||||
|
server-26#131 — the re-correlation sweep's orphan filter checked incident_id/
|
||||||
|
incident_ids/corr_path but had no way to tell "never processed" apart from
|
||||||
|
"real-time pipeline (routers/upload.py _run_intelligence_pipeline) is still
|
||||||
|
mid-flight". Racing the sweep against an in-flight real-time correlation could
|
||||||
|
land the same call on two different incidents — the exact duplicate-link bug
|
||||||
|
#131 found in 3 live dumps (~2% of linked calls). This pins the fix: a call
|
||||||
|
whose intelligence_started_at marker is recent is held back from the sweep
|
||||||
|
regardless of how orphaned it otherwise looks.
|
||||||
|
"""
|
||||||
|
from datetime import datetime, timezone, timedelta
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.internal import recorrelation_sweep
|
||||||
|
|
||||||
|
|
||||||
|
def _iso(dt: datetime) -> str:
|
||||||
|
return dt.isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
class TestPipelineLikelyStillRunning:
|
||||||
|
def test_recent_marker_is_still_running(self):
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
call = {"intelligence_started_at": _iso(now - timedelta(minutes=1))}
|
||||||
|
assert recorrelation_sweep._pipeline_likely_still_running(call, now) is True
|
||||||
|
|
||||||
|
def test_old_marker_is_not_still_running(self):
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
call = {"intelligence_started_at": _iso(now - timedelta(minutes=30))}
|
||||||
|
assert recorrelation_sweep._pipeline_likely_still_running(call, now) is False
|
||||||
|
|
||||||
|
def test_marker_exactly_at_the_threshold_is_not_held_back(self):
|
||||||
|
# age_minutes < MIN_MINUTES_SINCE_PIPELINE_START (strict), so exactly
|
||||||
|
# at the threshold is old enough to release — pins the boundary so it
|
||||||
|
# can't drift to <= by accident and silently double the hold time.
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
threshold = recorrelation_sweep.MIN_MINUTES_SINCE_PIPELINE_START
|
||||||
|
call = {"intelligence_started_at": _iso(now - timedelta(minutes=threshold))}
|
||||||
|
assert recorrelation_sweep._pipeline_likely_still_running(call, now) is False
|
||||||
|
|
||||||
|
def test_no_marker_at_all_is_not_held_back(self):
|
||||||
|
"""A pre-#131 call doc, or the marker write itself failed — absence
|
||||||
|
isn't evidence of an in-flight pipeline, so the sweep must still be
|
||||||
|
able to pick these up (that's its whole job)."""
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
assert recorrelation_sweep._pipeline_likely_still_running({}, now) is False
|
||||||
|
|
||||||
|
def test_unparseable_marker_is_not_held_back(self):
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
call = {"intelligence_started_at": "not-a-timestamp"}
|
||||||
|
assert recorrelation_sweep._pipeline_likely_still_running(call, now) is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_sweep_pass_skips_a_call_whose_pipeline_just_started():
|
||||||
|
"""Integration-shaped: a call that looks orphaned by every OTHER filter
|
||||||
|
(no incident_ids, no corr_path, no skip_reason, under the attempt budget)
|
||||||
|
but has a fresh intelligence_started_at must not reach correlate_call —
|
||||||
|
that's the race #131 found."""
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
racing_call = {
|
||||||
|
"call_id": "call-racing",
|
||||||
|
"started_at": _iso(now - timedelta(minutes=2)),
|
||||||
|
"ended_at": _iso(now - timedelta(minutes=1)),
|
||||||
|
"intelligence_started_at": _iso(now - timedelta(seconds=30)),
|
||||||
|
}
|
||||||
|
genuinely_orphaned_call = {
|
||||||
|
"call_id": "call-genuine-orphan",
|
||||||
|
"started_at": _iso(now - timedelta(minutes=20)),
|
||||||
|
"ended_at": _iso(now - timedelta(minutes=19)),
|
||||||
|
"intelligence_started_at": _iso(now - timedelta(minutes=19)),
|
||||||
|
}
|
||||||
|
|
||||||
|
async def fake_collection_where(collection, clauses):
|
||||||
|
assert collection == "calls"
|
||||||
|
return [racing_call, genuinely_orphaned_call]
|
||||||
|
|
||||||
|
correlate_calls: list[str] = []
|
||||||
|
|
||||||
|
async def fake_correlate_call(**kwargs):
|
||||||
|
correlate_calls.append(kwargs["call_id"])
|
||||||
|
return None # no match — exercises the "not linked" branch too
|
||||||
|
|
||||||
|
doc_sets: list[tuple] = []
|
||||||
|
|
||||||
|
async def fake_doc_set(collection, doc_id, data, merge=True):
|
||||||
|
doc_sets.append((collection, doc_id, data))
|
||||||
|
|
||||||
|
with patch.object(recorrelation_sweep, "fstore") as mock_fstore, \
|
||||||
|
patch("app.internal.incident_correlator.correlate_call", fake_correlate_call):
|
||||||
|
mock_fstore.collection_where = fake_collection_where
|
||||||
|
mock_fstore.doc_set = fake_doc_set
|
||||||
|
await recorrelation_sweep._run_sweep_pass()
|
||||||
|
|
||||||
|
assert correlate_calls == ["call-genuine-orphan"]
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
"""
|
||||||
|
server-26#96/#114 review (PR #132): `PATCH /calls/{id}/transcript` clears
|
||||||
|
stale intelligence fields before re-extraction runs, but `doc_set(...,
|
||||||
|
merge=True)` can only add/overwrite keys in a nested map, never remove one.
|
||||||
|
A call corrected from 3 scenes down to 1 would keep `scenes.1`/`scenes.2`
|
||||||
|
with pre-correction transcripts and incident_ids forever -- corrupting the
|
||||||
|
per-scene tally #96 exists to make trustworthy, and re-feeding stale text
|
||||||
|
into #114's summarizer fix if a stale scene's incident_id still names a real
|
||||||
|
incident. The fix deletes the field with `fstore.DELETE_FIELD` instead of
|
||||||
|
merging over it with an empty map (which is a no-op).
|
||||||
|
"""
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi import BackgroundTasks
|
||||||
|
|
||||||
|
from app.internal import firestore as fstore
|
||||||
|
from app.routers.calls import TranscriptUpdate, patch_transcript
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_transcript_correction_deletes_the_scenes_field_not_merges_over_it():
|
||||||
|
call = {
|
||||||
|
"call_id": "call-1",
|
||||||
|
"system_id": "sys-1",
|
||||||
|
"node_id": "node-1",
|
||||||
|
"transcript": "old raw text",
|
||||||
|
# Simulates a prior 3-scene call, per #96's schema.
|
||||||
|
"scenes": {
|
||||||
|
"0": {"transcript": "scene zero", "incident_id": "inc-a", "corr_debug": {}},
|
||||||
|
"1": {"transcript": "scene one", "incident_id": "inc-b", "corr_debug": {}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
doc_set_calls: list[tuple] = []
|
||||||
|
doc_update_calls: list[tuple] = []
|
||||||
|
|
||||||
|
async def fake_doc_get(collection, doc_id):
|
||||||
|
if collection == "calls" and doc_id == "call-1":
|
||||||
|
return call
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def fake_doc_set(collection, doc_id, data, merge=True):
|
||||||
|
doc_set_calls.append((collection, doc_id, data))
|
||||||
|
|
||||||
|
async def fake_doc_update(collection, doc_id, data):
|
||||||
|
doc_update_calls.append((collection, doc_id, data))
|
||||||
|
|
||||||
|
fake_flags = (None, lambda name: name == "correlation_enabled")
|
||||||
|
|
||||||
|
with patch("app.routers.calls.fstore.doc_get", new=fake_doc_get), \
|
||||||
|
patch("app.routers.calls.fstore.doc_set", new=fake_doc_set), \
|
||||||
|
patch("app.routers.calls.fstore.doc_update", new=fake_doc_update), \
|
||||||
|
patch("app.internal.feature_flags.resolve_flags", new=AsyncMock(return_value=fake_flags)):
|
||||||
|
result = await patch_transcript(
|
||||||
|
call_id="call-1",
|
||||||
|
body=TranscriptUpdate(transcript="corrected text"),
|
||||||
|
background_tasks=BackgroundTasks(),
|
||||||
|
_={},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == {"ok": True, "call_id": "call-1"}
|
||||||
|
|
||||||
|
# The stale scenes map must be DELETED, not merged over with {} (a no-op
|
||||||
|
# under Firestore's set(merge=True) semantics) and not left untouched by
|
||||||
|
# a doc_set call that never mentions it.
|
||||||
|
scenes_deletions = [
|
||||||
|
(coll, doc_id, data) for (coll, doc_id, data) in doc_update_calls
|
||||||
|
if coll == "calls" and doc_id == "call-1" and "scenes" in data
|
||||||
|
]
|
||||||
|
assert len(scenes_deletions) == 1, (
|
||||||
|
f"expected exactly one doc_update clearing 'scenes', got {doc_update_calls}"
|
||||||
|
)
|
||||||
|
assert scenes_deletions[0][2]["scenes"] is fstore.DELETE_FIELD
|
||||||
|
|
||||||
|
# And no doc_set call should paper over the same field with an empty map
|
||||||
|
# instead -- that would silently do nothing and leave stale scenes intact.
|
||||||
|
for (coll, doc_id, data) in doc_set_calls:
|
||||||
|
if coll == "calls" and doc_id == "call-1":
|
||||||
|
assert "scenes" not in data, (
|
||||||
|
"a doc_set (merge=True) write must never carry 'scenes' -- "
|
||||||
|
"merging {} over an existing map is a no-op, not a delete"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete_field_is_the_real_firestore_sentinel():
|
||||||
|
"""Catches an import-path typo turning this into a silent no-op sentinel."""
|
||||||
|
from firebase_admin import firestore as fs
|
||||||
|
assert fstore.DELETE_FIELD is fs.DELETE_FIELD
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
"""
|
||||||
|
server-26#102 — a scene is correlated on its OWN transcript, not the whole call.
|
||||||
|
|
||||||
|
_scene_transcript_text slices the segments a scene owns. It must never return
|
||||||
|
"" (an empty slice would let incident_correlator._build_context fall back to
|
||||||
|
the call doc's whole-call transcript, re-opening the leak in exactly the case
|
||||||
|
— bad indices — where it matters).
|
||||||
|
"""
|
||||||
|
from app.internal.intelligence import _scene_transcript_text
|
||||||
|
|
||||||
|
SEGS = [
|
||||||
|
{"text": "structure fire, 12 Main"},
|
||||||
|
{"text": "engine 4 responding"},
|
||||||
|
{"text": "traffic stop, plate ABC"},
|
||||||
|
{"text": "one occupant"},
|
||||||
|
]
|
||||||
|
WHOLE = "structure fire, 12 Main engine 4 responding traffic stop, plate ABC one occupant"
|
||||||
|
|
||||||
|
|
||||||
|
def test_scene_owns_a_subset_of_segments():
|
||||||
|
assert _scene_transcript_text(WHOLE, SEGS, [0, 1], None) == "structure fire, 12 Main engine 4 responding"
|
||||||
|
assert _scene_transcript_text(WHOLE, SEGS, [2, 3], None) == "traffic stop, plate ABC one occupant"
|
||||||
|
|
||||||
|
|
||||||
|
def test_corrected_text_wins_when_present():
|
||||||
|
assert _scene_transcript_text(WHOLE, SEGS, [0], "cleaned up text") == "cleaned up text"
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_segment_indices_falls_back_to_whole_call():
|
||||||
|
# single-segment calls are never numbered by _build_transcript_block → null indices
|
||||||
|
assert _scene_transcript_text(WHOLE, SEGS, None, None) == WHOLE
|
||||||
|
assert _scene_transcript_text(WHOLE, None, [0, 1], None) == WHOLE
|
||||||
|
|
||||||
|
|
||||||
|
def test_out_of_range_or_nonint_indices_fall_back_never_empty():
|
||||||
|
assert _scene_transcript_text(WHOLE, SEGS, [9, 10], None) == WHOLE # all out of range
|
||||||
|
assert _scene_transcript_text(WHOLE, SEGS, ["1", "2"], None) == WHOLE # 1-based strings, rejected
|
||||||
|
assert _scene_transcript_text(WHOLE, SEGS, [-1], None) == WHOLE # negative
|
||||||
|
# partial validity: keep what's in range
|
||||||
|
assert _scene_transcript_text(WHOLE, SEGS, [3, 99], None) == "one occupant"
|
||||||
@@ -0,0 +1,124 @@
|
|||||||
|
"""
|
||||||
|
server-26#114 — the incident summarizer used to read doc["transcript"] (the
|
||||||
|
WHOLE call, raw) for every linked call, so a multi-scene call contributed
|
||||||
|
text from scenes it wasn't part of into an incident's summary, and
|
||||||
|
transcript_corrected was never consulted at all.
|
||||||
|
|
||||||
|
Fix: _scene_text_for_incident reads the server-26#96 `scenes` map to find the
|
||||||
|
scene(s) that actually resolved into a given incident_id, and falls back to
|
||||||
|
transcript_corrected-or-transcript for a call doc with no `scenes` field
|
||||||
|
(predates #96).
|
||||||
|
"""
|
||||||
|
import pytest
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
from app.internal import summarizer
|
||||||
|
from app.internal.summarizer import _scene_text_for_incident
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# _scene_text_for_incident — pure function, no Firestore
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_picks_the_scene_that_linked_to_this_incident():
|
||||||
|
doc = {
|
||||||
|
"transcript": "whole raw transcript blend",
|
||||||
|
"transcript_corrected": "whole corrected transcript blend",
|
||||||
|
"scenes": {
|
||||||
|
"0": {"transcript": "scene zero text", "incident_id": "inc-A", "corr_debug": {}},
|
||||||
|
"1": {"transcript": "scene one text", "incident_id": "inc-B", "corr_debug": {}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
assert _scene_text_for_incident(doc, "inc-A") == "scene zero text"
|
||||||
|
assert _scene_text_for_incident(doc, "inc-B") == "scene one text"
|
||||||
|
|
||||||
|
|
||||||
|
def test_joins_multiple_scenes_linked_to_the_same_incident_in_scene_order():
|
||||||
|
doc = {
|
||||||
|
"scenes": {
|
||||||
|
"1": {"transcript": "second", "incident_id": "inc-A"},
|
||||||
|
"0": {"transcript": "first", "incident_id": "inc-A"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
assert _scene_text_for_incident(doc, "inc-A") == "first\nsecond"
|
||||||
|
|
||||||
|
|
||||||
|
def test_old_schema_doc_falls_back_to_transcript_corrected_over_transcript():
|
||||||
|
doc = {"transcript": "raw", "transcript_corrected": "corrected"}
|
||||||
|
assert _scene_text_for_incident(doc, "inc-A") == "corrected"
|
||||||
|
|
||||||
|
|
||||||
|
def test_old_schema_doc_with_only_raw_transcript_still_returns_it():
|
||||||
|
doc = {"transcript": "raw only"}
|
||||||
|
assert _scene_text_for_incident(doc, "inc-A") == "raw only"
|
||||||
|
|
||||||
|
|
||||||
|
def test_scenes_present_but_none_match_falls_back_defensively():
|
||||||
|
"""Should not happen for a call_id genuinely in this incident's call_ids,
|
||||||
|
but silently dropping the call's contribution would be worse than a
|
||||||
|
whole-call fallback."""
|
||||||
|
doc = {
|
||||||
|
"transcript": "raw",
|
||||||
|
"transcript_corrected": "corrected",
|
||||||
|
"scenes": {"0": {"transcript": "x", "incident_id": "inc-OTHER"}},
|
||||||
|
}
|
||||||
|
assert _scene_text_for_incident(doc, "inc-A") == "corrected"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# _summarize_incident — end to end with fstore/Gemini mocked
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_summarize_incident_uses_scene_specific_text_for_a_multiscene_call():
|
||||||
|
"""
|
||||||
|
call-1 is a 2-scene call: scene 0 linked into inc-OTHER, scene 1 linked
|
||||||
|
into inc-1 (the incident being summarized). Only scene 1's text may reach
|
||||||
|
the model.
|
||||||
|
"""
|
||||||
|
call_1 = {
|
||||||
|
"call_id": "call-1",
|
||||||
|
"transcript": "scene zero text scene one text", # the old, wrong, whole-call blend
|
||||||
|
"scenes": {
|
||||||
|
"0": {"transcript": "scene zero text", "incident_id": "inc-OTHER"},
|
||||||
|
"1": {"transcript": "scene one text", "incident_id": "inc-1"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
async def fake_doc_get(collection, doc_id):
|
||||||
|
assert collection == "calls"
|
||||||
|
return call_1 if doc_id == "call-1" else None
|
||||||
|
|
||||||
|
with patch("app.internal.feature_flags.get_flags",
|
||||||
|
AsyncMock(return_value={"summaries_enabled": True})), \
|
||||||
|
patch.object(summarizer, "fstore") as fs, \
|
||||||
|
patch.object(summarizer, "_sync_summarize", return_value="a summary") as sync:
|
||||||
|
fs.doc_get = AsyncMock(side_effect=fake_doc_get)
|
||||||
|
fs.doc_set = AsyncMock()
|
||||||
|
await summarizer._summarize_incident({"incident_id": "inc-1", "call_ids": ["call-1"]})
|
||||||
|
|
||||||
|
sync.assert_called_once()
|
||||||
|
_inc_arg, transcripts_arg = sync.call_args.args
|
||||||
|
assert transcripts_arg == ["scene one text"]
|
||||||
|
assert "scene zero text scene one text" not in transcripts_arg
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_summarize_incident_falls_back_for_old_schema_call_doc():
|
||||||
|
"""A call doc with no `scenes` field at all — summarizer must still work,
|
||||||
|
using transcript_corrected over raw transcript."""
|
||||||
|
call_1 = {"call_id": "call-1", "transcript": "raw", "transcript_corrected": "corrected"}
|
||||||
|
|
||||||
|
async def fake_doc_get(collection, doc_id):
|
||||||
|
return call_1 if doc_id == "call-1" else None
|
||||||
|
|
||||||
|
with patch("app.internal.feature_flags.get_flags",
|
||||||
|
AsyncMock(return_value={"summaries_enabled": True})), \
|
||||||
|
patch.object(summarizer, "fstore") as fs, \
|
||||||
|
patch.object(summarizer, "_sync_summarize", return_value="a summary") as sync:
|
||||||
|
fs.doc_get = AsyncMock(side_effect=fake_doc_get)
|
||||||
|
fs.doc_set = AsyncMock()
|
||||||
|
await summarizer._summarize_incident({"incident_id": "inc-1", "call_ids": ["call-1"]})
|
||||||
|
|
||||||
|
_inc_arg, transcripts_arg = sync.call_args.args
|
||||||
|
assert transcripts_arg == ["corrected"]
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
"""
|
||||||
|
node-26#9 — second-SDR ADS-B telemetry ingestion.
|
||||||
|
|
||||||
|
Two things matter here: the endpoint requires node identity (a service/admin
|
||||||
|
token has no node_id to attribute the sighting to, so it must 400 rather than
|
||||||
|
silently write an orphan doc), and org_id gets stamped from the node's own
|
||||||
|
Firestore doc so firestore.rules' docInMyOrg() can gate the frontend's read —
|
||||||
|
the same defensive-stamp pattern upload.py already uses for `calls`.
|
||||||
|
"""
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
from app.main import app
|
||||||
|
from app.internal.auth import require_node_service_or_firebase_token
|
||||||
|
from app.routers import telemetry
|
||||||
|
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
|
||||||
|
def _override(decoded: dict):
|
||||||
|
app.dependency_overrides[require_node_service_or_firebase_token] = lambda: decoded
|
||||||
|
|
||||||
|
|
||||||
|
def teardown_function():
|
||||||
|
app.dependency_overrides.pop(require_node_service_or_firebase_token, None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_service_token_without_node_id_is_rejected():
|
||||||
|
_override({"service": True})
|
||||||
|
resp = client.post("/telemetry/adsb", json={"aircraft": []})
|
||||||
|
assert resp.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
def test_node_upload_upserts_and_stamps_org_id():
|
||||||
|
_override({"node": True, "node_id": "node-1"})
|
||||||
|
with patch.object(telemetry.fstore, "doc_get_cached", AsyncMock(return_value={"org_id": "org-A"})), \
|
||||||
|
patch.object(telemetry.fstore, "doc_set", AsyncMock()) as mock_set:
|
||||||
|
resp = client.post("/telemetry/adsb", json={
|
||||||
|
"aircraft": [{"icao": "A1B2C3", "callsign": "UAL123", "lat": 41.1, "lon": -73.8}],
|
||||||
|
})
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert resp.json() == {"ok": True, "count": 1}
|
||||||
|
mock_set.assert_awaited_once()
|
||||||
|
(collection, doc_id, doc), kwargs = mock_set.await_args
|
||||||
|
assert collection == "aircraft"
|
||||||
|
assert doc_id == "A1B2C3"
|
||||||
|
assert doc["node_id"] == "node-1"
|
||||||
|
assert doc["org_id"] == "org-A"
|
||||||
|
assert kwargs.get("merge") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_node_upload_skips_entries_missing_icao():
|
||||||
|
_override({"node": True, "node_id": "node-1"})
|
||||||
|
with patch.object(telemetry.fstore, "doc_get_cached", AsyncMock(return_value=None)), \
|
||||||
|
patch.object(telemetry.fstore, "doc_set", AsyncMock()) as mock_set:
|
||||||
|
resp = client.post("/telemetry/adsb", json={"aircraft": [{"icao": ""}]})
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert resp.json() == {"ok": True, "count": 0}
|
||||||
|
mock_set.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
def test_ais_service_token_without_node_id_is_rejected():
|
||||||
|
_override({"service": True})
|
||||||
|
resp = client.post("/telemetry/ais", json={"vessels": []})
|
||||||
|
assert resp.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
def test_ais_node_upload_upserts_and_stamps_org_id():
|
||||||
|
_override({"node": True, "node_id": "node-1"})
|
||||||
|
with patch.object(telemetry.fstore, "doc_get_cached", AsyncMock(return_value={"org_id": "org-A"})), \
|
||||||
|
patch.object(telemetry.fstore, "doc_set", AsyncMock()) as mock_set:
|
||||||
|
resp = client.post("/telemetry/ais", json={
|
||||||
|
"vessels": [{"mmsi": "123456789", "name": "MV TEST", "lat": 41.0, "lon": -73.9}],
|
||||||
|
})
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert resp.json() == {"ok": True, "count": 1}
|
||||||
|
mock_set.assert_awaited_once()
|
||||||
|
(collection, doc_id, doc), kwargs = mock_set.await_args
|
||||||
|
assert collection == "vessels"
|
||||||
|
assert doc_id == "123456789"
|
||||||
|
assert doc["node_id"] == "node-1"
|
||||||
|
assert doc["org_id"] == "org-A"
|
||||||
|
assert kwargs.get("merge") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_ais_node_upload_skips_entries_missing_mmsi():
|
||||||
|
_override({"node": True, "node_id": "node-1"})
|
||||||
|
with patch.object(telemetry.fstore, "doc_get_cached", AsyncMock(return_value=None)), \
|
||||||
|
patch.object(telemetry.fstore, "doc_set", AsyncMock()) as mock_set:
|
||||||
|
resp = client.post("/telemetry/ais", json={"vessels": [{"mmsi": ""}]})
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert resp.json() == {"ok": True, "count": 0}
|
||||||
|
mock_set.assert_not_awaited()
|
||||||
@@ -211,6 +211,60 @@ async def test_model_failure_leaves_the_transcript_alone():
|
|||||||
assert await tc.correct("c1", "x y z w", SEGS, system_id="sys-1") == (None, None, False)
|
assert await tc.correct("c1", "x y z w", SEGS, system_id="sys-1") == (None, None, False)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Code-token guard (server-26#162) ────────────────────────────────────────
|
||||||
|
# Caught live: the same call came back with "10-7" rewritten to "10-13" in one
|
||||||
|
# place and "10-4" in another. A real code swapped for a different real code
|
||||||
|
# reads exactly as trustworthy as a correct one — worse than leaving the raw
|
||||||
|
# mishearing in place, since nothing downstream can tell it happened.
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_changed_ten_code_is_discarded():
|
||||||
|
payload = {"corrected": "10-13, we're back in town."}
|
||||||
|
with _system(), _gemini(payload):
|
||||||
|
text, _, _ = await tc.correct("c1", "10-7, we're back in town.", None, system_id="sys-1")
|
||||||
|
assert text is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_invented_code_token_is_discarded():
|
||||||
|
"""Nothing code-shaped in the original — the model added one from nothing."""
|
||||||
|
payload = {"corrected": "ShotSpotter, 10-4, group of 3 shooting outside."}
|
||||||
|
with _system(), _gemini(payload):
|
||||||
|
text, _, _ = await tc.correct("c1", "Seven, group of 3 shooting outside.", None, system_id="sys-1")
|
||||||
|
assert text is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_legitimate_place_correction_with_unchanged_codes_still_applies():
|
||||||
|
"""The guard must not collateral-damage a correction that never touches
|
||||||
|
a code token — Home/Forest for Holmes/4th-and-Rowe is exactly the kind of
|
||||||
|
fix this pass exists to make."""
|
||||||
|
payload = {"corrected": "10-13 coming over on Home Street and Forest Ave, 4-2."}
|
||||||
|
with _system(), _gemini(payload):
|
||||||
|
text, _, _ = await tc.correct(
|
||||||
|
"c1", "10-13 coming over on Holmes Street and 4th and Rowe, 4-2.",
|
||||||
|
None, system_id="sys-1",
|
||||||
|
)
|
||||||
|
assert text == "10-13 coming over on Home Street and Forest Ave, 4-2."
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_segment_code_change_discards_segments_only():
|
||||||
|
"""A code change in one segment discards the whole segments array (same
|
||||||
|
all-or-nothing rule as a length mismatch), but the independently-checked
|
||||||
|
joined correction still stands if it kept its own codes intact. The
|
||||||
|
joined `text`/`corrected` pair here is deliberately code-free — this test
|
||||||
|
isolates the segment-level guard, not the joined-text one."""
|
||||||
|
payload = {
|
||||||
|
"corrected": "Show it out to Ossining, back to Route 9.",
|
||||||
|
"segments": ["Headquarters, 10-13.", "Show it out to Ossining.", "360 north, back to Route 9."],
|
||||||
|
}
|
||||||
|
with _system(), _gemini(payload):
|
||||||
|
text, segs, _ = await tc.correct("c1", "x y z w", SEGS, system_id="sys-1", talkgroup_id=9048)
|
||||||
|
assert segs is None
|
||||||
|
assert text == "Show it out to Ossining, back to Route 9."
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_reference_data_reaches_the_prompt():
|
async def test_reference_data_reaches_the_prompt():
|
||||||
seen = {}
|
seen = {}
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
"""
|
||||||
|
server-26#<pending> — a transcript_too_short call (<=5 words: "10-8", "show me
|
||||||
|
clear", a unit check-in) never reached correlation at all. upload.py's
|
||||||
|
no-scenes fallback (the path that lets a no-transcript call still thin-link
|
||||||
|
by talkgroup) explicitly excluded ANY skip_reason, so short-but-real follow-up
|
||||||
|
and clearance traffic was permanently unlinkable — not just unextracted by
|
||||||
|
GPT, but never even attempted against the fast/thin path that already exists
|
||||||
|
for exactly this kind of content-free signal. garbage_transcript (Whisper
|
||||||
|
hallucination) has no real content behind it and should stay excluded.
|
||||||
|
"""
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.routers import upload
|
||||||
|
|
||||||
|
ALL_ON = {
|
||||||
|
"stt_enabled": True,
|
||||||
|
"correlation_enabled": True,
|
||||||
|
"summaries_enabled": True,
|
||||||
|
"vocabulary_learning_enabled": True,
|
||||||
|
"transcript_correction_enabled": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_ingest(skip_reason):
|
||||||
|
with patch("app.internal.feature_flags.get_flags",
|
||||||
|
AsyncMock(return_value=ALL_ON)), \
|
||||||
|
patch("app.internal.firestore.doc_get_cached",
|
||||||
|
AsyncMock(return_value={"system_id": "sys-1", "ai_flags": {}})), \
|
||||||
|
patch.object(upload, "fstore") as fs, \
|
||||||
|
patch.object(upload, "_correlate_with_consensus", AsyncMock(return_value=None)) as corr, \
|
||||||
|
patch("app.internal.transcription.transcribe_call",
|
||||||
|
AsyncMock(return_value=("10-8", []))), \
|
||||||
|
patch("app.internal.intelligence.extract_scenes", AsyncMock(return_value=[])), \
|
||||||
|
patch("app.internal.alerter.check_and_dispatch", AsyncMock()):
|
||||||
|
fs.doc_get = AsyncMock(return_value={"skip_reason": skip_reason} if skip_reason else {})
|
||||||
|
fs.doc_set = AsyncMock()
|
||||||
|
await upload._run_intelligence_pipeline(
|
||||||
|
call_id="call-1", node_id="node-1", system_id="sys-1",
|
||||||
|
talkgroup_id=101, talkgroup_name="PD Dispatch",
|
||||||
|
gcs_uri="gs://bucket/call-1.mp3",
|
||||||
|
)
|
||||||
|
return corr
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_transcript_too_short_now_attempts_correlation():
|
||||||
|
corr = await _run_ingest("transcript_too_short")
|
||||||
|
corr.assert_awaited_once()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_garbage_transcript_still_skips_correlation():
|
||||||
|
corr = await _run_ingest("garbage_transcript")
|
||||||
|
corr.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_no_skip_reason_still_attempts_correlation():
|
||||||
|
corr = await _run_ingest(None)
|
||||||
|
corr.assert_awaited_once()
|
||||||
@@ -4,7 +4,7 @@ import { useAuth } from "@/components/AuthProvider";
|
|||||||
import { c2api } from "@/lib/c2api";
|
import { c2api } from "@/lib/c2api";
|
||||||
import { useEffect, useState, useRef, useCallback } from "react";
|
import { useEffect, useState, useRef, useCallback } from "react";
|
||||||
import { useRouter } from "next/navigation";
|
import { useRouter } from "next/navigation";
|
||||||
import type { UserRecord, AuditEntry, UserRole } from "@/lib/types";
|
import type { UserRecord, AuditEntry, UserRole, CallRecord } from "@/lib/types";
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Shared primitives
|
// Shared primitives
|
||||||
@@ -1047,16 +1047,193 @@ function StaleCallsTab() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// STT eval (server-26#163) — the eval harness for real transcription
|
||||||
|
// accuracy. Separate from patchTranscript's "fix this call" flow: this never
|
||||||
|
// re-runs extraction or touches an incident, it only records what was
|
||||||
|
// actually said next to what Whisper heard, so eval-stats can report a real
|
||||||
|
// WER instead of a guess. Built to be worked in short sessions, a handful of
|
||||||
|
// calls at a time, over however many sittings it takes — not a one-shot form.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
function fmtPct(x: number | null | undefined): string {
|
||||||
|
return x === null || x === undefined ? "—" : `${(x * 100).toFixed(1)}%`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function EvalStatsBar({ stats }: { stats: { eval_count: number; raw_wer: number | null; corrected_wer: number | null } | null }) {
|
||||||
|
return (
|
||||||
|
<div className="bg-gray-900 border border-gray-800 rounded-xl p-4 flex flex-wrap gap-x-8 gap-y-2">
|
||||||
|
<div>
|
||||||
|
<p className="text-xs text-gray-500 font-mono">Calls verified</p>
|
||||||
|
<p className="text-white text-lg font-mono">{stats?.eval_count ?? "—"}</p>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<p className="text-xs text-gray-500 font-mono">Raw WER (whisper-1)</p>
|
||||||
|
<p className="text-white text-lg font-mono">{fmtPct(stats?.raw_wer)}</p>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<p className="text-xs text-gray-500 font-mono">Corrected WER (shipped)</p>
|
||||||
|
<p className="text-white text-lg font-mono">{fmtPct(stats?.corrected_wer)}</p>
|
||||||
|
</div>
|
||||||
|
{stats && stats.eval_count > 0 && stats.eval_count < 20 && (
|
||||||
|
<p className="text-xs text-amber-400 font-mono self-end">
|
||||||
|
fewer than 20 calls — numbers will move a lot until this grows
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function SttEvalTab() {
|
||||||
|
const [stats, setStats] = useState<{ eval_count: number; raw_wer: number | null; corrected_wer: number | null } | null>(null);
|
||||||
|
const [queue, setQueue] = useState<CallRecord[]>([]);
|
||||||
|
const [cursor, setCursor] = useState<string | null>(null);
|
||||||
|
const [exhausted, setExhausted] = useState(false);
|
||||||
|
const [draft, setDraft] = useState("");
|
||||||
|
const [loadingBatch, setLoadingBatch] = useState(false);
|
||||||
|
const [saving, setSaving] = useState(false);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const fetching = useRef(false);
|
||||||
|
|
||||||
|
const current = queue[0] ?? null;
|
||||||
|
|
||||||
|
const refreshStats = useCallback(() => {
|
||||||
|
c2api.getEvalStats().then(setStats).catch(() => { /* stats are a nice-to-have, not load-bearing */ });
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const loadBatch = useCallback(async () => {
|
||||||
|
if (fetching.current) return;
|
||||||
|
fetching.current = true;
|
||||||
|
setLoadingBatch(true);
|
||||||
|
setError(null);
|
||||||
|
try {
|
||||||
|
const res = await c2api.getEvalQueue(5, cursor);
|
||||||
|
setQueue((q) => [...q, ...res.calls]);
|
||||||
|
setCursor(res.next_cursor);
|
||||||
|
if (res.calls.length === 0 && !res.next_cursor) setExhausted(true);
|
||||||
|
} catch (e) {
|
||||||
|
setError(String(e));
|
||||||
|
} finally {
|
||||||
|
setLoadingBatch(false);
|
||||||
|
fetching.current = false;
|
||||||
|
}
|
||||||
|
}, [cursor]);
|
||||||
|
|
||||||
|
useEffect(() => { refreshStats(); }, [refreshStats]);
|
||||||
|
|
||||||
|
// Auto-refill: whenever the local queue runs dry and there's more to scan
|
||||||
|
// (or we haven't checked yet), pull another batch. Covers the sparse-window
|
||||||
|
// case too — a page with matches:0 but a next_cursor just means "keep
|
||||||
|
// scanning", not "done", so this fires again on its own.
|
||||||
|
useEffect(() => {
|
||||||
|
if (queue.length === 0 && !exhausted) loadBatch();
|
||||||
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
|
}, [queue.length, exhausted]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
setDraft(current ? (current.transcript_corrected || current.transcript || "") : "");
|
||||||
|
}, [current]);
|
||||||
|
|
||||||
|
async function saveAndNext() {
|
||||||
|
if (!current) return;
|
||||||
|
setSaving(true);
|
||||||
|
setError(null);
|
||||||
|
try {
|
||||||
|
await c2api.putEvalTranscript(current.call_id, draft);
|
||||||
|
setQueue((q) => q.slice(1));
|
||||||
|
refreshStats();
|
||||||
|
} catch (e) {
|
||||||
|
setError(String(e));
|
||||||
|
} finally {
|
||||||
|
setSaving(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function skip() {
|
||||||
|
setQueue((q) => q.slice(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="space-y-4">
|
||||||
|
<p className="text-xs text-gray-500 font-mono">
|
||||||
|
Listen to the audio, correct the transcript below until it matches what was actually said, then save.
|
||||||
|
This never touches the call's real transcript or re-runs anything — it only records ground truth
|
||||||
|
for measuring the pipeline. Do as many or as few as you have time for; it picks up where you left off.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<EvalStatsBar stats={stats} />
|
||||||
|
|
||||||
|
{error && (
|
||||||
|
<div className="bg-red-950 border border-red-800 rounded-lg p-3">
|
||||||
|
<p className="text-red-400 text-sm font-mono">{error}</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{current ? (
|
||||||
|
<div className="bg-gray-900 border border-gray-800 rounded-xl p-4 space-y-3">
|
||||||
|
<div className="flex flex-wrap items-center gap-x-3 gap-y-1 text-xs font-mono text-gray-400">
|
||||||
|
<span>{new Date(current.started_at).toLocaleString()}</span>
|
||||||
|
<span>{current.talkgroup_name || (current.talkgroup_id ? `TGID ${current.talkgroup_id}` : "unknown talkgroup")}</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{current.audio_url ? (
|
||||||
|
// eslint-disable-next-line jsx-a11y/media-has-caption
|
||||||
|
<audio controls src={current.audio_url} className="w-full h-9" />
|
||||||
|
) : (
|
||||||
|
<p className="text-xs text-gray-500 italic">No audio on this call — skip it.</p>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label className="text-xs text-gray-400 block mb-1">
|
||||||
|
Machine transcript (pre-filled) — correct it into what was actually said
|
||||||
|
</label>
|
||||||
|
<textarea
|
||||||
|
value={draft}
|
||||||
|
onChange={(e) => setDraft(e.target.value)}
|
||||||
|
rows={4}
|
||||||
|
className="w-full bg-gray-800 border border-gray-700 rounded-lg px-3 py-2 text-white text-sm font-mono focus:outline-none focus:border-indigo-500"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex gap-2">
|
||||||
|
<button
|
||||||
|
onClick={saveAndNext}
|
||||||
|
disabled={saving || !draft.trim()}
|
||||||
|
className="bg-indigo-600 hover:bg-indigo-500 disabled:opacity-50 text-white text-sm font-mono px-4 py-1.5 rounded-lg transition-colors"
|
||||||
|
>
|
||||||
|
{saving ? "Saving…" : "Save & next"}
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
onClick={skip}
|
||||||
|
disabled={saving}
|
||||||
|
className="bg-gray-800 hover:bg-gray-700 disabled:opacity-50 border border-gray-700 text-white text-sm font-mono px-4 py-1.5 rounded-lg transition-colors"
|
||||||
|
>
|
||||||
|
Skip
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="bg-gray-900 border border-gray-800 rounded-xl p-4">
|
||||||
|
<p className="text-sm font-mono text-gray-400">
|
||||||
|
{loadingBatch ? "Loading calls…" : exhausted ? "Nothing left to verify right now — check back after more calls come in." : "Loading…"}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Main admin page
|
// Main admin page
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
type AdminTab = "features" | "correlation" | "users" | "audit" | "calls";
|
type AdminTab = "features" | "correlation" | "users" | "audit" | "calls" | "eval";
|
||||||
|
|
||||||
const TAB_LABELS: { key: AdminTab; label: string }[] = [
|
const TAB_LABELS: { key: AdminTab; label: string }[] = [
|
||||||
{ key: "features", label: "AI Features" },
|
{ key: "features", label: "AI Features" },
|
||||||
{ key: "correlation", label: "Correlation Debug" },
|
{ key: "correlation", label: "Correlation Debug" },
|
||||||
{ key: "calls", label: "Calls" },
|
{ key: "calls", label: "Calls" },
|
||||||
|
{ key: "eval", label: "STT Eval" },
|
||||||
{ key: "users", label: "Users" },
|
{ key: "users", label: "Users" },
|
||||||
{ key: "audit", label: "Audit Log" },
|
{ key: "audit", label: "Audit Log" },
|
||||||
];
|
];
|
||||||
@@ -1102,6 +1279,7 @@ export default function AdminPage() {
|
|||||||
{tab === "features" && <FeaturesTab />}
|
{tab === "features" && <FeaturesTab />}
|
||||||
{tab === "correlation" && <CorrelationDebugTab />}
|
{tab === "correlation" && <CorrelationDebugTab />}
|
||||||
{tab === "calls" && <StaleCallsTab />}
|
{tab === "calls" && <StaleCallsTab />}
|
||||||
|
{tab === "eval" && <SttEvalTab />}
|
||||||
{tab === "users" && <UsersTab currentUid={user?.uid ?? ""} />}
|
{tab === "users" && <UsersTab currentUid={user?.uid ?? ""} />}
|
||||||
{tab === "audit" && <AuditLogTab />}
|
{tab === "audit" && <AuditLogTab />}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
import { useState } from "react";
|
import { useEffect, useState } from "react";
|
||||||
import { useAuth } from "@/components/AuthProvider";
|
import { useAuth } from "@/components/AuthProvider";
|
||||||
import { useAlerts } from "@/lib/useAlerts";
|
import { useAlerts } from "@/lib/useAlerts";
|
||||||
import { MachineOutputNotice } from "@/components/ui/MachineOutputNotice";
|
import { MachineOutputNotice } from "@/components/ui/MachineOutputNotice";
|
||||||
@@ -32,8 +32,8 @@ function RulesTab({ isAdmin }: { isAdmin: boolean }) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load on first render of this tab
|
// Load once when this tab mounts (load() self-guards on `loaded`).
|
||||||
if (!loaded) { load(); }
|
useEffect(() => { load(); }, []);
|
||||||
|
|
||||||
async function handleCreate(e: React.FormEvent) {
|
async function handleCreate(e: React.FormEvent) {
|
||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
|
|||||||
@@ -6,8 +6,9 @@
|
|||||||
// never correlated was invisible. That is the wrong way round when correlation
|
// never correlated was invisible. That is the wrong way round when correlation
|
||||||
// quality is the thing under development — the orphans are the evidence.
|
// quality is the thing under development — the orphans are the evidence.
|
||||||
//
|
//
|
||||||
// Admin-only, because it exposes every call in the org regardless of node
|
// Readable by every org member — the Firestore rules already let any member
|
||||||
// ownership and carries the manual attribution controls.
|
// read every call in their org. The manual attribution controls stay
|
||||||
|
// admin-only, matching the admin gate on the link/unlink routes.
|
||||||
|
|
||||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||||
import { useRouter } from "next/navigation";
|
import { useRouter } from "next/navigation";
|
||||||
@@ -23,6 +24,7 @@ import { Button } from "@/components/ui/Button";
|
|||||||
import { EmptyState, ErrorBanner } from "@/components/ui/EmptyState";
|
import { EmptyState, ErrorBanner } from "@/components/ui/EmptyState";
|
||||||
import { SkeletonCard } from "@/components/ui/Skeleton";
|
import { SkeletonCard } from "@/components/ui/Skeleton";
|
||||||
import { MachineOutputNotice } from "@/components/ui/MachineOutputNotice";
|
import { MachineOutputNotice } from "@/components/ui/MachineOutputNotice";
|
||||||
|
import { DateRange, dayStart, dayEnd } from "@/components/ui/DateRange";
|
||||||
|
|
||||||
type LinkFilter = "any" | "orphan" | "linked";
|
type LinkFilter = "any" | "orphan" | "linked";
|
||||||
type TranscriptFilter = "any" | "yes" | "no";
|
type TranscriptFilter = "any" | "yes" | "no";
|
||||||
@@ -68,11 +70,13 @@ function ArchiveRow({
|
|||||||
call,
|
call,
|
||||||
systemName,
|
systemName,
|
||||||
incidents,
|
incidents,
|
||||||
|
canEdit,
|
||||||
onChanged,
|
onChanged,
|
||||||
}: {
|
}: {
|
||||||
call: CallRecord;
|
call: CallRecord;
|
||||||
systemName?: string;
|
systemName?: string;
|
||||||
incidents: IncidentRecord[];
|
incidents: IncidentRecord[];
|
||||||
|
canEdit: boolean;
|
||||||
onChanged: () => void;
|
onChanged: () => void;
|
||||||
}) {
|
}) {
|
||||||
const [open, setOpen] = useState(false);
|
const [open, setOpen] = useState(false);
|
||||||
@@ -178,18 +182,18 @@ function ArchiveRow({
|
|||||||
<div key={id} className="flex items-center gap-2 text-xs">
|
<div key={id} className="flex items-center gap-2 text-xs">
|
||||||
<span className="text-ink-muted">attached to</span>
|
<span className="text-ink-muted">attached to</span>
|
||||||
<span className="text-ink-2 truncate">{inc?.title ?? id.slice(0, 8)}</span>
|
<span className="text-ink-2 truncate">{inc?.title ?? id.slice(0, 8)}</span>
|
||||||
<button
|
{canEdit && <button
|
||||||
onClick={() => detach(id)}
|
onClick={() => detach(id)}
|
||||||
disabled={busy}
|
disabled={busy}
|
||||||
className="text-sev-major hover:underline disabled:opacity-50 shrink-0"
|
className="text-sev-major hover:underline disabled:opacity-50 shrink-0"
|
||||||
>
|
>
|
||||||
detach
|
detach
|
||||||
</button>
|
</button>}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
})}
|
})}
|
||||||
|
|
||||||
<div className="flex flex-wrap items-center gap-2">
|
{canEdit && <div className="flex flex-wrap items-center gap-2">
|
||||||
<select
|
<select
|
||||||
value={attachTo}
|
value={attachTo}
|
||||||
onChange={(e) => setAttachTo(e.target.value)}
|
onChange={(e) => setAttachTo(e.target.value)}
|
||||||
@@ -208,7 +212,7 @@ function ArchiveRow({
|
|||||||
<Button size="sm" variant="secondary" onClick={attach} disabled={!attachTo || busy}>
|
<Button size="sm" variant="secondary" onClick={attach} disabled={!attachTo || busy}>
|
||||||
{busy ? "Saving…" : "Attach"}
|
{busy ? "Saving…" : "Attach"}
|
||||||
</Button>
|
</Button>
|
||||||
</div>
|
</div>}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{error && <ErrorBanner message={error} />}
|
{error && <ErrorBanner message={error} />}
|
||||||
@@ -219,8 +223,9 @@ function ArchiveRow({
|
|||||||
}
|
}
|
||||||
|
|
||||||
export default function ArchivePage() {
|
export default function ArchivePage() {
|
||||||
const { isAdmin, loading: authLoading } = useAuth();
|
const { user, orgId, isAdmin, loading: authLoading } = useAuth();
|
||||||
const router = useRouter();
|
const router = useRouter();
|
||||||
|
const canView = Boolean(user && (orgId || isAdmin));
|
||||||
const { systems } = useSystems();
|
const { systems } = useSystems();
|
||||||
const { incidents } = useIncidents(200);
|
const { incidents } = useIncidents(200);
|
||||||
|
|
||||||
@@ -235,10 +240,12 @@ export default function ArchivePage() {
|
|||||||
const [systemId, setSystemId] = useState("");
|
const [systemId, setSystemId] = useState("");
|
||||||
const [q, setQ] = useState("");
|
const [q, setQ] = useState("");
|
||||||
const [submittedQ, setSubmittedQ] = useState("");
|
const [submittedQ, setSubmittedQ] = useState("");
|
||||||
|
const [dateFrom, setDateFrom] = useState("");
|
||||||
|
const [dateTo, setDateTo] = useState("");
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!authLoading && !isAdmin) router.replace("/");
|
if (!authLoading && !canView) router.replace("/");
|
||||||
}, [authLoading, isAdmin, router]);
|
}, [authLoading, canView, router]);
|
||||||
|
|
||||||
const load = useCallback(
|
const load = useCallback(
|
||||||
async (nextCursor: string | null, append: boolean) => {
|
async (nextCursor: string | null, append: boolean) => {
|
||||||
@@ -252,6 +259,8 @@ export default function ArchivePage() {
|
|||||||
transcript,
|
transcript,
|
||||||
system_id: systemId || undefined,
|
system_id: systemId || undefined,
|
||||||
q: submittedQ || undefined,
|
q: submittedQ || undefined,
|
||||||
|
date_from: dayStart(dateFrom)?.toISOString(),
|
||||||
|
date_to: dayEnd(dateTo)?.toISOString(),
|
||||||
});
|
});
|
||||||
setCalls((prev) => (append ? [...prev, ...res.calls] : res.calls));
|
setCalls((prev) => (append ? [...prev, ...res.calls] : res.calls));
|
||||||
setCursor(res.next_cursor);
|
setCursor(res.next_cursor);
|
||||||
@@ -262,14 +271,14 @@ export default function ArchivePage() {
|
|||||||
setLoading(false);
|
setLoading(false);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
[link, transcript, systemId, submittedQ],
|
[link, transcript, systemId, submittedQ, dateFrom, dateTo],
|
||||||
);
|
);
|
||||||
|
|
||||||
// Reload from the top whenever a filter changes.
|
// Reload from the top whenever a filter changes.
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (authLoading || !isAdmin) return;
|
if (authLoading || !canView) return;
|
||||||
load(null, false);
|
load(null, false);
|
||||||
}, [authLoading, isAdmin, load]);
|
}, [authLoading, canView, load]);
|
||||||
|
|
||||||
const systemName = useMemo(() => {
|
const systemName = useMemo(() => {
|
||||||
const m = new Map(systems.map((s) => [s.system_id, s.name]));
|
const m = new Map(systems.map((s) => [s.system_id, s.name]));
|
||||||
@@ -277,7 +286,7 @@ export default function ArchivePage() {
|
|||||||
}, [systems]);
|
}, [systems]);
|
||||||
|
|
||||||
// Every hook runs before this guard — see the note in app/nodes/page.tsx.
|
// Every hook runs before this guard — see the note in app/nodes/page.tsx.
|
||||||
if (authLoading || !isAdmin) return null;
|
if (authLoading || !canView) return null;
|
||||||
|
|
||||||
const orphanCount = calls.filter((c) => callIncidentIds(c).length === 0).length;
|
const orphanCount = calls.filter((c) => callIncidentIds(c).length === 0).length;
|
||||||
const noTranscript = calls.filter((c) => !(c.transcript_corrected || c.transcript)).length;
|
const noTranscript = calls.filter((c) => !(c.transcript_corrected || c.transcript)).length;
|
||||||
@@ -286,7 +295,9 @@ export default function ArchivePage() {
|
|||||||
<div className="space-y-6">
|
<div className="space-y-6">
|
||||||
<PageHeader
|
<PageHeader
|
||||||
title="Archive"
|
title="Archive"
|
||||||
description="Every call on the account, correlated or not. Attach an orphan to the incident it belongs to, or detach one the correlator got wrong."
|
description={isAdmin
|
||||||
|
? "Every call on the account, correlated or not. Attach an orphan to the incident it belongs to, or detach one the correlator got wrong."
|
||||||
|
: "Every call on the account, correlated or not."}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<div className="flex flex-wrap items-center gap-3">
|
<div className="flex flex-wrap items-center gap-3">
|
||||||
@@ -329,6 +340,8 @@ export default function ArchivePage() {
|
|||||||
))}
|
))}
|
||||||
</select>
|
</select>
|
||||||
|
|
||||||
|
<DateRange from={dateFrom} to={dateTo} onChange={(f, t) => { setDateFrom(f); setDateTo(t); }} />
|
||||||
|
|
||||||
<form
|
<form
|
||||||
onSubmit={(e) => { e.preventDefault(); setSubmittedQ(q.trim()); }}
|
onSubmit={(e) => { e.preventDefault(); setSubmittedQ(q.trim()); }}
|
||||||
className="flex items-center gap-2 ml-auto"
|
className="flex items-center gap-2 ml-auto"
|
||||||
@@ -373,6 +386,7 @@ export default function ArchivePage() {
|
|||||||
call={call}
|
call={call}
|
||||||
systemName={systemName(call.system_id)}
|
systemName={systemName(call.system_id)}
|
||||||
incidents={incidents}
|
incidents={incidents}
|
||||||
|
canEdit={isAdmin}
|
||||||
onChanged={() => load(null, false)}
|
onChanged={() => load(null, false)}
|
||||||
/>
|
/>
|
||||||
))}
|
))}
|
||||||
|
|||||||
@@ -100,6 +100,7 @@ export default function IncidentDetailPage() {
|
|||||||
const displayTags = incident.tags.filter((t) => t !== "auto-generated");
|
const displayTags = incident.tags.filter((t) => t !== "auto-generated");
|
||||||
const unitsActive = incident.units_active ?? incident.units ?? [];
|
const unitsActive = incident.units_active ?? incident.units ?? [];
|
||||||
const unitsCleared = incident.units_cleared ?? [];
|
const unitsCleared = incident.units_cleared ?? [];
|
||||||
|
const vehicles = incident.vehicles ?? [];
|
||||||
const active = incident.status === "active";
|
const active = incident.status === "active";
|
||||||
|
|
||||||
const visible = newestFirst.slice(0, earlierShown);
|
const visible = newestFirst.slice(0, earlierShown);
|
||||||
@@ -213,11 +214,11 @@ export default function IncidentDetailPage() {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{incident.vehicles?.length > 0 && (
|
{vehicles.length > 0 && (
|
||||||
<div>
|
<div>
|
||||||
<p className="text-xs text-ink-muted uppercase tracking-wide mb-2">Vehicles</p>
|
<p className="text-xs text-ink-muted uppercase tracking-wide mb-2">Vehicles</p>
|
||||||
<div className="flex flex-wrap gap-1">
|
<div className="flex flex-wrap gap-1">
|
||||||
{incident.vehicles.map((v) => (
|
{vehicles.map((v) => (
|
||||||
<span key={v} className="text-xs bg-raised text-ink-2 px-2 py-0.5 rounded font-mono">{v}</span>
|
<span key={v} className="text-xs bg-raised text-ink-2 px-2 py-0.5 rounded font-mono">{v}</span>
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import { Badge } from "@/components/ui/Badge";
|
|||||||
import { EmptyState, ErrorBanner } from "@/components/ui/EmptyState";
|
import { EmptyState, ErrorBanner } from "@/components/ui/EmptyState";
|
||||||
import { SkeletonCard } from "@/components/ui/Skeleton";
|
import { SkeletonCard } from "@/components/ui/Skeleton";
|
||||||
import { MachineOutputNotice } from "@/components/ui/MachineOutputNotice";
|
import { MachineOutputNotice } from "@/components/ui/MachineOutputNotice";
|
||||||
|
import { DateRange, dayStart, dayEnd } from "@/components/ui/DateRange";
|
||||||
import { isKnownSeverity, severityRank } from "@/lib/severity";
|
import { isKnownSeverity, severityRank } from "@/lib/severity";
|
||||||
import { SeverityMark, SeveritySpine } from "@/components/marks/SeverityMark";
|
import { SeverityMark, SeveritySpine } from "@/components/marks/SeverityMark";
|
||||||
import { TypeGlyph } from "@/components/marks/TypeGlyph";
|
import { TypeGlyph } from "@/components/marks/TypeGlyph";
|
||||||
@@ -27,6 +28,23 @@ const SEVERITY_FILTERS: { key: SeverityFilter; label: string }[] = [
|
|||||||
const FILTER_THRESHOLD: Record<SeverityFilter, number> = { all: -1, minor: 1, moderate: 2, major: 3 };
|
const FILTER_THRESHOLD: Record<SeverityFilter, number> = { all: -1, minor: 1, moderate: 2, major: 3 };
|
||||||
|
|
||||||
type SortMode = "recent" | "severity";
|
type SortMode = "recent" | "severity";
|
||||||
|
type StatusFilter = "any" | "active" | "resolved";
|
||||||
|
|
||||||
|
const INCIDENT_TYPES = ["fire", "police", "ems", "accident", "other"];
|
||||||
|
|
||||||
|
// Firestore holds the paging; text/type/status filtering runs over the loaded
|
||||||
|
// window, so "Load more" also widens what the search can find.
|
||||||
|
const PAGE_SIZE = 100;
|
||||||
|
|
||||||
|
function matchesSearch(inc: IncidentRecord, needle: string): boolean {
|
||||||
|
if (!needle) return true;
|
||||||
|
const hay = [
|
||||||
|
inc.title, inc.location, inc.summary, inc.type,
|
||||||
|
...(inc.units ?? []), ...(inc.vehicles ?? []), ...(inc.tags ?? []),
|
||||||
|
...(inc.location_mentions ?? []),
|
||||||
|
].filter(Boolean).join(" ").toLowerCase();
|
||||||
|
return hay.includes(needle);
|
||||||
|
}
|
||||||
|
|
||||||
// The Firestore client surfaces a missing composite index or an undeployed
|
// The Firestore client surfaces a missing composite index or an undeployed
|
||||||
// ruleset as a raw multi-line string with a console URL in it — not something
|
// ruleset as a raw multi-line string with a console URL in it — not something
|
||||||
@@ -178,11 +196,19 @@ function CreateModal({ onClose, onCreate }: { onClose: () => void; onCreate: (bo
|
|||||||
|
|
||||||
export default function IncidentsPage() {
|
export default function IncidentsPage() {
|
||||||
const { isAdmin } = useAuth();
|
const { isAdmin } = useAuth();
|
||||||
const { incidents, loading, error } = useIncidents();
|
const [pageLimit, setPageLimit] = useState(PAGE_SIZE);
|
||||||
|
const [dateFrom, setDateFrom] = useState("");
|
||||||
|
const [dateTo, setDateTo] = useState("");
|
||||||
|
const rangeFrom = useMemo(() => dayStart(dateFrom), [dateFrom]);
|
||||||
|
const rangeTo = useMemo(() => dayEnd(dateTo), [dateTo]);
|
||||||
|
const { incidents, loading, error, hasMore } = useIncidents(pageLimit, rangeFrom, rangeTo);
|
||||||
const activeCalls = useActiveCalls();
|
const activeCalls = useActiveCalls();
|
||||||
const [showCreate, setShowCreate] = useState(false);
|
const [showCreate, setShowCreate] = useState(false);
|
||||||
const [severityFilter, setSeverityFilter] = useState<SeverityFilter>("all");
|
const [severityFilter, setSeverityFilter] = useState<SeverityFilter>("all");
|
||||||
const [sortMode, setSortMode] = useState<SortMode>("recent");
|
const [sortMode, setSortMode] = useState<SortMode>("recent");
|
||||||
|
const [statusFilter, setStatusFilter] = useState<StatusFilter>("any");
|
||||||
|
const [typeFilter, setTypeFilter] = useState("");
|
||||||
|
const [search, setSearch] = useState("");
|
||||||
|
|
||||||
const onAirIncidentIds = useMemo(() => {
|
const onAirIncidentIds = useMemo(() => {
|
||||||
const s = new Set<string>();
|
const s = new Set<string>();
|
||||||
@@ -194,12 +220,24 @@ export default function IncidentsPage() {
|
|||||||
|
|
||||||
const filtered = useMemo(() => {
|
const filtered = useMemo(() => {
|
||||||
const threshold = FILTER_THRESHOLD[severityFilter];
|
const threshold = FILTER_THRESHOLD[severityFilter];
|
||||||
const list = incidents.filter((i) => severityRank(i.severity) >= threshold);
|
const needle = search.trim().toLowerCase();
|
||||||
|
const list = incidents.filter((i) =>
|
||||||
|
severityRank(i.severity) >= threshold &&
|
||||||
|
(statusFilter === "any" || i.status === statusFilter) &&
|
||||||
|
(!typeFilter || i.type === typeFilter) &&
|
||||||
|
matchesSearch(i, needle)
|
||||||
|
);
|
||||||
if (sortMode === "severity") {
|
if (sortMode === "severity") {
|
||||||
return [...list].sort((a, b) => severityRank(b.severity) - severityRank(a.severity) || b.started_at.localeCompare(a.started_at));
|
return [...list].sort((a, b) => severityRank(b.severity) - severityRank(a.severity) || b.started_at.localeCompare(a.started_at));
|
||||||
}
|
}
|
||||||
return list; // useIncidents() already orders by started_at desc
|
return list; // useIncidents() already orders by started_at desc
|
||||||
}, [incidents, severityFilter, sortMode]);
|
}, [incidents, severityFilter, sortMode, statusFilter, typeFilter, search]);
|
||||||
|
|
||||||
|
const filtersActive = severityFilter !== "all" || statusFilter !== "any" || typeFilter !== "" || search.trim() !== "" || dateFrom !== "" || dateTo !== "";
|
||||||
|
function clearFilters() {
|
||||||
|
setSeverityFilter("all"); setStatusFilter("any"); setTypeFilter(""); setSearch("");
|
||||||
|
setDateFrom(""); setDateTo(""); setPageLimit(PAGE_SIZE);
|
||||||
|
}
|
||||||
|
|
||||||
const hiddenCount = incidents.length - filtered.length;
|
const hiddenCount = incidents.length - filtered.length;
|
||||||
const activeCount = filtered.filter((i) => i.status === "active").length;
|
const activeCount = filtered.filter((i) => i.status === "active").length;
|
||||||
@@ -249,7 +287,39 @@ export default function IncidentsPage() {
|
|||||||
</button>
|
</button>
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
<label className="flex items-center gap-2 text-xs text-ink-muted">
|
<input
|
||||||
|
type="search"
|
||||||
|
value={search}
|
||||||
|
onChange={(e) => setSearch(e.target.value)}
|
||||||
|
placeholder="Search title, location, units…"
|
||||||
|
className="bg-surface border border-line rounded-lg text-sm text-ink px-3 py-2 w-full sm:w-64 focus:outline-none focus:border-accent"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex flex-wrap items-center gap-3">
|
||||||
|
<select
|
||||||
|
value={statusFilter}
|
||||||
|
onChange={(e) => setStatusFilter(e.target.value as StatusFilter)}
|
||||||
|
className="bg-surface border border-line rounded-lg px-2 py-1.5 text-sm text-ink-2 focus:outline-none focus:border-accent"
|
||||||
|
>
|
||||||
|
<option value="any">Any status</option>
|
||||||
|
<option value="active">Active</option>
|
||||||
|
<option value="resolved">Resolved</option>
|
||||||
|
</select>
|
||||||
|
<select
|
||||||
|
value={typeFilter}
|
||||||
|
onChange={(e) => setTypeFilter(e.target.value)}
|
||||||
|
className="bg-surface border border-line rounded-lg px-2 py-1.5 text-sm text-ink-2 focus:outline-none focus:border-accent"
|
||||||
|
>
|
||||||
|
<option value="">All types</option>
|
||||||
|
{INCIDENT_TYPES.map((t) => <option key={t} value={t}>{t}</option>)}
|
||||||
|
</select>
|
||||||
|
<DateRange
|
||||||
|
from={dateFrom}
|
||||||
|
to={dateTo}
|
||||||
|
onChange={(f, t) => { setDateFrom(f); setDateTo(t); setPageLimit(PAGE_SIZE); }}
|
||||||
|
/>
|
||||||
|
<label className="flex items-center gap-2 text-xs text-ink-muted ml-auto">
|
||||||
Sort
|
Sort
|
||||||
<select
|
<select
|
||||||
value={sortMode}
|
value={sortMode}
|
||||||
@@ -270,7 +340,8 @@ export default function IncidentsPage() {
|
|||||||
<>
|
<>
|
||||||
{hiddenCount > 0 && (
|
{hiddenCount > 0 && (
|
||||||
<p className="text-xs text-ink-muted">
|
<p className="text-xs text-ink-muted">
|
||||||
{hiddenCount} incident{hiddenCount !== 1 ? "s" : ""} hidden by the severity filter.
|
{hiddenCount} of {incidents.length} loaded incident{incidents.length !== 1 ? "s" : ""} hidden by filters
|
||||||
|
{hasMore && " — load more to search further back"}.
|
||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
@@ -302,19 +373,27 @@ export default function IncidentsPage() {
|
|||||||
|
|
||||||
{filtered.length === 0 && !error && (
|
{filtered.length === 0 && !error && (
|
||||||
<EmptyState
|
<EmptyState
|
||||||
title={incidents.length === 0 ? "No incidents recorded yet" : "No incidents match this filter"}
|
title={incidents.length === 0 && !filtersActive ? "No incidents recorded yet" : "No incidents match these filters"}
|
||||||
description={
|
description={
|
||||||
incidents.length === 0
|
incidents.length === 0 && !filtersActive
|
||||||
? "Incidents appear automatically once calls start correlating."
|
? "Incidents appear automatically once calls start correlating."
|
||||||
: "Try a lower severity threshold."
|
: "Try clearing a filter, or load older incidents."
|
||||||
}
|
}
|
||||||
action={
|
action={
|
||||||
incidents.length > 0 && severityFilter !== "all" ? (
|
filtersActive ? (
|
||||||
<Button variant="secondary" size="sm" onClick={() => setSeverityFilter("all")}>Clear filter</Button>
|
<Button variant="secondary" size="sm" onClick={clearFilters}>Clear filters</Button>
|
||||||
) : undefined
|
) : undefined
|
||||||
}
|
}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{hasMore && (
|
||||||
|
<div className="flex justify-center">
|
||||||
|
<Button variant="secondary" onClick={() => setPageLimit((n) => n + PAGE_SIZE)}>
|
||||||
|
Load more
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
|||||||
@@ -60,7 +60,7 @@ function DiscordJoinModal({
|
|||||||
<div className="fixed inset-0 bg-black/60 flex items-center justify-center z-50 p-4">
|
<div className="fixed inset-0 bg-black/60 flex items-center justify-center z-50 p-4">
|
||||||
<form
|
<form
|
||||||
onSubmit={handleSubmit}
|
onSubmit={handleSubmit}
|
||||||
className="bg-gray-900 border border-gray-700 rounded-xl p-6 space-y-4 font-mono w-full max-w-sm"
|
className="bg-gray-900 border border-gray-700 rounded-xl p-6 space-y-4 font-mono w-full max-w-sm max-h-[90vh] overflow-y-auto"
|
||||||
>
|
>
|
||||||
<h3 className="text-white font-semibold">Join Discord Voice</h3>
|
<h3 className="text-white font-semibold">Join Discord Voice</h3>
|
||||||
<div>
|
<div>
|
||||||
@@ -120,7 +120,10 @@ export default function NodeDetailPage() {
|
|||||||
const [approving, setApproving] = useState(false);
|
const [approving, setApproving] = useState(false);
|
||||||
const [deleting, setDeleting] = useState(false);
|
const [deleting, setDeleting] = useState(false);
|
||||||
const { systems } = useSystems();
|
const { systems } = useSystems();
|
||||||
const { calls } = useCalls(20);
|
// TODO(server-26#109 item5): server-side node_id filter. A where("node_id","==",id)
|
||||||
|
// alongside the existing org_id equality + started_at orderBy needs a brand-new
|
||||||
|
// composite index, so for now pull a wider window and filter client-side.
|
||||||
|
const { calls } = useCalls(200);
|
||||||
const { isAdmin } = useAuth();
|
const { isAdmin } = useAuth();
|
||||||
|
|
||||||
const systemMap = Object.fromEntries(systems.map((s) => [s.system_id, s]));
|
const systemMap = Object.fromEntries(systems.map((s) => [s.system_id, s]));
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ export default function NodesPage() {
|
|||||||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
|
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
|
||||||
{pending.map((n) => (
|
{pending.map((n) => (
|
||||||
<div key={n.node_id} onClick={() => setConfigNode(n)} className="cursor-pointer">
|
<div key={n.node_id} onClick={() => setConfigNode(n)} className="cursor-pointer">
|
||||||
<NodeCard node={n} system={systemMap[n.assigned_system_id ?? ""]} />
|
<NodeCard node={n} system={systemMap[n.assigned_system_id ?? ""]} linkToDetail={false} />
|
||||||
</div>
|
</div>
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -22,7 +22,9 @@ function TripCard({ trip, isAdmin, onDelete }: {
|
|||||||
}) {
|
}) {
|
||||||
const router = useRouter();
|
const router = useRouter();
|
||||||
const today = new Date().toISOString().slice(0, 10);
|
const today = new Date().toISOString().slice(0, 10);
|
||||||
const upcoming = trip.start_date >= today;
|
// Bucket and badge must agree: the list groups on end_date (page.tsx ~L176),
|
||||||
|
// so a trip isn't "Past" until it's over, not when it starts.
|
||||||
|
const upcoming = trip.end_date >= today;
|
||||||
const attendeeCount = Object.keys(trip.attendees ?? {}).length;
|
const attendeeCount = Object.keys(trip.attendees ?? {}).length;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
@@ -97,10 +99,10 @@ function CreateModal({ onClose, onCreate }: {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="fixed inset-0 bg-black/60 flex items-center justify-center z-50">
|
<div className="fixed inset-0 bg-black/60 flex items-center justify-center z-50 p-4">
|
||||||
<form
|
<form
|
||||||
onSubmit={handleSubmit}
|
onSubmit={handleSubmit}
|
||||||
className="bg-gray-900 border border-gray-700 rounded-xl p-6 w-full max-w-md space-y-4"
|
className="bg-gray-900 border border-gray-700 rounded-xl p-6 w-full max-w-md space-y-4 max-h-[90vh] overflow-y-auto"
|
||||||
>
|
>
|
||||||
<h2 className="text-white font-bold">New Trip</h2>
|
<h2 className="text-white font-bold">New Trip</h2>
|
||||||
|
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ function fmtClock(s: number): string {
|
|||||||
return `${m}:${r.toString().padStart(2, "0")}`;
|
return `${m}:${r.toString().padStart(2, "0")}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function InlinePlayer({ callId, hasAudio }: { callId: string; hasAudio: boolean }) {
|
function InlinePlayer({ callId }: { callId: string }) {
|
||||||
const [url, setUrl] = useState<string | null>(null);
|
const [url, setUrl] = useState<string | null>(null);
|
||||||
const [error, setError] = useState(false);
|
const [error, setError] = useState(false);
|
||||||
const [loading, setLoading] = useState(false);
|
const [loading, setLoading] = useState(false);
|
||||||
@@ -32,8 +32,6 @@ function InlinePlayer({ callId, hasAudio }: { callId: string; hasAudio: boolean
|
|||||||
const [duration, setDuration] = useState(0);
|
const [duration, setDuration] = useState(0);
|
||||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||||
|
|
||||||
if (!hasAudio) return null;
|
|
||||||
|
|
||||||
async function ensureUrl() {
|
async function ensureUrl() {
|
||||||
if (url || loading) return;
|
if (url || loading) return;
|
||||||
setLoading(true);
|
setLoading(true);
|
||||||
@@ -169,7 +167,7 @@ export function CallSpineEntry({
|
|||||||
|
|
||||||
{hasAudio && (
|
{hasAudio && (
|
||||||
<div className="mt-1.5">
|
<div className="mt-1.5">
|
||||||
<InlinePlayer callId={call.call_id} hasAudio={hasAudio} />
|
<InlinePlayer callId={call.call_id} />
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,8 @@ import L from "leaflet";
|
|||||||
import type { CallRecord, IncidentRecord, NodeRecord, NodeStatus } from "@/lib/types";
|
import type { CallRecord, IncidentRecord, NodeRecord, NodeStatus } from "@/lib/types";
|
||||||
import { isKnownSeverity, SEVERITY_COLORS, SEVERITY_LABEL, type Severity } from "@/lib/severity";
|
import { isKnownSeverity, SEVERITY_COLORS, SEVERITY_LABEL, type Severity } from "@/lib/severity";
|
||||||
import { MachineOutputNotice } from "@/components/ui/MachineOutputNotice";
|
import { MachineOutputNotice } from "@/components/ui/MachineOutputNotice";
|
||||||
|
import { useAircraft } from "@/lib/useAircraft";
|
||||||
|
import { useVessels } from "@/lib/useVessels";
|
||||||
|
|
||||||
// ── Leaflet icon fix ──────────────────────────────────────────────────────────
|
// ── Leaflet icon fix ──────────────────────────────────────────────────────────
|
||||||
delete (L.Icon.Default.prototype as unknown as Record<string, unknown>)._getIconUrl;
|
delete (L.Icon.Default.prototype as unknown as Record<string, unknown>)._getIconUrl;
|
||||||
@@ -25,15 +27,15 @@ L.Icon.Default.mergeOptions({
|
|||||||
});
|
});
|
||||||
|
|
||||||
// ── Basemap tiles ─────────────────────────────────────────────────────────────
|
// ── Basemap tiles ─────────────────────────────────────────────────────────────
|
||||||
// Default is CARTO's keyless dark raster basemap — no token, fits the dark UI.
|
// Prod sets NEXT_PUBLIC_MAP_TILE_URL to a keyed style (a CARTO account style,
|
||||||
// Overridable via NEXT_PUBLIC_MAP_TILE_URL so a keyed style (a CARTO account
|
// MapTiler, Mapbox, …). The in-code fallback is plain OpenStreetMap so the map
|
||||||
// style, MapTiler, Mapbox, …) can be dropped in for prod without a code change.
|
// still renders if that var is missing — CARTO's keyless CDN has proven flaky.
|
||||||
// Whatever is supplied must use Leaflet's {s}/{z}/{x}/{y}{r} placeholder scheme.
|
// Whatever is supplied must use Leaflet's {s}/{z}/{x}/{y}{r} placeholder scheme;
|
||||||
|
// the {z}/{x}/{y} tokens below are substituted by Leaflet at runtime.
|
||||||
const MAP_TILE_URL =
|
const MAP_TILE_URL =
|
||||||
process.env.NEXT_PUBLIC_MAP_TILE_URL ||
|
process.env.NEXT_PUBLIC_MAP_TILE_URL ||
|
||||||
"https://{s}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}{r}.png";
|
"https://tile.openstreetmap.org/{z}/{x}/{y}.png";
|
||||||
const MAP_TILE_ATTRIBUTION =
|
const MAP_TILE_ATTRIBUTION = "© OpenStreetMap contributors";
|
||||||
'© <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors © <a href="https://carto.com/">CARTO</a>';
|
|
||||||
|
|
||||||
// ── Colour ────────────────────────────────────────────────────────────────────
|
// ── Colour ────────────────────────────────────────────────────────────────────
|
||||||
// Severity is the only hue on this map — see UI_REDESIGN.md §2.3. Incident
|
// Severity is the only hue on this map — see UI_REDESIGN.md §2.3. Incident
|
||||||
@@ -90,6 +92,73 @@ function nodeIcon(status: NodeStatus): L.DivIcon {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Aircraft icon — node-26#9 second-SDR ADS-B overlay ────────────────────────
|
||||||
|
function aircraftIcon(trackDeg: number | null): L.DivIcon {
|
||||||
|
const size = 16;
|
||||||
|
const rotation = trackDeg ?? 0;
|
||||||
|
return L.divIcon({
|
||||||
|
className: "",
|
||||||
|
html: `<div style="width:${size}px;height:${size}px;transform:rotate(${rotation}deg)"><svg width="${size}" height="${size}" viewBox="0 0 24 24" fill="var(--accent)" stroke="var(--surface)" stroke-width="1"><path d="M12 2 L15 11 L22 15 L15 15.5 L14 21 L17 22.5 L12 21.5 L7 22.5 L10 21 L9 15.5 L2 15 L9 11 Z"/></svg></div>`,
|
||||||
|
iconSize: [size, size],
|
||||||
|
iconAnchor: [size / 2, size / 2],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function AircraftLayer() {
|
||||||
|
const { aircraft } = useAircraft();
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
{aircraft
|
||||||
|
.filter((a) => a.lat != null && a.lon != null)
|
||||||
|
.map((a) => (
|
||||||
|
<Marker key={a.icao} position={[a.lat as number, a.lon as number]} icon={aircraftIcon(a.track_deg)}>
|
||||||
|
<Popup minWidth={160}>
|
||||||
|
<div className="space-y-1">
|
||||||
|
<div className="font-semibold">{a.callsign || a.icao}</div>
|
||||||
|
<div className="text-xs text-ink-muted">ICAO {a.icao}</div>
|
||||||
|
{a.altitude_ft != null && <div className="text-xs">Altitude: {Math.round(a.altitude_ft)} ft</div>}
|
||||||
|
{a.ground_speed_kt != null && <div className="text-xs">Speed: {Math.round(a.ground_speed_kt)} kt</div>}
|
||||||
|
</div>
|
||||||
|
</Popup>
|
||||||
|
</Marker>
|
||||||
|
))}
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Vessel icon — node-26#9 second-SDR AIS overlay ─────────────────────────────
|
||||||
|
function vesselIcon(headingDeg: number | null): L.DivIcon {
|
||||||
|
const size = 14;
|
||||||
|
const rotation = headingDeg ?? 0;
|
||||||
|
return L.divIcon({
|
||||||
|
className: "",
|
||||||
|
html: `<div style="width:${size}px;height:${size}px;transform:rotate(${rotation}deg)"><svg width="${size}" height="${size}" viewBox="0 0 24 24" fill="var(--accent)" stroke="var(--surface)" stroke-width="1"><path d="M12 2 L18 14 L18 20 L6 20 L6 14 Z"/></svg></div>`,
|
||||||
|
iconSize: [size, size],
|
||||||
|
iconAnchor: [size / 2, size / 2],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function VesselLayer() {
|
||||||
|
const { vessels } = useVessels();
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
{vessels
|
||||||
|
.filter((v) => v.lat != null && v.lon != null)
|
||||||
|
.map((v) => (
|
||||||
|
<Marker key={v.mmsi} position={[v.lat as number, v.lon as number]} icon={vesselIcon(v.heading_deg)}>
|
||||||
|
<Popup minWidth={160}>
|
||||||
|
<div className="space-y-1">
|
||||||
|
<div className="font-semibold">{v.name || v.mmsi}</div>
|
||||||
|
<div className="text-xs text-ink-muted">MMSI {v.mmsi}</div>
|
||||||
|
{v.speed_kt != null && <div className="text-xs">Speed: {Math.round(v.speed_kt)} kt</div>}
|
||||||
|
</div>
|
||||||
|
</Popup>
|
||||||
|
</Marker>
|
||||||
|
))}
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function nodeFanIcon(members: NodeRecord[]): L.DivIcon {
|
function nodeFanIcon(members: NodeRecord[]): L.DivIcon {
|
||||||
const n = members.length;
|
const n = members.length;
|
||||||
const CARD = 13;
|
const CARD = 13;
|
||||||
@@ -459,9 +528,6 @@ export default function MapView({ nodes, activeCalls, incidents = [], calls = []
|
|||||||
const [drawerOpen, setDrawerOpen] = useState(false);
|
const [drawerOpen, setDrawerOpen] = useState(false);
|
||||||
const [agoClock, setAgoClock] = useState(0);
|
const [agoClock, setAgoClock] = useState(0);
|
||||||
const [radarEpoch, setRadarEpoch] = useState(() => Date.now());
|
const [radarEpoch, setRadarEpoch] = useState(() => Date.now());
|
||||||
const [clockStr, setClockStr] = useState(() =>
|
|
||||||
new Date().toLocaleTimeString([], { hour12: false, hour: "2-digit", minute: "2-digit", second: "2-digit" })
|
|
||||||
);
|
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const id = setInterval(() => setAgoClock((t: number) => t + 1), 10_000);
|
const id = setInterval(() => setAgoClock((t: number) => t + 1), 10_000);
|
||||||
@@ -474,15 +540,6 @@ export default function MapView({ nodes, activeCalls, incidents = [], calls = []
|
|||||||
return () => clearInterval(id);
|
return () => clearInterval(id);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
// Live clock for TOC situational awareness
|
|
||||||
useEffect(() => {
|
|
||||||
const id = setInterval(() =>
|
|
||||||
setClockStr(new Date().toLocaleTimeString([], { hour12: false, hour: "2-digit", minute: "2-digit", second: "2-digit" })),
|
|
||||||
1000
|
|
||||||
);
|
|
||||||
return () => clearInterval(id);
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
const ago = useMemo(() => (lastUpdated ? timeAgo(lastUpdated) : null), [lastUpdated, agoClock]);
|
const ago = useMemo(() => (lastUpdated ? timeAgo(lastUpdated) : null), [lastUpdated, agoClock]);
|
||||||
|
|
||||||
@@ -589,6 +646,20 @@ export default function MapView({ nodes, activeCalls, incidents = [], calls = []
|
|||||||
</FeatureGroup>
|
</FeatureGroup>
|
||||||
</LayersControl.Overlay>
|
</LayersControl.Overlay>
|
||||||
|
|
||||||
|
{/* Overlay: Aircraft — node-26#9 second-SDR ADS-B live snapshot, opt-in */}
|
||||||
|
<LayersControl.Overlay name="Aircraft">
|
||||||
|
<FeatureGroup>
|
||||||
|
<AircraftLayer />
|
||||||
|
</FeatureGroup>
|
||||||
|
</LayersControl.Overlay>
|
||||||
|
|
||||||
|
{/* Overlay: Vessels — node-26#9 second-SDR AIS live snapshot, opt-in */}
|
||||||
|
<LayersControl.Overlay name="Vessels">
|
||||||
|
<FeatureGroup>
|
||||||
|
<VesselLayer />
|
||||||
|
</FeatureGroup>
|
||||||
|
</LayersControl.Overlay>
|
||||||
|
|
||||||
{/* Overlay: Weather Radar — NEXRAD via Iowa Env Mesonet; key forces remount on refresh */}
|
{/* Overlay: Weather Radar — NEXRAD via Iowa Env Mesonet; key forces remount on refresh */}
|
||||||
<LayersControl.Overlay name="Weather Radar">
|
<LayersControl.Overlay name="Weather Radar">
|
||||||
<TileLayer
|
<TileLayer
|
||||||
@@ -623,13 +694,8 @@ export default function MapView({ nodes, activeCalls, incidents = [], calls = []
|
|||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* ── Clock — bottom-left for TOC situational awareness ───────────────── */}
|
|
||||||
<div className="absolute bottom-8 left-3 z-[1001] bg-surface/90 border border-line rounded-lg px-3 py-2 pointer-events-none">
|
|
||||||
<span className="text-ink text-sm font-mono tabular-nums">{clockStr}</span>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{/* ── Legend — shape-first, both themes. Never a bare colour swatch. ──── */}
|
{/* ── Legend — shape-first, both themes. Never a bare colour swatch. ──── */}
|
||||||
<div className="absolute bottom-8 right-3 z-[1001] bg-surface/90 border border-line rounded-lg px-3 py-2.5 text-xs pointer-events-none space-y-2">
|
<div className="absolute bottom-8 right-3 z-[1001] bg-surface/90 border border-line rounded-lg px-3 py-2.5 text-xs pointer-events-none space-y-2 max-h-[calc(100%-4rem)] overflow-y-auto">
|
||||||
<div className="space-y-1">
|
<div className="space-y-1">
|
||||||
<p className="text-ink-muted font-medium text-[10px] uppercase tracking-wide">Severity</p>
|
<p className="text-ink-muted font-medium text-[10px] uppercase tracking-wide">Severity</p>
|
||||||
{(["major", "moderate", "minor", "routine"] as Severity[]).map((sev) => (
|
{(["major", "moderate", "minor", "routine"] as Severity[]).map((sev) => (
|
||||||
@@ -673,15 +739,19 @@ export default function MapView({ nodes, activeCalls, incidents = [], calls = []
|
|||||||
{/* ── Incident overlay panel ───────────────────────────────────────────── */}
|
{/* ── Incident overlay panel ───────────────────────────────────────────── */}
|
||||||
{incidents.length > 0 && (
|
{incidents.length > 0 && (
|
||||||
<>
|
<>
|
||||||
{/* Desktop: left sidebar — starts below zoom controls + fit-all button */}
|
{/* Desktop: left sidebar — offset below the zoom stack + fit-all button
|
||||||
<div className="absolute top-[8rem] left-3 bottom-[4.5rem] z-[1001] hidden md:flex flex-col w-56 gap-1.5">
|
so it never overlaps the Leaflet +/- controls (#118). Height is
|
||||||
|
capped and the list scrolls on its own, so the rail never reaches
|
||||||
|
the bottom-right legend. pointer-events are off on the wrapper and
|
||||||
|
back on for the cards, so the map still pans in the gaps. */}
|
||||||
|
<div className="absolute top-[9.5rem] left-3 z-[1001] hidden md:flex flex-col w-56 gap-1.5 max-h-[calc(100%-12rem)] pointer-events-none">
|
||||||
{/* Gate A / A2 (server-26#46) — the rail's titles, locations and
|
{/* Gate A / A2 (server-26#46) — the rail's titles, locations and
|
||||||
unit counts are pipeline output. Pinned above the scroll area
|
unit counts are pipeline output. Pinned above the scroll area
|
||||||
so it cannot be scrolled off the screen it qualifies. */}
|
so it cannot be scrolled off the screen it qualifies. */}
|
||||||
<div className="bg-surface/90 backdrop-blur-sm border border-line rounded-lg px-2 py-1.5 shrink-0">
|
<div className="bg-surface/90 backdrop-blur-sm border border-line rounded-lg px-2 py-1.5 shrink-0 pointer-events-auto">
|
||||||
<MachineOutputNotice variant="inline" className="text-[10px] leading-snug items-start" />
|
<MachineOutputNotice variant="inline" className="text-[10px] leading-snug items-start" />
|
||||||
</div>
|
</div>
|
||||||
<div className="flex flex-col gap-1.5 overflow-y-auto">
|
<div className="flex flex-col gap-1.5 overflow-y-auto min-h-0 pointer-events-auto">
|
||||||
{incidents.map((inc) => {
|
{incidents.map((inc) => {
|
||||||
const color = severityColor(inc.severity);
|
const color = severityColor(inc.severity);
|
||||||
const age = inc.started_at ? timeAgo(new Date(inc.started_at)) : null;
|
const age = inc.started_at ? timeAgo(new Date(inc.started_at)) : null;
|
||||||
|
|||||||
@@ -5,15 +5,20 @@ import type { NodeRecord, SystemRecord } from "@/lib/types";
|
|||||||
interface Props {
|
interface Props {
|
||||||
node: NodeRecord;
|
node: NodeRecord;
|
||||||
system?: SystemRecord;
|
system?: SystemRecord;
|
||||||
|
/**
|
||||||
|
* When false, the card renders without its `/nodes/[id]` Link wrapper so a
|
||||||
|
* parent click handler can take the interaction (pending nodes open the
|
||||||
|
* config modal instead of navigating). Defaults to true.
|
||||||
|
*/
|
||||||
|
linkToDetail?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function NodeCard({ node, system }: Props) {
|
export function NodeCard({ node, system, linkToDetail = true }: Props) {
|
||||||
const lastSeen = node.last_seen
|
const lastSeen = node.last_seen
|
||||||
? new Date(node.last_seen).toLocaleTimeString()
|
? new Date(node.last_seen).toLocaleTimeString()
|
||||||
: "never";
|
: "never";
|
||||||
|
|
||||||
return (
|
const body = (
|
||||||
<Link href={`/nodes/${node.node_id}`}>
|
|
||||||
<div className="bg-gray-900 border border-gray-800 rounded-lg p-4 hover:border-gray-600 transition-colors cursor-pointer">
|
<div className="bg-gray-900 border border-gray-800 rounded-lg p-4 hover:border-gray-600 transition-colors cursor-pointer">
|
||||||
<div className="flex items-start justify-between mb-3">
|
<div className="flex items-start justify-between mb-3">
|
||||||
<div>
|
<div>
|
||||||
@@ -58,6 +63,7 @@ export function NodeCard({ node, system }: Props) {
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
</Link>
|
|
||||||
);
|
);
|
||||||
|
|
||||||
|
return linkToDetail ? <Link href={`/nodes/${node.node_id}`}>{body}</Link> : body;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -50,8 +50,8 @@ export function NodeConfigModal({ node, systems, onClose }: Props) {
|
|||||||
const selectedPreset = PRESETS.find((p) => p.value === preset);
|
const selectedPreset = PRESETS.find((p) => p.value === preset);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="fixed inset-0 bg-black/70 flex items-center justify-center z-50">
|
<div className="fixed inset-0 bg-black/70 flex items-center justify-center z-50 p-4">
|
||||||
<div className="bg-gray-900 border border-gray-700 rounded-xl p-6 w-full max-w-md font-mono">
|
<div className="bg-gray-900 border border-gray-700 rounded-xl p-6 w-full max-w-md font-mono max-h-[90vh] overflow-y-auto">
|
||||||
<h2 className="text-white font-semibold mb-1">Configure Node</h2>
|
<h2 className="text-white font-semibold mb-1">Configure Node</h2>
|
||||||
<p className="text-gray-400 text-sm mb-5">
|
<p className="text-gray-400 text-sm mb-5">
|
||||||
<span className="text-indigo-400">{node.node_id}</span> connected for the first time.
|
<span className="text-indigo-400">{node.node_id}</span> connected for the first time.
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
// A from/to pair of native date inputs. Values are the inputs' own
|
||||||
|
// "YYYY-MM-DD" strings; dayStart/dayEnd turn them into the local-midnight
|
||||||
|
// bounds a started_at range query needs, so "to" includes the whole day.
|
||||||
|
|
||||||
|
export function dayStart(ymd: string): Date | undefined {
|
||||||
|
if (!ymd) return undefined;
|
||||||
|
const [y, m, d] = ymd.split("-").map(Number);
|
||||||
|
return new Date(y, m - 1, d, 0, 0, 0, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function dayEnd(ymd: string): Date | undefined {
|
||||||
|
if (!ymd) return undefined;
|
||||||
|
const [y, m, d] = ymd.split("-").map(Number);
|
||||||
|
return new Date(y, m - 1, d, 23, 59, 59, 999);
|
||||||
|
}
|
||||||
|
|
||||||
|
const inputClass =
|
||||||
|
"bg-surface border border-line rounded-lg px-2 py-1.5 text-sm text-ink-2 focus:outline-none focus:border-accent";
|
||||||
|
|
||||||
|
export function DateRange({
|
||||||
|
from,
|
||||||
|
to,
|
||||||
|
onChange,
|
||||||
|
}: {
|
||||||
|
from: string;
|
||||||
|
to: string;
|
||||||
|
onChange: (from: string, to: string) => void;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<div className="flex items-center gap-2 text-xs text-ink-muted">
|
||||||
|
<input
|
||||||
|
type="date"
|
||||||
|
aria-label="From date"
|
||||||
|
value={from}
|
||||||
|
max={to || undefined}
|
||||||
|
onChange={(e) => onChange(e.target.value, to)}
|
||||||
|
className={inputClass}
|
||||||
|
/>
|
||||||
|
<span>to</span>
|
||||||
|
<input
|
||||||
|
type="date"
|
||||||
|
aria-label="To date"
|
||||||
|
value={to}
|
||||||
|
min={from || undefined}
|
||||||
|
onChange={(e) => onChange(from, e.target.value)}
|
||||||
|
className={inputClass}
|
||||||
|
/>
|
||||||
|
{(from || to) && (
|
||||||
|
<button onClick={() => onChange("", "")} className="text-ink-muted hover:text-ink-2">
|
||||||
|
clear
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -82,6 +82,8 @@ export const c2api = {
|
|||||||
link?: "any" | "orphan" | "linked";
|
link?: "any" | "orphan" | "linked";
|
||||||
transcript?: "any" | "yes" | "no";
|
transcript?: "any" | "yes" | "no";
|
||||||
q?: string;
|
q?: string;
|
||||||
|
date_from?: string;
|
||||||
|
date_to?: string;
|
||||||
}) => {
|
}) => {
|
||||||
const qs = new URLSearchParams();
|
const qs = new URLSearchParams();
|
||||||
for (const [k, v] of Object.entries(params)) {
|
for (const [k, v] of Object.entries(params)) {
|
||||||
@@ -100,6 +102,30 @@ export const c2api = {
|
|||||||
closeStallCalls: (olderThanMinutes: number, dryRun: boolean) =>
|
closeStallCalls: (olderThanMinutes: number, dryRun: boolean) =>
|
||||||
request<{ dry_run: boolean; older_than_minutes: number; count: number; call_ids: string[] }>(`/calls/close-stale?older_than_minutes=${olderThanMinutes}&dry_run=${dryRun}`, { method: "POST" }),
|
request<{ dry_run: boolean; older_than_minutes: number; count: number; call_ids: string[] }>(`/calls/close-stale?older_than_minutes=${olderThanMinutes}&dry_run=${dryRun}`, { method: "POST" }),
|
||||||
|
|
||||||
|
// STT eval harness (server-26#163) — separate from patchTranscript above,
|
||||||
|
// which is a production correction with real side effects (re-extraction,
|
||||||
|
// incident unlinking, vocabulary learning). This is pure measurement.
|
||||||
|
getEvalQueue: (limit: number, cursor?: string | null) => {
|
||||||
|
const qs = new URLSearchParams({ limit: String(limit) });
|
||||||
|
if (cursor) qs.set("cursor", cursor);
|
||||||
|
return request<{
|
||||||
|
calls: import("@/lib/types").CallRecord[];
|
||||||
|
next_cursor: string | null;
|
||||||
|
scanned: number;
|
||||||
|
matched: number;
|
||||||
|
window_exhausted: boolean;
|
||||||
|
}>(`/calls/eval-queue?${qs.toString()}`);
|
||||||
|
},
|
||||||
|
getEvalStats: () =>
|
||||||
|
request<{ eval_count: number; raw_wer: number | null; corrected_wer: number | null }>(
|
||||||
|
"/calls/eval-stats"
|
||||||
|
),
|
||||||
|
putEvalTranscript: (callId: string, text: string) =>
|
||||||
|
request<{ ok: boolean; call_id: string }>(`/calls/${callId}/eval-transcript`, {
|
||||||
|
method: "PUT",
|
||||||
|
body: JSON.stringify({ text }),
|
||||||
|
}),
|
||||||
|
|
||||||
// Incidents
|
// Incidents
|
||||||
getIncidents: (params?: { status?: string; type?: string }) => {
|
getIncidents: (params?: { status?: string; type?: string }) => {
|
||||||
const qs = params ? "?" + new URLSearchParams(params as Record<string, string>).toString() : "";
|
const qs = params ? "?" + new URLSearchParams(params as Record<string, string>).toString() : "";
|
||||||
|
|||||||
@@ -53,12 +53,39 @@ export interface NodeRecord {
|
|||||||
hardware_preset?: string;
|
hardware_preset?: string;
|
||||||
ppm_override?: number | null;
|
ppm_override?: number | null;
|
||||||
node_type?: string;
|
node_type?: string;
|
||||||
|
secondary_sdr_mode?: string;
|
||||||
|
sdr_count?: number;
|
||||||
enforce_override_timeout?: boolean;
|
enforce_override_timeout?: boolean;
|
||||||
is_overridden?: boolean;
|
is_overridden?: boolean;
|
||||||
override_system_id?: string | null;
|
override_system_id?: string | null;
|
||||||
override_timeout_at?: string | null;
|
override_timeout_at?: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface AircraftTrack {
|
||||||
|
icao: string;
|
||||||
|
org_id?: string;
|
||||||
|
node_id: string;
|
||||||
|
callsign: string | null;
|
||||||
|
lat: number | null;
|
||||||
|
lon: number | null;
|
||||||
|
altitude_ft: number | null;
|
||||||
|
ground_speed_kt: number | null;
|
||||||
|
track_deg: number | null;
|
||||||
|
last_seen: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VesselTrack {
|
||||||
|
mmsi: string;
|
||||||
|
org_id?: string;
|
||||||
|
node_id: string;
|
||||||
|
name: string | null;
|
||||||
|
lat: number | null;
|
||||||
|
lon: number | null;
|
||||||
|
speed_kt: number | null;
|
||||||
|
heading_deg: number | null;
|
||||||
|
last_seen: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface VocabularyPendingTerm {
|
export interface VocabularyPendingTerm {
|
||||||
term: string;
|
term: string;
|
||||||
source: "induction" | "correction";
|
source: "induction" | "correction";
|
||||||
@@ -131,6 +158,10 @@ export interface CallRecord {
|
|||||||
corr_incident_idle_min?: number | null;
|
corr_incident_idle_min?: number | null;
|
||||||
corr_shared_units?: number | null;
|
corr_shared_units?: number | null;
|
||||||
corr_candidates?: number | null;
|
corr_candidates?: number | null;
|
||||||
|
/** Human-verified reference transcript for the STT eval harness (server-26#163) — never read by anything downstream. */
|
||||||
|
eval_transcript?: string | null;
|
||||||
|
eval_transcript_by?: string | null;
|
||||||
|
eval_transcript_at?: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface IncidentRecord {
|
export interface IncidentRecord {
|
||||||
@@ -143,8 +174,9 @@ export interface IncidentRecord {
|
|||||||
call_ids: string[];
|
call_ids: string[];
|
||||||
system_ids: string[];
|
system_ids: string[];
|
||||||
talkgroup_ids: string[];
|
talkgroup_ids: string[];
|
||||||
units: string[];
|
/** Omitted on incident docs written before these fields existed. */
|
||||||
vehicles: string[];
|
units?: string[];
|
||||||
|
vehicles?: string[];
|
||||||
/** Units currently believed on scene — maintained by incident_correlator.py `_attach`. */
|
/** Units currently believed on scene — maintained by incident_correlator.py `_attach`. */
|
||||||
units_active?: string[];
|
units_active?: string[];
|
||||||
/** Units that reported clearing/back in service on this incident. */
|
/** Units that reported clearing/back in service on this incident. */
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useEffect, useState } from "react";
|
||||||
|
import { collection, onSnapshot, query, where, FirestoreError } from "firebase/firestore";
|
||||||
|
import { onAuthStateChanged } from "firebase/auth";
|
||||||
|
import { db, auth } from "@/lib/firebase";
|
||||||
|
import { useAuth } from "@/components/AuthProvider";
|
||||||
|
import type { AircraftTrack } from "@/lib/types";
|
||||||
|
|
||||||
|
// `aircraft` docs are a live snapshot (one per icao, overwritten on every
|
||||||
|
// sighting, node-26#9) — nothing prunes a doc when a plane leaves range, so
|
||||||
|
// staleness is filtered client-side rather than assuming the collection only
|
||||||
|
// ever holds current traffic.
|
||||||
|
const STALE_AFTER_MS = 2 * 60 * 1000;
|
||||||
|
|
||||||
|
export function useAircraft() {
|
||||||
|
const [aircraft, setAircraft] = useState<AircraftTrack[]>([]);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const { orgId } = useAuth();
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let unsubFirestore: (() => void) | undefined;
|
||||||
|
|
||||||
|
const unsubAuth = onAuthStateChanged(auth, (user) => {
|
||||||
|
if (unsubFirestore) { unsubFirestore(); unsubFirestore = undefined; }
|
||||||
|
|
||||||
|
if (!user || !orgId) {
|
||||||
|
setAircraft([]);
|
||||||
|
setLoading(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const q = query(collection(db, "aircraft"), where("org_id", "==", orgId));
|
||||||
|
unsubFirestore = onSnapshot(q, (snap) => {
|
||||||
|
const now = Date.now();
|
||||||
|
const fresh = snap.docs
|
||||||
|
.map((d) => d.data() as AircraftTrack)
|
||||||
|
.filter((a) => now - new Date(a.last_seen).getTime() < STALE_AFTER_MS);
|
||||||
|
setAircraft(fresh);
|
||||||
|
setLoading(false);
|
||||||
|
}, (err: FirestoreError) => { console.error("useAircraft:", err); setError(err.message); setLoading(false); });
|
||||||
|
});
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
unsubAuth();
|
||||||
|
if (unsubFirestore) unsubFirestore();
|
||||||
|
};
|
||||||
|
}, [orgId]);
|
||||||
|
|
||||||
|
return { aircraft, loading, error };
|
||||||
|
}
|
||||||
@@ -11,12 +11,19 @@ const toISO = (v: unknown): string =>
|
|||||||
(v as { toDate?: () => Date })?.toDate?.()?.toISOString?.() ??
|
(v as { toDate?: () => Date })?.toDate?.()?.toISOString?.() ??
|
||||||
(typeof v === "string" ? v : new Date().toISOString());
|
(typeof v === "string" ? v : new Date().toISOString());
|
||||||
|
|
||||||
export function useIncidents(limitCount = 100) {
|
export function useIncidents(limitCount = 100, dateFrom?: Date, dateTo?: Date) {
|
||||||
const [incidents, setIncidents] = useState<IncidentRecord[]>([]);
|
const [incidents, setIncidents] = useState<IncidentRecord[]>([]);
|
||||||
const [loading, setLoading] = useState(true);
|
const [loading, setLoading] = useState(true);
|
||||||
const [error, setError] = useState<string | null>(null);
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
// A full page means there may be older incidents past the limit; a short
|
||||||
|
// page means the query reached the end of the collection.
|
||||||
|
const [hasMore, setHasMore] = useState(false);
|
||||||
const { orgId } = useAuth();
|
const { orgId } = useAuth();
|
||||||
|
|
||||||
|
// Stable ms values so the effect dependency doesn't fire on every render
|
||||||
|
const dateFromMs = dateFrom?.getTime();
|
||||||
|
const dateToMs = dateTo?.getTime();
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
let unsubFirestore: (() => void) | undefined;
|
let unsubFirestore: (() => void) | undefined;
|
||||||
|
|
||||||
@@ -34,9 +41,18 @@ export function useIncidents(limitCount = 100) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A range on the ordered field rides the existing org_id/started_at index.
|
||||||
|
// Incident started_at is stored as a Python isoformat() STRING
|
||||||
|
// ("2026-09-20T12:00:00.123456+00:00", incident_correlator.py), not a
|
||||||
|
// Firestore timestamp — unlike calls. A Date bound compares by type and
|
||||||
|
// matches nothing, so the bounds go in as UTC ISO strings in the same
|
||||||
|
// shape, which then compare lexicographically in time order.
|
||||||
|
const isoBound = (ms: number) => new Date(ms).toISOString().replace("Z", "+00:00");
|
||||||
const q = query(
|
const q = query(
|
||||||
collection(db, "incidents"),
|
collection(db, "incidents"),
|
||||||
where("org_id", "==", orgId),
|
where("org_id", "==", orgId),
|
||||||
|
...(dateFromMs != null ? [where("started_at", ">=", isoBound(dateFromMs))] : []),
|
||||||
|
...(dateToMs != null ? [where("started_at", "<=", isoBound(dateToMs))] : []),
|
||||||
orderBy("started_at", "desc"),
|
orderBy("started_at", "desc"),
|
||||||
limit(limitCount)
|
limit(limitCount)
|
||||||
);
|
);
|
||||||
@@ -49,6 +65,7 @@ export function useIncidents(limitCount = 100) {
|
|||||||
updated_at: toISO(data.updated_at),
|
updated_at: toISO(data.updated_at),
|
||||||
} as IncidentRecord;
|
} as IncidentRecord;
|
||||||
}));
|
}));
|
||||||
|
setHasMore(snap.size >= limitCount);
|
||||||
setLoading(false);
|
setLoading(false);
|
||||||
}, (err: FirestoreError) => {
|
}, (err: FirestoreError) => {
|
||||||
console.error("useIncidents:", err);
|
console.error("useIncidents:", err);
|
||||||
@@ -61,9 +78,9 @@ export function useIncidents(limitCount = 100) {
|
|||||||
unsubAuth();
|
unsubAuth();
|
||||||
if (unsubFirestore) unsubFirestore();
|
if (unsubFirestore) unsubFirestore();
|
||||||
};
|
};
|
||||||
}, [limitCount, orgId]);
|
}, [limitCount, dateFromMs, dateToMs, orgId]);
|
||||||
|
|
||||||
return { incidents, loading, error };
|
return { incidents, loading, error, hasMore };
|
||||||
}
|
}
|
||||||
|
|
||||||
export function useIncident(incidentId: string | null) {
|
export function useIncident(incidentId: string | null) {
|
||||||
|
|||||||
@@ -0,0 +1,53 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useEffect, useState } from "react";
|
||||||
|
import { collection, onSnapshot, query, where, FirestoreError } from "firebase/firestore";
|
||||||
|
import { onAuthStateChanged } from "firebase/auth";
|
||||||
|
import { db, auth } from "@/lib/firebase";
|
||||||
|
import { useAuth } from "@/components/AuthProvider";
|
||||||
|
import type { VesselTrack } from "@/lib/types";
|
||||||
|
|
||||||
|
// Same shape as useAircraft — `vessels` is a live snapshot (one per mmsi,
|
||||||
|
// overwritten on every sighting, node-26#9), nothing prunes a doc when a
|
||||||
|
// vessel goes out of range, so staleness is filtered client-side. AIS
|
||||||
|
// position reports are much less frequent than ADS-B (minutes, not
|
||||||
|
// seconds), so this window is longer than useAircraft's.
|
||||||
|
const STALE_AFTER_MS = 10 * 60 * 1000;
|
||||||
|
|
||||||
|
export function useVessels() {
|
||||||
|
const [vessels, setVessels] = useState<VesselTrack[]>([]);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const { orgId } = useAuth();
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let unsubFirestore: (() => void) | undefined;
|
||||||
|
|
||||||
|
const unsubAuth = onAuthStateChanged(auth, (user) => {
|
||||||
|
if (unsubFirestore) { unsubFirestore(); unsubFirestore = undefined; }
|
||||||
|
|
||||||
|
if (!user || !orgId) {
|
||||||
|
setVessels([]);
|
||||||
|
setLoading(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const q = query(collection(db, "vessels"), where("org_id", "==", orgId));
|
||||||
|
unsubFirestore = onSnapshot(q, (snap) => {
|
||||||
|
const now = Date.now();
|
||||||
|
const fresh = snap.docs
|
||||||
|
.map((d) => d.data() as VesselTrack)
|
||||||
|
.filter((v) => now - new Date(v.last_seen).getTime() < STALE_AFTER_MS);
|
||||||
|
setVessels(fresh);
|
||||||
|
setLoading(false);
|
||||||
|
}, (err: FirestoreError) => { console.error("useVessels:", err); setError(err.message); setLoading(false); });
|
||||||
|
});
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
unsubAuth();
|
||||||
|
if (unsubFirestore) unsubFirestore();
|
||||||
|
};
|
||||||
|
}, [orgId]);
|
||||||
|
|
||||||
|
return { vessels, loading, error };
|
||||||
|
}
|
||||||
@@ -1,15 +1,15 @@
|
|||||||
{
|
{
|
||||||
"//": "Composite indexes for the c2-server database. Firestore auto-indexes single-field lookups and equality-only compound queries; an equality filter combined with an inequality, an orderBy on a different field, or array-contains needs an explicit composite index or the query fails at runtime with FAILED_PRECONDITION. Deploy with: firebase deploy --only firestore:indexes --project <project-id> (firebase.json pins database c2-server — without that key the CLI targets (default) and changes nothing the app can see).",
|
"//": "Composite indexes for the c2-server database. Firestore auto-indexes single-field lookups and equality-only compound queries; an equality filter combined with an inequality, an orderBy on a different field, or array-contains needs an explicit composite index or the query fails at runtime with FAILED_PRECONDITION. Deploy with: firebase deploy --only firestore:indexes --project <project-id> (firebase.json pins database c2-server — without that key the CLI targets (default) and changes nothing the app can see).",
|
||||||
"//direction": "Every index here is declared ASCENDING. Firestore scans an index in either direction, so org_id+started_at ASC serves orderBy(started_at, 'desc') as well — which is what every frontend hook actually asks for. Declaring only the ASC form keeps one index per query shape instead of a matched pair.",
|
"//direction": "The sort field's ORDER here must match the query's orderBy direction. The old note claimed 'Firestore scans either direction so ASC serves orderBy(desc)' — that is WRONG for these query shapes and cost us three broken pages (server-26 #33/#51/#110-followup, 2026-09-08): useCalls/useIncidents/useAlerts and c2-core search_calls all orderBy(x,'desc') and each got FAILED_PRECONDITION until an explicit DESCENDING index existed. A range/inequality filter with no orderBy (the backend status/ended_at, system_id/started_at, system_id/ended_at entries) is genuinely direction-agnostic and stays ASCENDING.",
|
||||||
"//drift-2026-08-23": "Reconciled against `gcloud firestore indexes composite list --database=c2-server` (server-26#33). The file had drifted four indexes behind the live database, and a deploy against the stale file then added ASC copies of indexes that already existed as DESC. The next deploy will offer to delete three live indexes that are deliberately not declared here — answer YES to all three: calls(org_id ASC, started_at DESC) and incidents(org_id ASC, started_at DESC) are duplicates of the ASC entries below, and alert_events(acknowledged ASC, triggered_at DESC) predates tenancy and is superseded by the org-scoped entry below. Nothing else may be deleted.",
|
"//drift-2026-09-08": "Reconciled against the live c2-server via `gcloud firestore indexes composite list` (server-26#33). Live already carries the three DESC indexes below (calls(org_id,started_at DESC), incidents(org_id,started_at DESC), alert_events(org_id,triggered_at DESC)) plus alert_events(acknowledged,org_id,triggered_at DESC) — created directly with gcloud on 2026-09-08 to unbreak Archive + Watch. This file now declares them so a `firebase deploy --only firestore:indexes` is a no-op, NOT a set of deletions. Do NOT delete calls(org_id,started_at DESC) or incidents(org_id,started_at DESC) — the pre-2026-09-08 note calling them deletable 'duplicates of the ASC entries' was the bug. The only genuinely dead index is the pre-tenancy alert_events(acknowledged,triggered_at) with no org_id, which may be deleted.",
|
||||||
"indexes": [
|
"indexes": [
|
||||||
{
|
{
|
||||||
"//": "drb-frontend lib/useCalls.ts — org-scoped call list, orderBy started_at desc.",
|
"//": "drb-frontend lib/useCalls.ts + c2-core routers/calls.py search_calls — org-scoped call list, orderBy started_at DESC.",
|
||||||
"collectionGroup": "calls",
|
"collectionGroup": "calls",
|
||||||
"queryScope": "COLLECTION",
|
"queryScope": "COLLECTION",
|
||||||
"fields": [
|
"fields": [
|
||||||
{ "fieldPath": "org_id", "order": "ASCENDING" },
|
{ "fieldPath": "org_id", "order": "ASCENDING" },
|
||||||
{ "fieldPath": "started_at", "order": "ASCENDING" }
|
{ "fieldPath": "started_at", "order": "DESCENDING" }
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -22,7 +22,7 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"//": "c2-core internal/recorrelation_sweep.py:45 — status == 'ended' AND ended_at >= cutoff. Backend only; was live but undeclared until 2026-08-23.",
|
"//": "c2-core internal/recorrelation_sweep.py:45 — status == 'ended' AND ended_at >= cutoff. Range filter, no orderBy: direction-agnostic. Backend only.",
|
||||||
"collectionGroup": "calls",
|
"collectionGroup": "calls",
|
||||||
"queryScope": "COLLECTION",
|
"queryScope": "COLLECTION",
|
||||||
"fields": [
|
"fields": [
|
||||||
@@ -31,7 +31,7 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"//": "c2-core internal/dedup.py:84 — system_id == X AND started_at within a +/- window. Was live-failing on essentially every inbound call (server-26#84): dedup caught the FAILED_PRECONDITION and degraded to \"not a duplicate\", so double-heard transmissions were stored twice and would have been transcribed and correlated twice the moment an AI window opened. Created directly on c2-server 2026-08-28.",
|
"//": "c2-core internal/dedup.py:84 — system_id == X AND started_at within a +/- window. Range filter, direction-agnostic. Was live-failing on essentially every inbound call (server-26#84): dedup caught the FAILED_PRECONDITION and degraded to \"not a duplicate\", so double-heard transmissions were stored twice. Created directly on c2-server 2026-08-28.",
|
||||||
"collectionGroup": "calls",
|
"collectionGroup": "calls",
|
||||||
"queryScope": "COLLECTION",
|
"queryScope": "COLLECTION",
|
||||||
"fields": [
|
"fields": [
|
||||||
@@ -40,7 +40,7 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"//": "c2-core internal/vocabulary_learner.py:290 — system_id == X AND ended_at >= cutoff. Backend only; was live but undeclared until 2026-08-23.",
|
"//": "c2-core internal/vocabulary_learner.py:290 — system_id == X AND ended_at >= cutoff. Range filter, direction-agnostic. Backend only.",
|
||||||
"collectionGroup": "calls",
|
"collectionGroup": "calls",
|
||||||
"queryScope": "COLLECTION",
|
"queryScope": "COLLECTION",
|
||||||
"fields": [
|
"fields": [
|
||||||
@@ -49,31 +49,31 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"//": "drb-frontend lib/useIncidents.ts — org-scoped incident browse, orderBy started_at desc.",
|
"//": "drb-frontend lib/useIncidents.ts — org-scoped incident browse, orderBy started_at DESC.",
|
||||||
"collectionGroup": "incidents",
|
"collectionGroup": "incidents",
|
||||||
"queryScope": "COLLECTION",
|
"queryScope": "COLLECTION",
|
||||||
"fields": [
|
"fields": [
|
||||||
{ "fieldPath": "org_id", "order": "ASCENDING" },
|
{ "fieldPath": "org_id", "order": "ASCENDING" },
|
||||||
{ "fieldPath": "started_at", "order": "ASCENDING" }
|
{ "fieldPath": "started_at", "order": "DESCENDING" }
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"//": "drb-frontend lib/useAlerts.ts — org-scoped alert feed, orderBy triggered_at desc.",
|
"//": "drb-frontend lib/useAlerts.ts — org-scoped alert feed, where(org_id ==) orderBy(triggered_at DESC).",
|
||||||
"collectionGroup": "alert_events",
|
"collectionGroup": "alert_events",
|
||||||
"queryScope": "COLLECTION",
|
"queryScope": "COLLECTION",
|
||||||
"fields": [
|
"fields": [
|
||||||
{ "fieldPath": "org_id", "order": "ASCENDING" },
|
{ "fieldPath": "org_id", "order": "ASCENDING" },
|
||||||
{ "fieldPath": "triggered_at", "order": "ASCENDING" }
|
{ "fieldPath": "triggered_at", "order": "DESCENDING" }
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"//": "drb-frontend lib/useAlerts.ts useUnacknowledgedAlerts — the nav badge.",
|
"//": "drb-frontend lib/useAlerts.ts useUnacknowledgedAlerts (nav badge) and the /watch \"Triggered Alerts\" tab — where(org_id ==) where(acknowledged == false) orderBy(triggered_at DESC). Field tuple + triggered_at DESCENDING copy the console create_composite link verbatim (server-26#51). Distinct from the (org_id, triggered_at) feed index above (no acknowledged filter).",
|
||||||
"collectionGroup": "alert_events",
|
"collectionGroup": "alert_events",
|
||||||
"queryScope": "COLLECTION",
|
"queryScope": "COLLECTION",
|
||||||
"fields": [
|
"fields": [
|
||||||
{ "fieldPath": "org_id", "order": "ASCENDING" },
|
|
||||||
{ "fieldPath": "acknowledged", "order": "ASCENDING" },
|
{ "fieldPath": "acknowledged", "order": "ASCENDING" },
|
||||||
{ "fieldPath": "triggered_at", "order": "ASCENDING" }
|
{ "fieldPath": "org_id", "order": "ASCENDING" },
|
||||||
|
{ "fieldPath": "triggered_at", "order": "DESCENDING" }
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -95,6 +95,18 @@ service cloud.firestore {
|
|||||||
allow write: if false;
|
allow write: if false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Live map overlays fed by a node's second SDR (node-26#9). Snapshot
|
||||||
|
// docs, one per icao/mmsi, last-seen-wins — not a history collection.
|
||||||
|
match /aircraft/{icao} {
|
||||||
|
allow read: if docInMyOrg();
|
||||||
|
allow write: if false;
|
||||||
|
}
|
||||||
|
|
||||||
|
match /vessels/{mmsi} {
|
||||||
|
allow read: if docInMyOrg();
|
||||||
|
allow write: if false;
|
||||||
|
}
|
||||||
|
|
||||||
match /alert_events/{alertId} {
|
match /alert_events/{alertId} {
|
||||||
allow read: if docInMyOrg();
|
allow read: if docInMyOrg();
|
||||||
allow write: if false;
|
allow write: if false;
|
||||||
|
|||||||
Reference in New Issue
Block a user