admin: STT eval harness — record human-verified transcripts, measure real WER (#163)
Backend: three new routes on the calls router, deliberately separate from
PATCH /{call_id}/transcript (a production correction with real side effects
-- re-extraction, incident unlinking, vocabulary learning). This is pure
measurement and must never share that code path.
GET /calls/eval-queue -- calls with a transcript but no
eval_transcript yet, paged (same bounded-
window-plus-cursor shape as /search)
PUT /{call_id}/eval-transcript -- records eval_transcript/_by/_at only;
never touches transcript/transcript_corrected
GET /calls/eval-stats -- eval_count + average word error rate of
the raw and corrected machine transcripts
against the human-verified ones
internal/wer.py: standard word-level Levenshtein WER. Returns None (not 0.0)
when the reference is empty -- a call nobody transcribed must not score as a
perfect match.
Frontend: a new "STT Eval" tab on /admin -- one call at a time, audio player,
a textarea pre-filled with the machine transcript to correct into ground
truth, Save & next / Skip, running WER stats at the top. Built for working a
handful of calls at a time over however many sittings it takes, not a
one-shot form: the queue auto-refills from where the last save left off.
Verified: 438 pass, 0 fail (12 new backend tests). Frontend is UNVERIFIED --
this box has no Node.js/npm (confirmed absent), so neither typecheck nor the
dev server could be run. Matches existing code patterns and the CallRecord/
c2api types by manual review only.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
241a15b8da
commit
5f85a878fa
@@ -0,0 +1,44 @@
|
|||||||
|
"""
|
||||||
|
Word error rate — server-26#163's eval harness needs a real number to compare
|
||||||
|
against, not a vibe. Standard definition: word-level Levenshtein distance
|
||||||
|
between a human-verified reference and the machine hypothesis, divided by the
|
||||||
|
reference's own word count. Case-insensitive, punctuation-insensitive — this
|
||||||
|
measures whether the right WORDS came out, not transcript formatting.
|
||||||
|
"""
|
||||||
|
import re
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
|
||||||
|
def _tokenize(text: str) -> list[str]:
|
||||||
|
return re.findall(r"[\w']+", (text or "").lower())
|
||||||
|
|
||||||
|
|
||||||
|
def word_error_rate(reference: str, hypothesis: str) -> Optional[float]:
|
||||||
|
"""
|
||||||
|
(substitutions + deletions + insertions) / len(reference words).
|
||||||
|
|
||||||
|
None when the reference has no words — WER is undefined there, not 0.0;
|
||||||
|
a caller that defaults a None to 0.0 would report a perfect score for a
|
||||||
|
call nobody actually transcribed.
|
||||||
|
"""
|
||||||
|
ref = _tokenize(reference)
|
||||||
|
hyp = _tokenize(hypothesis)
|
||||||
|
if not ref:
|
||||||
|
return None
|
||||||
|
if not hyp:
|
||||||
|
return 1.0
|
||||||
|
|
||||||
|
n, m = len(ref), len(hyp)
|
||||||
|
# Single-row DP over Levenshtein distance — O(n*m) time, O(m) space.
|
||||||
|
row = list(range(m + 1))
|
||||||
|
for i in range(1, n + 1):
|
||||||
|
prev_diag = row[0]
|
||||||
|
row[0] = i
|
||||||
|
for j in range(1, m + 1):
|
||||||
|
prev_row_j = row[j]
|
||||||
|
if ref[i - 1] == hyp[j - 1]:
|
||||||
|
row[j] = prev_diag
|
||||||
|
else:
|
||||||
|
row[j] = 1 + min(prev_diag, row[j], row[j - 1])
|
||||||
|
prev_diag = prev_row_j
|
||||||
|
return row[m] / n
|
||||||
@@ -15,6 +15,10 @@ from app.internal.storage import gcs_uri_for_call, with_playback_url
|
|||||||
class TranscriptUpdate(BaseModel):
|
class TranscriptUpdate(BaseModel):
|
||||||
transcript: str
|
transcript: str
|
||||||
|
|
||||||
|
|
||||||
|
class EvalTranscriptUpdate(BaseModel):
|
||||||
|
text: str
|
||||||
|
|
||||||
router = APIRouter(prefix="/calls", tags=["calls"])
|
router = APIRouter(prefix="/calls", tags=["calls"])
|
||||||
|
|
||||||
|
|
||||||
@@ -130,6 +134,109 @@ async def search_calls(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/eval-queue")
|
||||||
|
async def eval_queue(
|
||||||
|
limit: int = Query(5, ge=1, le=20),
|
||||||
|
cursor: Optional[str] = Query(None, description="started_at of the last row of the previous page"),
|
||||||
|
decoded: dict = Depends(require_admin_token),
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
A batch of calls that have a machine transcript but no human-verified one
|
||||||
|
yet — the backend for the STT eval page (server-26#163).
|
||||||
|
|
||||||
|
Deliberately separate from `PATCH /{call_id}/transcript`: that route is a
|
||||||
|
PRODUCTION correction — it re-runs extraction, unlinks incidents, and
|
||||||
|
feeds the vocabulary learner. An eval annotation must never trigger any
|
||||||
|
of that; it only exists to measure the pipeline, not to change what it
|
||||||
|
already decided. `eval_transcript` lives next to `transcript`/
|
||||||
|
`transcript_corrected` on the call doc and nothing downstream reads it.
|
||||||
|
|
||||||
|
Same bounded-window-scan-plus-cursor shape as `/search`, for the same
|
||||||
|
reason: no composite index exists for "eval_transcript is unset", and one
|
||||||
|
scan ordered by started_at is already trusted here. Paging through with
|
||||||
|
the returned cursor is how "however many, over time" actually works —
|
||||||
|
each call is where the last session left off, not a fresh random sample.
|
||||||
|
"""
|
||||||
|
org_id = await resolve_caller_org_id(decoded)
|
||||||
|
if org_id is None:
|
||||||
|
org_id = decoded.get("org_id")
|
||||||
|
if not org_id:
|
||||||
|
raise HTTPException(403, "No organization scope for this caller.")
|
||||||
|
|
||||||
|
window = max(limit * 20, 300)
|
||||||
|
rows = await fstore.collection_where(
|
||||||
|
"calls",
|
||||||
|
[("org_id", "==", org_id)],
|
||||||
|
order_by=[("started_at", "DESCENDING")],
|
||||||
|
limit_to=window,
|
||||||
|
start_after={"started_at": cursor} if cursor else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _eligible(c: dict) -> bool:
|
||||||
|
text = c.get("transcript_corrected") or c.get("transcript") or ""
|
||||||
|
return bool(text) and not c.get("eval_transcript")
|
||||||
|
|
||||||
|
matches = [c for c in rows if _eligible(c)]
|
||||||
|
page = matches[:limit]
|
||||||
|
|
||||||
|
next_cursor = None
|
||||||
|
if len(rows) == window:
|
||||||
|
last_scanned = rows[-1].get("started_at")
|
||||||
|
next_cursor = last_scanned.isoformat() if hasattr(last_scanned, "isoformat") else last_scanned
|
||||||
|
|
||||||
|
return {
|
||||||
|
"calls": [with_playback_url(c) for c in page],
|
||||||
|
"next_cursor": next_cursor,
|
||||||
|
"scanned": len(rows),
|
||||||
|
"matched": len(matches),
|
||||||
|
"window_exhausted": len(rows) == window,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/eval-stats")
|
||||||
|
async def eval_stats(decoded: dict = Depends(require_admin_token)):
|
||||||
|
"""
|
||||||
|
How many calls have a human-verified transcript, and the WER of the raw
|
||||||
|
and corrected machine transcripts against them (server-26#163).
|
||||||
|
|
||||||
|
Whole-collection scan, matching `GET /calls` (list_calls above) rather
|
||||||
|
than the bounded-window pattern the paged routes use: the eval set this
|
||||||
|
is measuring is built a few calls at a time and expected to stay small
|
||||||
|
(tens to hundreds), so a full scan filtered in Python is the honest
|
||||||
|
answer rather than a windowed guess that could miss eval'd calls sitting
|
||||||
|
outside a recency window.
|
||||||
|
"""
|
||||||
|
from app.internal.wer import word_error_rate
|
||||||
|
|
||||||
|
org_id = await resolve_caller_org_id(decoded)
|
||||||
|
filters = {"org_id": org_id} if org_id is not None else {}
|
||||||
|
calls = await fstore.collection_list("calls", **filters)
|
||||||
|
|
||||||
|
raw_wers: list[float] = []
|
||||||
|
corrected_wers: list[float] = []
|
||||||
|
for c in calls:
|
||||||
|
ref = c.get("eval_transcript")
|
||||||
|
if not ref:
|
||||||
|
continue
|
||||||
|
raw = c.get("transcript") or ""
|
||||||
|
corrected = c.get("transcript_corrected") or raw
|
||||||
|
raw_wer = word_error_rate(ref, raw)
|
||||||
|
corrected_wer = word_error_rate(ref, corrected)
|
||||||
|
if raw_wer is not None:
|
||||||
|
raw_wers.append(raw_wer)
|
||||||
|
if corrected_wer is not None:
|
||||||
|
corrected_wers.append(corrected_wer)
|
||||||
|
|
||||||
|
def _avg(xs: list[float]) -> Optional[float]:
|
||||||
|
return round(sum(xs) / len(xs), 4) if xs else None
|
||||||
|
|
||||||
|
return {
|
||||||
|
"eval_count": len(raw_wers),
|
||||||
|
"raw_wer": _avg(raw_wers),
|
||||||
|
"corrected_wer": _avg(corrected_wers),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{call_id}")
|
@router.get("/{call_id}")
|
||||||
async def get_call(call_id: str, decoded: dict = Depends(require_service_or_firebase_token)):
|
async def get_call(call_id: str, decoded: dict = Depends(require_service_or_firebase_token)):
|
||||||
call = await fstore.doc_get("calls", call_id)
|
call = await fstore.doc_get("calls", call_id)
|
||||||
@@ -313,3 +420,29 @@ async def patch_transcript(
|
|||||||
preserve_transcript_correction=True,
|
preserve_transcript_correction=True,
|
||||||
)
|
)
|
||||||
return {"ok": True, "call_id": call_id}
|
return {"ok": True, "call_id": call_id}
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/{call_id}/eval-transcript")
|
||||||
|
async def put_eval_transcript(
|
||||||
|
call_id: str,
|
||||||
|
body: EvalTranscriptUpdate,
|
||||||
|
decoded: dict = Depends(require_admin_token),
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Record a human-verified reference transcript for the STT eval harness
|
||||||
|
(server-26#163). Pure data capture — unlike `PATCH /{call_id}/transcript`
|
||||||
|
above, this never touches `transcript`/`transcript_corrected`, never
|
||||||
|
re-runs extraction, never unlinks incidents, and never feeds the
|
||||||
|
vocabulary learner. It exists to MEASURE the pipeline's output, not to
|
||||||
|
change it; the two must not share a code path.
|
||||||
|
"""
|
||||||
|
call = await fstore.doc_get("calls", call_id)
|
||||||
|
if not call:
|
||||||
|
raise HTTPException(404, f"Call '{call_id}' not found.")
|
||||||
|
|
||||||
|
await fstore.doc_set("calls", call_id, {
|
||||||
|
"eval_transcript": body.text,
|
||||||
|
"eval_transcript_by": decoded.get("email") or decoded.get("uid"),
|
||||||
|
"eval_transcript_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
})
|
||||||
|
return {"ok": True, "call_id": call_id}
|
||||||
|
|||||||
@@ -0,0 +1,145 @@
|
|||||||
|
"""
|
||||||
|
server-26#163 — the STT eval harness: word_error_rate() and the three routes
|
||||||
|
that back the /admin "STT Eval" tab.
|
||||||
|
|
||||||
|
Load-bearing property, checked directly: eval annotation must never touch
|
||||||
|
`transcript`/`transcript_corrected`, re-run extraction, or unlink incidents —
|
||||||
|
that's PATCH /{call_id}/transcript's job, a production correction with real
|
||||||
|
side effects. This is pure measurement and must stay pure.
|
||||||
|
"""
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
from app.internal.wer import word_error_rate
|
||||||
|
from app.main import app
|
||||||
|
from app.internal.auth import require_admin_token, require_service_or_firebase_token
|
||||||
|
from app.routers import calls
|
||||||
|
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
ADMIN = {"role": "admin", "org_id": "org-A"}
|
||||||
|
|
||||||
|
|
||||||
|
def _override(decoded: dict):
|
||||||
|
# calls.router carries its own router-level require_service_or_firebase_token
|
||||||
|
# (app/main.py) ON TOP OF each admin route's own require_admin_token — both
|
||||||
|
# have to be overridden or the router-level one 401s before the route's own
|
||||||
|
# dependency is ever evaluated.
|
||||||
|
app.dependency_overrides[require_admin_token] = lambda: decoded
|
||||||
|
app.dependency_overrides[require_service_or_firebase_token] = lambda: decoded
|
||||||
|
|
||||||
|
|
||||||
|
def teardown_function():
|
||||||
|
app.dependency_overrides.pop(require_admin_token, None)
|
||||||
|
app.dependency_overrides.pop(require_service_or_firebase_token, None)
|
||||||
|
|
||||||
|
|
||||||
|
# ── word_error_rate ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_identical_transcripts_are_zero_wer():
|
||||||
|
assert word_error_rate("K on the 600, I'm on Jackson Avenue.",
|
||||||
|
"K on the 600, I'm on Jackson Avenue.") == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_case_and_punctuation_are_ignored():
|
||||||
|
assert word_error_rate("Home Street and Forest Ave!", "home street and forest ave") == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_one_substitution_out_of_three_words():
|
||||||
|
assert word_error_rate("the cat sat", "the cat sit") == pytest.approx(1 / 3)
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_reference_is_undefined_not_zero():
|
||||||
|
"""A call nobody transcribed must not score as a perfect match."""
|
||||||
|
assert word_error_rate("", "anything") is None
|
||||||
|
assert word_error_rate(None, "anything") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_hypothesis_against_real_reference_is_total_loss():
|
||||||
|
assert word_error_rate("home street and forest ave", "") == 1.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_insertion_counts_against_the_hypothesis():
|
||||||
|
# reference 3 words, hypothesis adds 2 extra -> 2 insertions / 3 ref words
|
||||||
|
assert word_error_rate("show me clear", "show me clear right now") == pytest.approx(2 / 3)
|
||||||
|
|
||||||
|
|
||||||
|
# ── GET /calls/eval-queue ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _call(call_id, transcript="a real transcript here", corrected=None, eval_transcript=None, org_id="org-A"):
|
||||||
|
return {
|
||||||
|
"call_id": call_id, "org_id": org_id, "started_at": "2026-09-21T00:00:00+00:00",
|
||||||
|
"transcript": transcript, "transcript_corrected": corrected, "eval_transcript": eval_transcript,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_eval_queue_skips_already_evaluated_and_transcript_less_calls():
|
||||||
|
rows = [
|
||||||
|
_call("c1", eval_transcript="already done"),
|
||||||
|
_call("c2", transcript=None),
|
||||||
|
_call("c3"),
|
||||||
|
]
|
||||||
|
_override(ADMIN)
|
||||||
|
with patch.object(calls.fstore, "collection_where", AsyncMock(return_value=rows)):
|
||||||
|
resp = client.get("/calls/eval-queue")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = resp.json()
|
||||||
|
assert [c["call_id"] for c in body["calls"]] == ["c3"]
|
||||||
|
assert body["matched"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_eval_queue_requires_an_org_scope():
|
||||||
|
_override({"role": "admin"}) # platform admin, no org claim
|
||||||
|
resp = client.get("/calls/eval-queue")
|
||||||
|
assert resp.status_code == 403
|
||||||
|
|
||||||
|
|
||||||
|
# ── GET /calls/eval-stats ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_eval_stats_averages_wer_across_evaluated_calls_only():
|
||||||
|
rows = [
|
||||||
|
_call("c1", transcript="the cat sat", corrected="the cat sat", eval_transcript="the cat sat"), # 0.0 / 0.0
|
||||||
|
_call("c2", transcript="the cat sit", corrected="the cat sat", eval_transcript="the cat sat"), # raw 1/3, corrected 0.0
|
||||||
|
_call("c3", eval_transcript=None), # excluded entirely
|
||||||
|
]
|
||||||
|
_override(ADMIN)
|
||||||
|
with patch.object(calls.fstore, "collection_list", AsyncMock(return_value=rows)):
|
||||||
|
resp = client.get("/calls/eval-stats")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = resp.json()
|
||||||
|
assert body["eval_count"] == 2
|
||||||
|
assert body["raw_wer"] == pytest.approx((0.0 + 1 / 3) / 2, abs=1e-4)
|
||||||
|
assert body["corrected_wer"] == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_eval_stats_with_nothing_evaluated_yet_reports_none_not_zero():
|
||||||
|
_override(ADMIN)
|
||||||
|
with patch.object(calls.fstore, "collection_list", AsyncMock(return_value=[_call("c1")])):
|
||||||
|
resp = client.get("/calls/eval-stats")
|
||||||
|
body = resp.json()
|
||||||
|
assert body == {"eval_count": 0, "raw_wer": None, "corrected_wer": None}
|
||||||
|
|
||||||
|
|
||||||
|
# ── PUT /{call_id}/eval-transcript ──────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_put_eval_transcript_writes_only_eval_fields():
|
||||||
|
_override(ADMIN)
|
||||||
|
existing = _call("c1", transcript="raw text", corrected="corrected text")
|
||||||
|
with patch.object(calls.fstore, "doc_get", AsyncMock(return_value=existing)), \
|
||||||
|
patch.object(calls.fstore, "doc_set", AsyncMock()) as mock_set:
|
||||||
|
resp = client.put("/calls/c1/eval-transcript", json={"text": "the verified ground truth"})
|
||||||
|
assert resp.status_code == 200
|
||||||
|
(collection, doc_id, doc), _ = mock_set.await_args
|
||||||
|
assert collection == "calls" and doc_id == "c1"
|
||||||
|
assert doc["eval_transcript"] == "the verified ground truth"
|
||||||
|
assert doc["eval_transcript_at"]
|
||||||
|
assert "transcript" not in doc and "transcript_corrected" not in doc
|
||||||
|
|
||||||
|
|
||||||
|
def test_put_eval_transcript_404s_on_missing_call():
|
||||||
|
_override(ADMIN)
|
||||||
|
with patch.object(calls.fstore, "doc_get", AsyncMock(return_value=None)):
|
||||||
|
resp = client.put("/calls/nope/eval-transcript", json={"text": "x"})
|
||||||
|
assert resp.status_code == 404
|
||||||
@@ -4,7 +4,7 @@ import { useAuth } from "@/components/AuthProvider";
|
|||||||
import { c2api } from "@/lib/c2api";
|
import { c2api } from "@/lib/c2api";
|
||||||
import { useEffect, useState, useRef, useCallback } from "react";
|
import { useEffect, useState, useRef, useCallback } from "react";
|
||||||
import { useRouter } from "next/navigation";
|
import { useRouter } from "next/navigation";
|
||||||
import type { UserRecord, AuditEntry, UserRole } from "@/lib/types";
|
import type { UserRecord, AuditEntry, UserRole, CallRecord } from "@/lib/types";
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Shared primitives
|
// Shared primitives
|
||||||
@@ -1047,16 +1047,193 @@ function StaleCallsTab() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// STT eval (server-26#163) — the eval harness for real transcription
|
||||||
|
// accuracy. Separate from patchTranscript's "fix this call" flow: this never
|
||||||
|
// re-runs extraction or touches an incident, it only records what was
|
||||||
|
// actually said next to what Whisper heard, so eval-stats can report a real
|
||||||
|
// WER instead of a guess. Built to be worked in short sessions, a handful of
|
||||||
|
// calls at a time, over however many sittings it takes — not a one-shot form.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
function fmtPct(x: number | null | undefined): string {
|
||||||
|
return x === null || x === undefined ? "—" : `${(x * 100).toFixed(1)}%`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function EvalStatsBar({ stats }: { stats: { eval_count: number; raw_wer: number | null; corrected_wer: number | null } | null }) {
|
||||||
|
return (
|
||||||
|
<div className="bg-gray-900 border border-gray-800 rounded-xl p-4 flex flex-wrap gap-x-8 gap-y-2">
|
||||||
|
<div>
|
||||||
|
<p className="text-xs text-gray-500 font-mono">Calls verified</p>
|
||||||
|
<p className="text-white text-lg font-mono">{stats?.eval_count ?? "—"}</p>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<p className="text-xs text-gray-500 font-mono">Raw WER (whisper-1)</p>
|
||||||
|
<p className="text-white text-lg font-mono">{fmtPct(stats?.raw_wer)}</p>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<p className="text-xs text-gray-500 font-mono">Corrected WER (shipped)</p>
|
||||||
|
<p className="text-white text-lg font-mono">{fmtPct(stats?.corrected_wer)}</p>
|
||||||
|
</div>
|
||||||
|
{stats && stats.eval_count > 0 && stats.eval_count < 20 && (
|
||||||
|
<p className="text-xs text-amber-400 font-mono self-end">
|
||||||
|
fewer than 20 calls — numbers will move a lot until this grows
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function SttEvalTab() {
|
||||||
|
const [stats, setStats] = useState<{ eval_count: number; raw_wer: number | null; corrected_wer: number | null } | null>(null);
|
||||||
|
const [queue, setQueue] = useState<CallRecord[]>([]);
|
||||||
|
const [cursor, setCursor] = useState<string | null>(null);
|
||||||
|
const [exhausted, setExhausted] = useState(false);
|
||||||
|
const [draft, setDraft] = useState("");
|
||||||
|
const [loadingBatch, setLoadingBatch] = useState(false);
|
||||||
|
const [saving, setSaving] = useState(false);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const fetching = useRef(false);
|
||||||
|
|
||||||
|
const current = queue[0] ?? null;
|
||||||
|
|
||||||
|
const refreshStats = useCallback(() => {
|
||||||
|
c2api.getEvalStats().then(setStats).catch(() => { /* stats are a nice-to-have, not load-bearing */ });
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const loadBatch = useCallback(async () => {
|
||||||
|
if (fetching.current) return;
|
||||||
|
fetching.current = true;
|
||||||
|
setLoadingBatch(true);
|
||||||
|
setError(null);
|
||||||
|
try {
|
||||||
|
const res = await c2api.getEvalQueue(5, cursor);
|
||||||
|
setQueue((q) => [...q, ...res.calls]);
|
||||||
|
setCursor(res.next_cursor);
|
||||||
|
if (res.calls.length === 0 && !res.next_cursor) setExhausted(true);
|
||||||
|
} catch (e) {
|
||||||
|
setError(String(e));
|
||||||
|
} finally {
|
||||||
|
setLoadingBatch(false);
|
||||||
|
fetching.current = false;
|
||||||
|
}
|
||||||
|
}, [cursor]);
|
||||||
|
|
||||||
|
useEffect(() => { refreshStats(); }, [refreshStats]);
|
||||||
|
|
||||||
|
// Auto-refill: whenever the local queue runs dry and there's more to scan
|
||||||
|
// (or we haven't checked yet), pull another batch. Covers the sparse-window
|
||||||
|
// case too — a page with matches:0 but a next_cursor just means "keep
|
||||||
|
// scanning", not "done", so this fires again on its own.
|
||||||
|
useEffect(() => {
|
||||||
|
if (queue.length === 0 && !exhausted) loadBatch();
|
||||||
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
|
}, [queue.length, exhausted]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
setDraft(current ? (current.transcript_corrected || current.transcript || "") : "");
|
||||||
|
}, [current]);
|
||||||
|
|
||||||
|
async function saveAndNext() {
|
||||||
|
if (!current) return;
|
||||||
|
setSaving(true);
|
||||||
|
setError(null);
|
||||||
|
try {
|
||||||
|
await c2api.putEvalTranscript(current.call_id, draft);
|
||||||
|
setQueue((q) => q.slice(1));
|
||||||
|
refreshStats();
|
||||||
|
} catch (e) {
|
||||||
|
setError(String(e));
|
||||||
|
} finally {
|
||||||
|
setSaving(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function skip() {
|
||||||
|
setQueue((q) => q.slice(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="space-y-4">
|
||||||
|
<p className="text-xs text-gray-500 font-mono">
|
||||||
|
Listen to the audio, correct the transcript below until it matches what was actually said, then save.
|
||||||
|
This never touches the call's real transcript or re-runs anything — it only records ground truth
|
||||||
|
for measuring the pipeline. Do as many or as few as you have time for; it picks up where you left off.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<EvalStatsBar stats={stats} />
|
||||||
|
|
||||||
|
{error && (
|
||||||
|
<div className="bg-red-950 border border-red-800 rounded-lg p-3">
|
||||||
|
<p className="text-red-400 text-sm font-mono">{error}</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{current ? (
|
||||||
|
<div className="bg-gray-900 border border-gray-800 rounded-xl p-4 space-y-3">
|
||||||
|
<div className="flex flex-wrap items-center gap-x-3 gap-y-1 text-xs font-mono text-gray-400">
|
||||||
|
<span>{new Date(current.started_at).toLocaleString()}</span>
|
||||||
|
<span>{current.talkgroup_name || (current.talkgroup_id ? `TGID ${current.talkgroup_id}` : "unknown talkgroup")}</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{current.audio_url ? (
|
||||||
|
// eslint-disable-next-line jsx-a11y/media-has-caption
|
||||||
|
<audio controls src={current.audio_url} className="w-full h-9" />
|
||||||
|
) : (
|
||||||
|
<p className="text-xs text-gray-500 italic">No audio on this call — skip it.</p>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label className="text-xs text-gray-400 block mb-1">
|
||||||
|
Machine transcript (pre-filled) — correct it into what was actually said
|
||||||
|
</label>
|
||||||
|
<textarea
|
||||||
|
value={draft}
|
||||||
|
onChange={(e) => setDraft(e.target.value)}
|
||||||
|
rows={4}
|
||||||
|
className="w-full bg-gray-800 border border-gray-700 rounded-lg px-3 py-2 text-white text-sm font-mono focus:outline-none focus:border-indigo-500"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex gap-2">
|
||||||
|
<button
|
||||||
|
onClick={saveAndNext}
|
||||||
|
disabled={saving || !draft.trim()}
|
||||||
|
className="bg-indigo-600 hover:bg-indigo-500 disabled:opacity-50 text-white text-sm font-mono px-4 py-1.5 rounded-lg transition-colors"
|
||||||
|
>
|
||||||
|
{saving ? "Saving…" : "Save & next"}
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
onClick={skip}
|
||||||
|
disabled={saving}
|
||||||
|
className="bg-gray-800 hover:bg-gray-700 disabled:opacity-50 border border-gray-700 text-white text-sm font-mono px-4 py-1.5 rounded-lg transition-colors"
|
||||||
|
>
|
||||||
|
Skip
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="bg-gray-900 border border-gray-800 rounded-xl p-4">
|
||||||
|
<p className="text-sm font-mono text-gray-400">
|
||||||
|
{loadingBatch ? "Loading calls…" : exhausted ? "Nothing left to verify right now — check back after more calls come in." : "Loading…"}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Main admin page
|
// Main admin page
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
type AdminTab = "features" | "correlation" | "users" | "audit" | "calls";
|
type AdminTab = "features" | "correlation" | "users" | "audit" | "calls" | "eval";
|
||||||
|
|
||||||
const TAB_LABELS: { key: AdminTab; label: string }[] = [
|
const TAB_LABELS: { key: AdminTab; label: string }[] = [
|
||||||
{ key: "features", label: "AI Features" },
|
{ key: "features", label: "AI Features" },
|
||||||
{ key: "correlation", label: "Correlation Debug" },
|
{ key: "correlation", label: "Correlation Debug" },
|
||||||
{ key: "calls", label: "Calls" },
|
{ key: "calls", label: "Calls" },
|
||||||
|
{ key: "eval", label: "STT Eval" },
|
||||||
{ key: "users", label: "Users" },
|
{ key: "users", label: "Users" },
|
||||||
{ key: "audit", label: "Audit Log" },
|
{ key: "audit", label: "Audit Log" },
|
||||||
];
|
];
|
||||||
@@ -1102,6 +1279,7 @@ export default function AdminPage() {
|
|||||||
{tab === "features" && <FeaturesTab />}
|
{tab === "features" && <FeaturesTab />}
|
||||||
{tab === "correlation" && <CorrelationDebugTab />}
|
{tab === "correlation" && <CorrelationDebugTab />}
|
||||||
{tab === "calls" && <StaleCallsTab />}
|
{tab === "calls" && <StaleCallsTab />}
|
||||||
|
{tab === "eval" && <SttEvalTab />}
|
||||||
{tab === "users" && <UsersTab currentUid={user?.uid ?? ""} />}
|
{tab === "users" && <UsersTab currentUid={user?.uid ?? ""} />}
|
||||||
{tab === "audit" && <AuditLogTab />}
|
{tab === "audit" && <AuditLogTab />}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -100,6 +100,30 @@ export const c2api = {
|
|||||||
closeStallCalls: (olderThanMinutes: number, dryRun: boolean) =>
|
closeStallCalls: (olderThanMinutes: number, dryRun: boolean) =>
|
||||||
request<{ dry_run: boolean; older_than_minutes: number; count: number; call_ids: string[] }>(`/calls/close-stale?older_than_minutes=${olderThanMinutes}&dry_run=${dryRun}`, { method: "POST" }),
|
request<{ dry_run: boolean; older_than_minutes: number; count: number; call_ids: string[] }>(`/calls/close-stale?older_than_minutes=${olderThanMinutes}&dry_run=${dryRun}`, { method: "POST" }),
|
||||||
|
|
||||||
|
// STT eval harness (server-26#163) — separate from patchTranscript above,
|
||||||
|
// which is a production correction with real side effects (re-extraction,
|
||||||
|
// incident unlinking, vocabulary learning). This is pure measurement.
|
||||||
|
getEvalQueue: (limit: number, cursor?: string | null) => {
|
||||||
|
const qs = new URLSearchParams({ limit: String(limit) });
|
||||||
|
if (cursor) qs.set("cursor", cursor);
|
||||||
|
return request<{
|
||||||
|
calls: import("@/lib/types").CallRecord[];
|
||||||
|
next_cursor: string | null;
|
||||||
|
scanned: number;
|
||||||
|
matched: number;
|
||||||
|
window_exhausted: boolean;
|
||||||
|
}>(`/calls/eval-queue?${qs.toString()}`);
|
||||||
|
},
|
||||||
|
getEvalStats: () =>
|
||||||
|
request<{ eval_count: number; raw_wer: number | null; corrected_wer: number | null }>(
|
||||||
|
"/calls/eval-stats"
|
||||||
|
),
|
||||||
|
putEvalTranscript: (callId: string, text: string) =>
|
||||||
|
request<{ ok: boolean; call_id: string }>(`/calls/${callId}/eval-transcript`, {
|
||||||
|
method: "PUT",
|
||||||
|
body: JSON.stringify({ text }),
|
||||||
|
}),
|
||||||
|
|
||||||
// Incidents
|
// Incidents
|
||||||
getIncidents: (params?: { status?: string; type?: string }) => {
|
getIncidents: (params?: { status?: string; type?: string }) => {
|
||||||
const qs = params ? "?" + new URLSearchParams(params as Record<string, string>).toString() : "";
|
const qs = params ? "?" + new URLSearchParams(params as Record<string, string>).toString() : "";
|
||||||
|
|||||||
@@ -158,6 +158,10 @@ export interface CallRecord {
|
|||||||
corr_incident_idle_min?: number | null;
|
corr_incident_idle_min?: number | null;
|
||||||
corr_shared_units?: number | null;
|
corr_shared_units?: number | null;
|
||||||
corr_candidates?: number | null;
|
corr_candidates?: number | null;
|
||||||
|
/** Human-verified reference transcript for the STT eval harness (server-26#163) — never read by anything downstream. */
|
||||||
|
eval_transcript?: string | null;
|
||||||
|
eval_transcript_by?: string | null;
|
||||||
|
eval_transcript_at?: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface IncidentRecord {
|
export interface IncidentRecord {
|
||||||
|
|||||||
Reference in New Issue
Block a user