admin: STT eval harness — record human-verified transcripts, measure real WER (#163)

Backend: three new routes on the calls router, deliberately separate from
PATCH /{call_id}/transcript (a production correction with real side effects
-- re-extraction, incident unlinking, vocabulary learning). This is pure
measurement and must never share that code path.

  GET  /calls/eval-queue        -- calls with a transcript but no
                                    eval_transcript yet, paged (same bounded-
                                    window-plus-cursor shape as /search)
  PUT  /{call_id}/eval-transcript -- records eval_transcript/_by/_at only;
                                      never touches transcript/transcript_corrected
  GET  /calls/eval-stats        -- eval_count + average word error rate of
                                    the raw and corrected machine transcripts
                                    against the human-verified ones

internal/wer.py: standard word-level Levenshtein WER. Returns None (not 0.0)
when the reference is empty -- a call nobody transcribed must not score as a
perfect match.

Frontend: a new "STT Eval" tab on /admin -- one call at a time, audio player,
a textarea pre-filled with the machine transcript to correct into ground
truth, Save & next / Skip, running WER stats at the top. Built for working a
handful of calls at a time over however many sittings it takes, not a
one-shot form: the queue auto-refills from where the last save left off.

Verified: 438 pass, 0 fail (12 new backend tests). Frontend is UNVERIFIED --
this box has no Node.js/npm (confirmed absent), so neither typecheck nor the
dev server could be run. Matches existing code patterns and the CallRecord/
c2api types by manual review only.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

drb-c2-core/app/internal/wer.py

@@ -0,0 +1,44 @@
+"""
+Word error rate — server-26#163's eval harness needs a real number to compare
+against, not a vibe. Standard definition: word-level Levenshtein distance
+between a human-verified reference and the machine hypothesis, divided by the
+reference's own word count. Case-insensitive, punctuation-insensitive — this
+measures whether the right WORDS came out, not transcript formatting.
+"""
+import re
+from typing import Optional
+
+
+def _tokenize(text: str) -> list[str]:
+    return re.findall(r"[\w']+", (text or "").lower())
+
+
+def word_error_rate(reference: str, hypothesis: str) -> Optional[float]:
+    """
+    (substitutions + deletions + insertions) / len(reference words).
+
+    None when the reference has no words — WER is undefined there, not 0.0;
+    a caller that defaults a None to 0.0 would report a perfect score for a
+    call nobody actually transcribed.
+    """
+    ref = _tokenize(reference)
+    hyp = _tokenize(hypothesis)
+    if not ref:
+        return None
+    if not hyp:
+        return 1.0
+
+    n, m = len(ref), len(hyp)
+    # Single-row DP over Levenshtein distance — O(n*m) time, O(m) space.
+    row = list(range(m + 1))
+    for i in range(1, n + 1):
+        prev_diag = row[0]
+        row[0] = i
+        for j in range(1, m + 1):
+            prev_row_j = row[j]
+            if ref[i - 1] == hyp[j - 1]:
+                row[j] = prev_diag
+            else:
+                row[j] = 1 + min(prev_diag, row[j], row[j - 1])
+            prev_diag = prev_row_j
+    return row[m] / n

drb-c2-core/app/routers/calls.py

@@ -15,6 +15,10 @@ from app.internal.storage import gcs_uri_for_call, with_playback_url
 class TranscriptUpdate(BaseModel):
     transcript: str
 
+
+class EvalTranscriptUpdate(BaseModel):
+    text: str
+
 router = APIRouter(prefix="/calls", tags=["calls"])
 
 
@@ -130,6 +134,109 @@ async def search_calls(
     }
 
 
+@router.get("/eval-queue")
+async def eval_queue(
+    limit: int = Query(5, ge=1, le=20),
+    cursor: Optional[str] = Query(None, description="started_at of the last row of the previous page"),
+    decoded: dict = Depends(require_admin_token),
+):
+    """
+    A batch of calls that have a machine transcript but no human-verified one
+    yet — the backend for the STT eval page (server-26#163).
+
+    Deliberately separate from `PATCH /{call_id}/transcript`: that route is a
+    PRODUCTION correction — it re-runs extraction, unlinks incidents, and
+    feeds the vocabulary learner. An eval annotation must never trigger any
+    of that; it only exists to measure the pipeline, not to change what it
+    already decided. `eval_transcript` lives next to `transcript`/
+    `transcript_corrected` on the call doc and nothing downstream reads it.
+
+    Same bounded-window-scan-plus-cursor shape as `/search`, for the same
+    reason: no composite index exists for "eval_transcript is unset", and one
+    scan ordered by started_at is already trusted here. Paging through with
+    the returned cursor is how "however many, over time" actually works —
+    each call is where the last session left off, not a fresh random sample.
+    """
+    org_id = await resolve_caller_org_id(decoded)
+    if org_id is None:
+        org_id = decoded.get("org_id")
+    if not org_id:
+        raise HTTPException(403, "No organization scope for this caller.")
+
+    window = max(limit * 20, 300)
+    rows = await fstore.collection_where(
+        "calls",
+        [("org_id", "==", org_id)],
+        order_by=[("started_at", "DESCENDING")],
+        limit_to=window,
+        start_after={"started_at": cursor} if cursor else None,
+    )
+
+    def _eligible(c: dict) -> bool:
+        text = c.get("transcript_corrected") or c.get("transcript") or ""
+        return bool(text) and not c.get("eval_transcript")
+
+    matches = [c for c in rows if _eligible(c)]
+    page = matches[:limit]
+
+    next_cursor = None
+    if len(rows) == window:
+        last_scanned = rows[-1].get("started_at")
+        next_cursor = last_scanned.isoformat() if hasattr(last_scanned, "isoformat") else last_scanned
+
+    return {
+        "calls": [with_playback_url(c) for c in page],
+        "next_cursor": next_cursor,
+        "scanned": len(rows),
+        "matched": len(matches),
+        "window_exhausted": len(rows) == window,
+    }
+
+
+@router.get("/eval-stats")
+async def eval_stats(decoded: dict = Depends(require_admin_token)):
+    """
+    How many calls have a human-verified transcript, and the WER of the raw
+    and corrected machine transcripts against them (server-26#163).
+
+    Whole-collection scan, matching `GET /calls` (list_calls above) rather
+    than the bounded-window pattern the paged routes use: the eval set this
+    is measuring is built a few calls at a time and expected to stay small
+    (tens to hundreds), so a full scan filtered in Python is the honest
+    answer rather than a windowed guess that could miss eval'd calls sitting
+    outside a recency window.
+    """
+    from app.internal.wer import word_error_rate
+
+    org_id = await resolve_caller_org_id(decoded)
+    filters = {"org_id": org_id} if org_id is not None else {}
+    calls = await fstore.collection_list("calls", **filters)
+
+    raw_wers: list[float] = []
+    corrected_wers: list[float] = []
+    for c in calls:
+        ref = c.get("eval_transcript")
+        if not ref:
+            continue
+        raw = c.get("transcript") or ""
+        corrected = c.get("transcript_corrected") or raw
+        raw_wer = word_error_rate(ref, raw)
+        corrected_wer = word_error_rate(ref, corrected)
+        if raw_wer is not None:
+            raw_wers.append(raw_wer)
+        if corrected_wer is not None:
+            corrected_wers.append(corrected_wer)
+
+    def _avg(xs: list[float]) -> Optional[float]:
+        return round(sum(xs) / len(xs), 4) if xs else None
+
+    return {
+        "eval_count": len(raw_wers),
+        "raw_wer": _avg(raw_wers),
+        "corrected_wer": _avg(corrected_wers),
+    }
+
+
 @router.get("/{call_id}")
 async def get_call(call_id: str, decoded: dict = Depends(require_service_or_firebase_token)):
     call = await fstore.doc_get("calls", call_id)
@@ -313,3 +420,29 @@ async def patch_transcript(
         preserve_transcript_correction=True,
     )
     return {"ok": True, "call_id": call_id}
+
+
+@router.put("/{call_id}/eval-transcript")
+async def put_eval_transcript(
+    call_id: str,
+    body: EvalTranscriptUpdate,
+    decoded: dict = Depends(require_admin_token),
+):
+    """
+    Record a human-verified reference transcript for the STT eval harness
+    (server-26#163). Pure data capture — unlike `PATCH /{call_id}/transcript`
+    above, this never touches `transcript`/`transcript_corrected`, never
+    re-runs extraction, never unlinks incidents, and never feeds the
+    vocabulary learner. It exists to MEASURE the pipeline's output, not to
+    change it; the two must not share a code path.
+    """
+    call = await fstore.doc_get("calls", call_id)
+    if not call:
+        raise HTTPException(404, f"Call '{call_id}' not found.")
+
+    await fstore.doc_set("calls", call_id, {
+        "eval_transcript": body.text,
+        "eval_transcript_by": decoded.get("email") or decoded.get("uid"),
+        "eval_transcript_at": datetime.now(timezone.utc).isoformat(),
+    })
+    return {"ok": True, "call_id": call_id}