server-26/drb-c2-core/tests/test_eval_transcript.py

"""
server-26#163 — the STT eval harness: word_error_rate() and the three routes
that back the /admin "STT Eval" tab.

Load-bearing property, checked directly: eval annotation must never touch
`transcript`/`transcript_corrected`, re-run extraction, or unlink incidents —
that's PATCH /{call_id}/transcript's job, a production correction with real
side effects. This is pure measurement and must stay pure.
"""
from unittest.mock import AsyncMock, patch

import pytest
from fastapi.testclient import TestClient

from app.internal.wer import word_error_rate
from app.main import app
from app.internal.auth import require_admin_token, require_service_or_firebase_token
from app.routers import calls

client = TestClient(app)

ADMIN = {"role": "admin", "org_id": "org-A"}


def _override(decoded: dict):
    # calls.router carries its own router-level require_service_or_firebase_token
    # (app/main.py) ON TOP OF each admin route's own require_admin_token — both
    # have to be overridden or the router-level one 401s before the route's own
    # dependency is ever evaluated.
    app.dependency_overrides[require_admin_token] = lambda: decoded
    app.dependency_overrides[require_service_or_firebase_token] = lambda: decoded


def teardown_function():
    app.dependency_overrides.pop(require_admin_token, None)
    app.dependency_overrides.pop(require_service_or_firebase_token, None)


# ── word_error_rate ─────────────────────────────────────────────────────────

def test_identical_transcripts_are_zero_wer():
    assert word_error_rate("K on the 600, I'm on Jackson Avenue.",
                            "K on the 600, I'm on Jackson Avenue.") == 0.0


def test_case_and_punctuation_are_ignored():
    assert word_error_rate("Home Street and Forest Ave!", "home street and forest ave") == 0.0


def test_one_substitution_out_of_three_words():
    assert word_error_rate("the cat sat", "the cat sit") == pytest.approx(1 / 3)


def test_empty_reference_is_undefined_not_zero():
    """A call nobody transcribed must not score as a perfect match."""
    assert word_error_rate("", "anything") is None
    assert word_error_rate(None, "anything") is None


def test_empty_hypothesis_against_real_reference_is_total_loss():
    assert word_error_rate("home street and forest ave", "") == 1.0


def test_insertion_counts_against_the_hypothesis():
    # reference 3 words, hypothesis adds 2 extra -> 2 insertions / 3 ref words
    assert word_error_rate("show me clear", "show me clear right now") == pytest.approx(2 / 3)


# ── GET /calls/eval-queue ───────────────────────────────────────────────────

def _call(call_id, transcript="a real transcript here", corrected=None, eval_transcript=None, org_id="org-A"):
    return {
        "call_id": call_id, "org_id": org_id, "started_at": "2026-09-21T00:00:00+00:00",
        "transcript": transcript, "transcript_corrected": corrected, "eval_transcript": eval_transcript,
    }


def test_eval_queue_skips_already_evaluated_and_transcript_less_calls():
    rows = [
        _call("c1", eval_transcript="already done"),
        _call("c2", transcript=None),
        _call("c3"),
    ]
    _override(ADMIN)
    with patch.object(calls.fstore, "collection_where", AsyncMock(return_value=rows)):
        resp = client.get("/calls/eval-queue")
    assert resp.status_code == 200
    body = resp.json()
    assert [c["call_id"] for c in body["calls"]] == ["c3"]
    assert body["matched"] == 1


def test_eval_queue_requires_an_org_scope():
    _override({"role": "admin"})  # platform admin, no org claim
    resp = client.get("/calls/eval-queue")
    assert resp.status_code == 403


# ── GET /calls/eval-stats ───────────────────────────────────────────────────

def test_eval_stats_averages_wer_across_evaluated_calls_only():
    rows = [
        _call("c1", transcript="the cat sat", corrected="the cat sat", eval_transcript="the cat sat"),  # 0.0 / 0.0
        _call("c2", transcript="the cat sit", corrected="the cat sat", eval_transcript="the cat sat"),  # raw 1/3, corrected 0.0
        _call("c3", eval_transcript=None),  # excluded entirely
    ]
    _override(ADMIN)
    with patch.object(calls.fstore, "collection_list", AsyncMock(return_value=rows)):
        resp = client.get("/calls/eval-stats")
    assert resp.status_code == 200
    body = resp.json()
    assert body["eval_count"] == 2
    assert body["raw_wer"] == pytest.approx((0.0 + 1 / 3) / 2, abs=1e-4)
    assert body["corrected_wer"] == 0.0


def test_eval_stats_with_nothing_evaluated_yet_reports_none_not_zero():
    _override(ADMIN)
    with patch.object(calls.fstore, "collection_list", AsyncMock(return_value=[_call("c1")])):
        resp = client.get("/calls/eval-stats")
    body = resp.json()
    assert body == {"eval_count": 0, "raw_wer": None, "corrected_wer": None}


# ── PUT /{call_id}/eval-transcript ──────────────────────────────────────────

def test_put_eval_transcript_writes_only_eval_fields():
    _override(ADMIN)
    existing = _call("c1", transcript="raw text", corrected="corrected text")
    with patch.object(calls.fstore, "doc_get", AsyncMock(return_value=existing)), \
         patch.object(calls.fstore, "doc_set", AsyncMock()) as mock_set:
        resp = client.put("/calls/c1/eval-transcript", json={"text": "the verified ground truth"})
    assert resp.status_code == 200
    (collection, doc_id, doc), _ = mock_set.await_args
    assert collection == "calls" and doc_id == "c1"
    assert doc["eval_transcript"] == "the verified ground truth"
    assert doc["eval_transcript_at"]
    assert "transcript" not in doc and "transcript_corrected" not in doc


def test_put_eval_transcript_404s_on_missing_call():
    _override(ADMIN)
    with patch.object(calls.fstore, "doc_get", AsyncMock(return_value=None)):
        resp = client.put("/calls/nope/eval-transcript", json={"text": "x"})
    assert resp.status_code == 404