Replay: fail fast on a dead AI account; extraction reports to ai_health
First replay (290 calls, 09-22 10:00-12:00 ET) produced 0 incidents and no errors: every gpt-4o-mini extraction failed and _sync_extract swallowed it as "no scenes". Same shape as #169 — and the live extraction tier in /health/ai had no reporter at all, so this has been invisible in production too. - intelligence: API failures propagate out of _sync_extract; extract_scenes reports them to ai_health ("extraction" tier, billing/dead-model classified) and still returns [] so the pipeline degrades as before. - ai_health: inside a replay sandbox, failures go to the run's own sink instead of being dropped. - replay: aborts after 5 permanent failures on a tier, naming the cause; run metrics carry ai_failures; UI shows them. - replay estimate: audio minutes from started_at/ended_at (no duration field exists on call docs). - ReplayTab exposes the loaded run on window.__drbReplay for in-page analysis. c2-core: 458 pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
aff3f16d32
commit
ec91a9175f
@@ -369,3 +369,44 @@ async def test_replay_never_touches_live_ai_health_or_review_queue():
|
||||
finally:
|
||||
fstore.exit_sandbox(tok)
|
||||
assert ai_health.snapshot() == before
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_aborts_when_an_ai_account_is_dead(store):
|
||||
"""An unfunded OpenAI account made the first smoke run a sandbox of 290
|
||||
orphans that looked like a result. A permanently failing tier now stops
|
||||
the run and names the cause."""
|
||||
store.data["calls"] = {
|
||||
f"call-{i}": _live_call(i, i, "Car 12 responding to an MVA on Main Street") for i in range(1, 30)
|
||||
}
|
||||
|
||||
def broke(*a, **kw):
|
||||
raise RuntimeError("Error code: 429 - You exceeded your current quota (insufficient_quota)")
|
||||
|
||||
with patch("app.internal.intelligence._sync_extract", broke), \
|
||||
patch("app.internal.intelligence.classify_chatter", return_value=(False, None)):
|
||||
run = await replay.start_run(
|
||||
org_id="org-1", date_from=T0 - timedelta(hours=1), date_to=T0 + timedelta(hours=1),
|
||||
mode="transcripts", system_ids=None, source_run_id=None, label="", actor="t")
|
||||
await replay._active_task
|
||||
|
||||
run = store.data["replay_runs"][run["run_id"]]
|
||||
assert run["status"] == "failed"
|
||||
assert any("out of credit" in e for e in run["errors"])
|
||||
assert run["progress"]["done"] < 29
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_live_extraction_failure_reports_to_ai_health():
|
||||
from app.internal import ai_health, intelligence
|
||||
|
||||
def broke(*a, **kw):
|
||||
raise RuntimeError("insufficient_quota")
|
||||
|
||||
with patch.object(intelligence, "_sync_extract", broke), \
|
||||
patch.object(ai_health, "report_degraded") as degraded, \
|
||||
patch.object(fstore, "doc_set"), patch.object(fstore, "doc_get_cached", return_value=None):
|
||||
scenes = await intelligence.extract_scenes("c1", "Car 12 responding to an MVA on Main Street")
|
||||
assert scenes == []
|
||||
assert degraded.call_args.args[0] == "extraction"
|
||||
assert degraded.call_args.kwargs["permanent"] is True
|
||||
|
||||
Reference in New Issue
Block a user