Add AI provider degradation registry and alerting (server-26#14)

Three AI dependency failures in one night (retired Gemini model IDs,
depleted Gemini balance, unpayable OpenAI account) each surfaced only
as a single ERROR log line that nobody was watching. Add
app/internal/ai_health.py, a shared in-memory registry that
transcription.py and llm_correlator.py report into on every call
(success and failure), distinguishing permanent conditions (dead
model, dead billing) which alert immediately from transient ones
(rate limits, network blips) which only alert after they persist.
Alerts POST once per degradation episode and once on recovery to an
optional Discord webhook (AI_ALERT_WEBHOOK_URL), reusing alerter.py's
httpx pattern. State is exposed unauthenticated at GET /health/ai
alongside the existing /health.

Closes logan/server-26#14

drb-c2-core/app/config.py

@@ -101,6 +101,12 @@ class Settings(BaseSettings):
     # Defaults to "*" for local development only.
     cors_origins: list[str] = ["*"]
 
+    # Discord webhook URL that app/internal/ai_health.py posts to when an AI
+    # tier (transcription/correlation) transitions into or out of degraded
+    # state. Empty disables the POST entirely — not every self-hosted
+    # deployment will set this up, and skipping it must be silent.
+    ai_alert_webhook_url: str = ""
+
     class Config:
         env_file = ".env"
 

drb-c2-core/app/internal/ai_health.py

@@ -0,0 +1,192 @@
+"""
+Shared AI-provider degradation registry.
+
+On the night of 2026-08-18 three independent AI dependency failures (a
+retired Gemini model ID, a depleted Gemini balance, an unpayable OpenAI
+account) each surfaced only as a single ERROR log line -- and nobody reads
+container logs continuously. This module is the fix: every AI call site
+reports its outcome here instead of (or in addition to) just logging, so the
+current state of every AI tier can be read back over HTTP (see
+app/main.py's /health/ai) and pushed out to Discord on state changes.
+
+Tiers are tracked independently and in memory only (module-level singleton,
+no Firestore/DI -- consistent with the rest of this codebase). State is lost
+on restart, which is fine: a fresh process should re-derive degradation from
+the next few calls rather than resurrect a possibly-stale alert.
+
+The load-bearing distinction, from the incident this module exists to
+prevent: a PERMANENT condition (retired model, dead billing account, bad API
+key) will never clear on its own and must alert on the very first
+occurrence. A TRANSIENT condition (rate limit, network blip) clears by
+itself constantly and must NOT page anyone for the first failure -- only if
+it persists. classify() is the one place that tells the two apart from a
+provider error message, because both this module's callers (llm_correlator.py,
+transcription.py) need the exact same judgment call and must not each grow
+their own slightly-different copy that drifts.
+"""
+import asyncio
+from datetime import datetime, timezone
+from typing import Optional
+
+from app.internal.logger import logger
+from app.config import settings
+
+TIERS = ("transcription", "correlation_cheap", "correlation_smart", "extraction")
+
+# Consecutive failures a TRANSIENT condition must reach before it alerts.
+# Permanent conditions skip this entirely and alert on failure #1.
+TRANSIENT_ALERT_THRESHOLD = 5
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _default_state() -> dict:
+    return {
+        "degraded": False,
+        "permanent": False,
+        "provider": None,
+        "model": None,
+        "problem": None,
+        "fix": None,
+        "first_seen": None,
+        "last_seen": None,
+        "consecutive_failures": 0,
+        "alerted": False,
+    }
+
+
+_state: dict[str, dict] = {t: _default_state() for t in TIERS}
+
+
+def classify(text: str) -> str:
+    """
+    Classify a provider failure message body.
+
+    Returns "dead_model", "billing", or "transient".
+
+    A depleted balance and an ordinary rate limit both arrive as HTTP 429 --
+    the status code can't tell them apart, only the message body can. This
+    logic previously lived independently in llm_correlator.py and (in a
+    slightly different shape) transcription.py; it now lives here once, and
+    both call in rather than re-matching the text themselves.
+    """
+    low = text.lower()
+
+    if "404" in text or "not found" in low or "no longer available" in low:
+        return "dead_model"
+
+    if (
+        "credits are depleted" in low
+        or "prepayment" in low
+        or "billing" in low
+        or "insufficient_quota" in low
+        or "credit" in low
+        or "exceeded your current quota" in low
+    ):
+        return "billing"
+
+    return "transient"
+
+
+async def report_degraded(
+    tier: str,
+    provider: str,
+    model: str,
+    problem: str,
+    fix: str,
+    permanent: bool = False,
+) -> None:
+    """
+    Record a failure for `tier`. Call this from a failure path, once per
+    failure (it does its own once-per-episode alert suppression -- do not
+    gate the call site on that yourself).
+
+    permanent=True (dead model, unpayable account, bad key) alerts on this
+    very call. permanent=False (rate limit, network blip) only alerts once
+    TRANSIENT_ALERT_THRESHOLD consecutive failures have been reported for
+    this tier, so an ordinary blip never pages anyone.
+    """
+    if tier not in _state:
+        _state[tier] = _default_state()
+    entry = _state[tier]
+
+    now = _now()
+    if entry["consecutive_failures"] == 0:
+        entry["first_seen"] = now
+    entry["last_seen"] = now
+    entry["consecutive_failures"] += 1
+    entry["provider"] = provider
+    entry["model"] = model
+    entry["problem"] = problem
+    entry["fix"] = fix
+    entry["permanent"] = permanent
+
+    should_alert_now = permanent or entry["consecutive_failures"] >= TRANSIENT_ALERT_THRESHOLD
+
+    if should_alert_now and not entry["degraded"]:
+        entry["degraded"] = True
+
+    if should_alert_now and not entry["alerted"]:
+        entry["alerted"] = True
+        await _post_webhook(
+            f"**AI tier degraded: {tier}**\n"
+            f"Provider: {provider} ({model})\n"
+            f"Problem: {problem}\n"
+            f"Fix: {fix}\n"
+            f"Kind: {'permanent' if permanent else 'transient, persisted ' + str(entry['consecutive_failures']) + ' calls'}"
+        )
+
+
+async def report_healthy(tier: str) -> None:
+    """
+    Record a successful call for `tier`. Call this on every success, not
+    just after a failure -- it is what lets a degraded tier recover on its
+    own instead of staying red forever after one transient blip.
+    """
+    if tier not in _state:
+        _state[tier] = _default_state()
+    entry = _state[tier]
+
+    was_alerted = entry["alerted"]
+    was_degraded = entry["degraded"]
+    provider, model = entry["provider"], entry["model"]
+
+    _state[tier] = _default_state()
+    # Keep the last-known provider/model around for the recovery message
+    # and for a quick glance at snapshot() even when healthy.
+    _state[tier]["provider"] = provider
+    _state[tier]["model"] = model
+
+    if was_alerted:
+        await _post_webhook(f"**AI tier recovered: {tier}**\nProvider: {provider} ({model})")
+    elif was_degraded:
+        # Reached "degraded" internally but never crossed the alert
+        # threshold before recovering -- nothing was ever posted, so
+        # nothing needs un-posting. Nothing to do.
+        pass
+
+
+def snapshot() -> dict:
+    """Current state of every tier, for /health/ai."""
+    return {tier: dict(entry) for tier, entry in _state.items()}
+
+
+async def _post_webhook(content: str) -> None:
+    """
+    POST a message to the AI-alert Discord webhook, if one is configured.
+
+    Same httpx pattern as app/internal/alerter.py's _post_webhook: short
+    timeout, never raises. Self-hosted deployments that don't set
+    ai_alert_webhook_url just skip this silently.
+    """
+    url = settings.ai_alert_webhook_url
+    if not url:
+        return
+    try:
+        import httpx
+        async with httpx.AsyncClient(timeout=5.0) as client:
+            await client.post(url, json={"content": content})
+    except Exception as e:
+        logger.warning(f"ai_health: Discord webhook POST failed: {e}")

drb-c2-core/app/internal/llm_correlator.py

@@ -24,6 +24,7 @@ import json
 from datetime import datetime, timezone
 from typing import Optional
 from app.internal.logger import logger
+from app.internal import ai_health
 from app.config import settings
 
 
@@ -239,18 +240,21 @@ async def decide(call_id: str, ctx: dict) -> Optional[dict]:
             f"action={decision['action']} incident={_id} "
             f"reasoning={decision['reasoning']!r}"
         )
+        await ai_health.report_healthy("correlation_cheap")
         return decision
     except Exception as e:
-        _log_llm_failure("LLM correlator", call_id, settings.corr_cheap_model, e)
+        await _log_llm_failure("LLM correlator", "correlation_cheap", call_id, settings.corr_cheap_model, e)
         return None
 
 
 _dead_models: set[str] = set()
 
 
-def _log_llm_failure(where: str, call_id: str, model: str, exc: Exception) -> None:
+async def _log_llm_failure(where: str, tier: str, call_id: str, model: str, exc: Exception) -> None:
     """
-    Log an LLM failure, escalating a dead model ID to ERROR once per model.
+    Log an LLM failure, escalating a dead model ID to ERROR once per model,
+    and report it to the shared app.internal.ai_health registry either way
+    (which is what drives /health/ai and the Discord degradation alert).
 
     A per-call WARNING was the only signal that gemini-2.0-flash had been shut
     down, and since every failure falls back to the rules decision the pipeline
@@ -260,27 +264,32 @@ def _log_llm_failure(where: str, call_id: str, model: str, exc: Exception) -> No
     that will never fix itself, so it gets ERROR and says what to do.
     """
     text = str(exc)
-    low  = text.lower()
+    kind = ai_health.classify(text)
 
-    if "404" in text or "not found" in low or "no longer available" in low:
-        _log_tier_down(where, model, "model is unavailable",
-                       "Update CORR_CHEAP_MODEL/CORR_SMART_MODEL in config.py", text)
+    if kind == "dead_model":
+        await _log_tier_down(where, tier, model, "model is unavailable",
+                             "Update CORR_CHEAP_MODEL/CORR_SMART_MODEL in config.py", text)
         return
 
     # A depleted balance reads as 429, the same status as an ordinary rate limit,
     # but it is the opposite kind of problem: a rate limit clears on its own and a
-    # dead account never does. Matching on the billing wording keeps a burst of
-    # rate limits at WARNING while an empty account escalates like a bad model ID.
-    if "credits are depleted" in low or "prepayment" in low or "billing" in low:
-        _log_tier_down(where, model, "the Gemini account is out of credit",
-                       "Top up billing at https://ai.studio/projects", text)
+    # dead account never does. ai_health.classify() keeps a burst of rate limits
+    # at WARNING while an empty account escalates like a bad model ID.
+    if kind == "billing":
+        await _log_tier_down(where, tier, model, "the Gemini account is out of credit",
+                             "Top up billing at https://ai.studio/projects", text)
         return
 
     logger.warning(f"{where} failed for call {call_id}: {text}")
+    await ai_health.report_degraded(
+        tier, "gemini", model, "transient API error",
+        "no action needed unless this persists", permanent=False,
+    )
 
 
-def _log_tier_down(where: str, model: str, problem: str, fix: str, text: str) -> None:
+async def _log_tier_down(where: str, tier: str, model: str, problem: str, fix: str, text: str) -> None:
     """ERROR once per model, not once per call — this runs at radio-traffic volume."""
+    await ai_health.report_degraded(tier, "gemini", model, problem, fix, permanent=True)
     if model in _dead_models:
         return
     _dead_models.add(model)
@@ -306,9 +315,10 @@ async def tiebreak(rules_decision: dict, llm_decision: dict, ctx: dict) -> dict:
             f"action={decision['action']} incident={_id} "
             f"reasoning={decision['reasoning']!r}"
         )
+        await ai_health.report_healthy("correlation_smart")
         return decision
     except Exception as e:
-        _log_llm_failure("LLM tiebreak", call_id, settings.corr_smart_model, e)
+        await _log_llm_failure("LLM tiebreak", "correlation_smart", call_id, settings.corr_smart_model, e)
         return rules_decision
 
 

drb-c2-core/app/internal/transcription.py

@@ -11,6 +11,8 @@ import os
 from typing import Optional
 from app.internal.logger import logger
 from app.internal import firestore as fstore
+from app.internal import ai_health
+from app.config import settings
 
 # Whisper treats `prompt` as preceding transcript text, not instructions.
 # Writing it as actual radio speech primes the vocabulary toward P25 codes
@@ -93,12 +95,12 @@ def _is_degenerate(text: str, segments: list[dict]) -> bool:
     return False
 
 
-_billing_reported = False
-
-
-def _log_transcribe_failure(call_id: str, exc: Exception) -> None:
+async def _log_transcribe_failure(call_id: str, exc: Exception) -> None:
     """
-    Log a transcription failure, escalating an unpayable account to ERROR once.
+    Log a transcription failure, escalating a permanent condition to ERROR
+    once (via app.internal.ai_health, which also drives the /health/ai
+    endpoint and the Discord degradation alert) and reporting it to the
+    shared registry either way.
 
     Transcription failing returns None and the pipeline carries on by design, so
     a per-call WARNING is invisible: no transcript means no extraction, which
@@ -110,23 +112,41 @@ def _log_transcribe_failure(call_id: str, exc: Exception) -> None:
     The same failure mode already bit the Gemini correlator twice (a retired
     model ID, then a depleted balance), which is why this is worth the code.
     """
-    global _billing_reported
     text = str(exc)
-    low  = text.lower()
+    kind = ai_health.classify(text)
 
-    if ("insufficient_quota" in low or "billing" in low
-            or "credit" in low or "exceeded your current quota" in low):
-        if not _billing_reported:
-            _billing_reported = True
-            logger.error(
-                "Transcription: the OpenAI account cannot be billed -- EVERY call is "
-                "now stored with no transcript, so extraction, correlation and "
-                "incidents are all dead downstream. Top up at "
-                f"https://platform.openai.com/settings/organization/billing. API said: {text}"
-            )
+    if kind == "billing":
+        problem = "the OpenAI account cannot be billed"
+        fix = "top up at https://platform.openai.com/settings/organization/billing"
+        logger.error(
+            "Transcription: the OpenAI account cannot be billed -- EVERY call is "
+            "now stored with no transcript, so extraction, correlation and "
+            "incidents are all dead downstream. Top up at "
+            f"https://platform.openai.com/settings/organization/billing. API said: {text}"
+        )
+        await ai_health.report_degraded(
+            "transcription", "openai", settings.stt_model, problem, fix, permanent=True
+        )
+        return
+
+    if kind == "dead_model":
+        problem = "the STT model is unavailable"
+        fix = "update STT_MODEL in config.py"
+        logger.error(
+            f"Transcription: the configured model ({settings.stt_model!r}) is unavailable "
+            "-- EVERY call is now stored with no transcript, so extraction, correlation "
+            f"and incidents are all dead downstream. Update STT_MODEL in config.py. API said: {text}"
+        )
+        await ai_health.report_degraded(
+            "transcription", "openai", settings.stt_model, problem, fix, permanent=True
+        )
         return
 
     logger.warning(f"Transcription failed for call {call_id}: {text}")
+    await ai_health.report_degraded(
+        "transcription", "openai", settings.stt_model,
+        "transient API error", "no action needed unless this persists", permanent=False,
+    )
 
 
 async def transcribe_call(
@@ -150,9 +170,15 @@ async def transcribe_call(
             _sync_transcribe, gcs_uri, talkgroup_name
         )
     except Exception as e:
-        _log_transcribe_failure(call_id, e)
+        await _log_transcribe_failure(call_id, e)
         return None, []
 
+    # No exception means the provider call itself succeeded (this also
+    # covers transcripts discarded as degenerate/hallucinated output —
+    # that's a filtering decision, not a provider failure), so the
+    # transcription tier is healthy and any prior degradation clears.
+    await ai_health.report_healthy("transcription")
+
     if transcript:
         updates: dict = {"transcript": transcript}
         if segments:

drb-c2-core/app/main.py

@@ -8,6 +8,7 @@ from app.internal.node_sweeper import sweeper_loop
 from app.internal.summarizer import summarizer_loop
 from app.internal.vocabulary_learner import vocabulary_induction_loop
 from app.internal.recorrelation_sweep import recorrelation_loop
+from app.internal import ai_health
 from app.config import settings
 from app.internal.auth import (
     require_firebase_token,
@@ -120,3 +121,12 @@ app.include_router(media.router)
 @app.get("/health")
 async def health():
     return {"ok": True, "mqtt_connected": mqtt_handler.is_connected}
+
+
+# Deliberately unauthenticated, same as /health above: the CI deploy step
+# curls /health with no credentials, and this is diagnostic state (which AI
+# tier is degraded and why), not a secret — no API keys or tokens appear in
+# it. Keeping it auth-free means an external uptime check can watch it too.
+@app.get("/health/ai")
+async def health_ai():
+    return {"tiers": ai_health.snapshot()}