Add AI provider degradation registry and alerting (server-26#14)

Three AI dependency failures in one night (retired Gemini model IDs,
depleted Gemini balance, unpayable OpenAI account) each surfaced only
as a single ERROR log line that nobody was watching. Add
app/internal/ai_health.py, a shared in-memory registry that
transcription.py and llm_correlator.py report into on every call
(success and failure), distinguishing permanent conditions (dead
model, dead billing) which alert immediately from transient ones
(rate limits, network blips) which only alert after they persist.
Alerts POST once per degradation episode and once on recovery to an
optional Discord webhook (AI_ALERT_WEBHOOK_URL), reusing alerter.py's
httpx pattern. State is exposed unauthenticated at GET /health/ai
alongside the existing /health.

Closes logan/server-26#14

drb-c2-core/app/internal/transcription.py

@@ -11,6 +11,8 @@ import os
 from typing import Optional
 from app.internal.logger import logger
 from app.internal import firestore as fstore
+from app.internal import ai_health
+from app.config import settings
 
 # Whisper treats `prompt` as preceding transcript text, not instructions.
 # Writing it as actual radio speech primes the vocabulary toward P25 codes
@@ -93,12 +95,12 @@ def _is_degenerate(text: str, segments: list[dict]) -> bool:
     return False
 
 
-_billing_reported = False
-
-
-def _log_transcribe_failure(call_id: str, exc: Exception) -> None:
+async def _log_transcribe_failure(call_id: str, exc: Exception) -> None:
     """
-    Log a transcription failure, escalating an unpayable account to ERROR once.
+    Log a transcription failure, escalating a permanent condition to ERROR
+    once (via app.internal.ai_health, which also drives the /health/ai
+    endpoint and the Discord degradation alert) and reporting it to the
+    shared registry either way.
 
     Transcription failing returns None and the pipeline carries on by design, so
     a per-call WARNING is invisible: no transcript means no extraction, which
@@ -110,23 +112,41 @@ def _log_transcribe_failure(call_id: str, exc: Exception) -> None:
     The same failure mode already bit the Gemini correlator twice (a retired
     model ID, then a depleted balance), which is why this is worth the code.
     """
-    global _billing_reported
     text = str(exc)
-    low  = text.lower()
+    kind = ai_health.classify(text)
 
-    if ("insufficient_quota" in low or "billing" in low
-            or "credit" in low or "exceeded your current quota" in low):
-        if not _billing_reported:
-            _billing_reported = True
-            logger.error(
-                "Transcription: the OpenAI account cannot be billed -- EVERY call is "
-                "now stored with no transcript, so extraction, correlation and "
-                "incidents are all dead downstream. Top up at "
-                f"https://platform.openai.com/settings/organization/billing. API said: {text}"
-            )
+    if kind == "billing":
+        problem = "the OpenAI account cannot be billed"
+        fix = "top up at https://platform.openai.com/settings/organization/billing"
+        logger.error(
+            "Transcription: the OpenAI account cannot be billed -- EVERY call is "
+            "now stored with no transcript, so extraction, correlation and "
+            "incidents are all dead downstream. Top up at "
+            f"https://platform.openai.com/settings/organization/billing. API said: {text}"
+        )
+        await ai_health.report_degraded(
+            "transcription", "openai", settings.stt_model, problem, fix, permanent=True
+        )
+        return
+
+    if kind == "dead_model":
+        problem = "the STT model is unavailable"
+        fix = "update STT_MODEL in config.py"
+        logger.error(
+            f"Transcription: the configured model ({settings.stt_model!r}) is unavailable "
+            "-- EVERY call is now stored with no transcript, so extraction, correlation "
+            f"and incidents are all dead downstream. Update STT_MODEL in config.py. API said: {text}"
+        )
+        await ai_health.report_degraded(
+            "transcription", "openai", settings.stt_model, problem, fix, permanent=True
+        )
         return
 
     logger.warning(f"Transcription failed for call {call_id}: {text}")
+    await ai_health.report_degraded(
+        "transcription", "openai", settings.stt_model,
+        "transient API error", "no action needed unless this persists", permanent=False,
+    )
 
 
 async def transcribe_call(
@@ -150,9 +170,15 @@ async def transcribe_call(
             _sync_transcribe, gcs_uri, talkgroup_name
         )
     except Exception as e:
-        _log_transcribe_failure(call_id, e)
+        await _log_transcribe_failure(call_id, e)
         return None, []
 
+    # No exception means the provider call itself succeeded (this also
+    # covers transcripts discarded as degenerate/hallucinated output —
+    # that's a filtering decision, not a provider failure), so the
+    # transcription tier is healthy and any prior degradation clears.
+    await ai_health.report_healthy("transcription")
+
     if transcript:
         updates: dict = {"transcript": transcript}
         if segments: