From 74faa553960e4de7de2b4c782e536422120be899 Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Tue, 18 Aug 2026 20:28:15 -0400 Subject: [PATCH] Point correlation at models that still exist, and make a dead one loud Both Gemini model IDs had been retired by Google. Production logs show every correlation call 404ing -- "models/gemini-2.0-flash is no longer available" -- and gemini-1.5-pro is gone from the model list as well. Because a failed LLM call falls back to the rules decision by design, nothing surfaced: the pipeline kept producing incidents, so the LLM tier and the consensus tiebreak were dead in production for an unknown number of days while correlation was being tuned. Some of what recent tuning was reacting to was rules-only behaviour that was never meant to run alone. Cheap model becomes gemini-3.6-flash, which is the migration target named in Google's own 404. Smart model becomes gemini-2.5-pro, the only stable Pro-tier text model left; the tiebreak fires rarely and its value comes from being a different, stronger model than the first pass, so a second Flash was not worth the consensus it would give up. Model list checked against https://ai.google.dev/gemini-api/docs/models on 2026-08-18. The more important half is the logging. A per-call WARNING was the only signal, and it is indistinguishable from an ordinary API hiccup, so a permanent misconfiguration read as noise. Failures that look like a missing model (404, "not found", "no longer available") now log once per model at ERROR, name the config keys to change, and say plainly that correlation is running rules-only. Transient errors keep the old per-call WARNING. Once per model, not once per call, so the alert stays readable at radio traffic volume. Gemini is used nowhere else in c2-core -- extraction, embeddings and summaries all run on OpenAI -- so the blast radius was exactly the correlation LLM tier. 38 correlator tests still pass. No new environment variables: both model IDs are config.py defaults and are not templated into any .env, so CI deploys this without an ansible run. Co-Authored-By: Claude Opus 5 --- drb-c2-core/app/config.py | 10 +++++-- drb-c2-core/app/internal/llm_correlator.py | 31 ++++++++++++++++++++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/drb-c2-core/app/config.py b/drb-c2-core/app/config.py index d4e0269..bfdd844 100644 --- a/drb-c2-core/app/config.py +++ b/drb-c2-core/app/config.py @@ -38,8 +38,14 @@ class Settings(BaseSettings): # Correlation consensus models # corr_cheap_model — first-pass LLM correlator (runs on every call) # corr_smart_model — tiebreaker (only fires when rules and cheap LLM disagree) - corr_cheap_model: str = "gemini-2.0-flash" - corr_smart_model: str = "gemini-1.5-pro" + # Both IDs below were retired by Google and returned 404 on every call from + # some point before 2026-08-18 until they were corrected. Because a failed + # LLM call falls back to the rules decision, nothing broke loudly -- the + # entire LLM tier and the consensus tiebreak were simply dead in production + # while correlation behaviour was being tuned against rules-only output. + # Verify against https://ai.google.dev/gemini-api/docs/models before changing. + corr_cheap_model: str = "gemini-3.6-flash" # was gemini-2.0-flash (shut down) + corr_smart_model: str = "gemini-2.5-pro" # was gemini-1.5-pro (shut down) summary_interval_minutes: int = 2 # how often the summary loop runs correlation_window_hours: int = 2 # slow/location path: max hours since last call embedding_similarity_threshold: float = 0.93 # slow-path: requires location corroboration diff --git a/drb-c2-core/app/internal/llm_correlator.py b/drb-c2-core/app/internal/llm_correlator.py index 9dfb6e4..3f2862b 100644 --- a/drb-c2-core/app/internal/llm_correlator.py +++ b/drb-c2-core/app/internal/llm_correlator.py @@ -241,10 +241,37 @@ async def decide(call_id: str, ctx: dict) -> Optional[dict]: ) return decision except Exception as e: - logger.warning(f"LLM correlator failed for call {call_id}: {e}") + _log_llm_failure("LLM correlator", call_id, settings.corr_cheap_model, e) return None +_dead_models: set[str] = set() + + +def _log_llm_failure(where: str, call_id: str, model: str, exc: Exception) -> None: + """ + Log an LLM failure, escalating a dead model ID to ERROR once per model. + + A per-call WARNING was the only signal that gemini-2.0-flash had been shut + down, and since every failure falls back to the rules decision the pipeline + kept running normally -- the LLM tier was dead for an unknown number of days + while correlation was being tuned against rules-only output. A transient API + error is genuinely a warning; a model that does not exist is a config bug + that will never fix itself, so it gets ERROR and says what to do. + """ + text = str(exc) + if "404" in text or "not found" in text.lower() or "no longer available" in text.lower(): + if model not in _dead_models: + _dead_models.add(model) + logger.error( + f"{where}: model {model!r} is unavailable -- the LLM correlation tier " + f"is DISABLED and every call is falling back to rules-only. Update " + f"CORR_CHEAP_MODEL/CORR_SMART_MODEL in config.py. Google said: {text}" + ) + return + logger.warning(f"{where} failed for call {call_id}: {text}") + + async def tiebreak(rules_decision: dict, llm_decision: dict, ctx: dict) -> dict: """ Run the smart tiebreaker (corr_smart_model) when rules and LLM disagree. @@ -263,7 +290,7 @@ async def tiebreak(rules_decision: dict, llm_decision: dict, ctx: dict) -> dict: ) return decision except Exception as e: - logger.warning(f"LLM tiebreak failed for call {call_id}: {e} — using rules decision") + _log_llm_failure("LLM tiebreak", call_id, settings.corr_smart_model, e) return rules_decision