diff --git a/drb-c2-core/app/config.py b/drb-c2-core/app/config.py index d4e0269..bfdd844 100644 --- a/drb-c2-core/app/config.py +++ b/drb-c2-core/app/config.py @@ -38,8 +38,14 @@ class Settings(BaseSettings): # Correlation consensus models # corr_cheap_model — first-pass LLM correlator (runs on every call) # corr_smart_model — tiebreaker (only fires when rules and cheap LLM disagree) - corr_cheap_model: str = "gemini-2.0-flash" - corr_smart_model: str = "gemini-1.5-pro" + # Both IDs below were retired by Google and returned 404 on every call from + # some point before 2026-08-18 until they were corrected. Because a failed + # LLM call falls back to the rules decision, nothing broke loudly -- the + # entire LLM tier and the consensus tiebreak were simply dead in production + # while correlation behaviour was being tuned against rules-only output. + # Verify against https://ai.google.dev/gemini-api/docs/models before changing. + corr_cheap_model: str = "gemini-3.6-flash" # was gemini-2.0-flash (shut down) + corr_smart_model: str = "gemini-2.5-pro" # was gemini-1.5-pro (shut down) summary_interval_minutes: int = 2 # how often the summary loop runs correlation_window_hours: int = 2 # slow/location path: max hours since last call embedding_similarity_threshold: float = 0.93 # slow-path: requires location corroboration diff --git a/drb-c2-core/app/internal/llm_correlator.py b/drb-c2-core/app/internal/llm_correlator.py index 9dfb6e4..3f2862b 100644 --- a/drb-c2-core/app/internal/llm_correlator.py +++ b/drb-c2-core/app/internal/llm_correlator.py @@ -241,10 +241,37 @@ async def decide(call_id: str, ctx: dict) -> Optional[dict]: ) return decision except Exception as e: - logger.warning(f"LLM correlator failed for call {call_id}: {e}") + _log_llm_failure("LLM correlator", call_id, settings.corr_cheap_model, e) return None +_dead_models: set[str] = set() + + +def _log_llm_failure(where: str, call_id: str, model: str, exc: Exception) -> None: + """ + Log an LLM failure, escalating a dead model ID to ERROR once per model. + + A per-call WARNING was the only signal that gemini-2.0-flash had been shut + down, and since every failure falls back to the rules decision the pipeline + kept running normally -- the LLM tier was dead for an unknown number of days + while correlation was being tuned against rules-only output. A transient API + error is genuinely a warning; a model that does not exist is a config bug + that will never fix itself, so it gets ERROR and says what to do. + """ + text = str(exc) + if "404" in text or "not found" in text.lower() or "no longer available" in text.lower(): + if model not in _dead_models: + _dead_models.add(model) + logger.error( + f"{where}: model {model!r} is unavailable -- the LLM correlation tier " + f"is DISABLED and every call is falling back to rules-only. Update " + f"CORR_CHEAP_MODEL/CORR_SMART_MODEL in config.py. Google said: {text}" + ) + return + logger.warning(f"{where} failed for call {call_id}: {text}") + + async def tiebreak(rules_decision: dict, llm_decision: dict, ctx: dict) -> dict: """ Run the smart tiebreaker (corr_smart_model) when rules and LLM disagree. @@ -263,7 +290,7 @@ async def tiebreak(rules_decision: dict, llm_decision: dict, ctx: dict) -> dict: ) return decision except Exception as e: - logger.warning(f"LLM tiebreak failed for call {call_id}: {e} — using rules decision") + _log_llm_failure("LLM tiebreak", call_id, settings.corr_smart_model, e) return rules_decision