diff --git a/drb-c2-core/app/internal/llm_correlator.py b/drb-c2-core/app/internal/llm_correlator.py index 3f2862b..e51f85b 100644 --- a/drb-c2-core/app/internal/llm_correlator.py +++ b/drb-c2-core/app/internal/llm_correlator.py @@ -260,18 +260,36 @@ def _log_llm_failure(where: str, call_id: str, model: str, exc: Exception) -> No that will never fix itself, so it gets ERROR and says what to do. """ text = str(exc) - if "404" in text or "not found" in text.lower() or "no longer available" in text.lower(): - if model not in _dead_models: - _dead_models.add(model) - logger.error( - f"{where}: model {model!r} is unavailable -- the LLM correlation tier " - f"is DISABLED and every call is falling back to rules-only. Update " - f"CORR_CHEAP_MODEL/CORR_SMART_MODEL in config.py. Google said: {text}" - ) + low = text.lower() + + if "404" in text or "not found" in low or "no longer available" in low: + _log_tier_down(where, model, "model is unavailable", + "Update CORR_CHEAP_MODEL/CORR_SMART_MODEL in config.py", text) return + + # A depleted balance reads as 429, the same status as an ordinary rate limit, + # but it is the opposite kind of problem: a rate limit clears on its own and a + # dead account never does. Matching on the billing wording keeps a burst of + # rate limits at WARNING while an empty account escalates like a bad model ID. + if "credits are depleted" in low or "prepayment" in low or "billing" in low: + _log_tier_down(where, model, "the Gemini account is out of credit", + "Top up billing at https://ai.studio/projects", text) + return + logger.warning(f"{where} failed for call {call_id}: {text}") +def _log_tier_down(where: str, model: str, problem: str, fix: str, text: str) -> None: + """ERROR once per model, not once per call — this runs at radio-traffic volume.""" + if model in _dead_models: + return + _dead_models.add(model) + logger.error( + f"{where}: {problem} ({model!r}) -- the LLM correlation tier is DISABLED " + f"and every call is falling back to rules-only. {fix}. API said: {text}" + ) + + async def tiebreak(rules_decision: dict, llm_decision: dict, ctx: dict) -> dict: """ Run the smart tiebreaker (corr_smart_model) when rules and LLM disagree.