From 4fab959ddbf6b670d0d9e8fddb098a7c0b088b5d Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Sun, 8 Nov 2026 01:27:16 -0500 Subject: [PATCH] AI providers: review follow-ups to 0060ccd Replay usage key gemini_usage -> llm_usage (backend + ReplayTab), re-price the correction/correlation estimates for gpt-4o-mini, no temperature for o-series or gpt-5 models, and refresh comments still naming Gemini. Co-Authored-By: Claude Opus 5.5 --- drb-c2-core/app/config.py | 10 +++++----- drb-c2-core/app/internal/llm.py | 6 +++++- drb-c2-core/app/internal/llm_correlator.py | 4 ++-- drb-c2-core/app/internal/recorrelation_sweep.py | 2 +- drb-c2-core/app/internal/replay.py | 6 +++--- drb-frontend/components/admin/ReplayTab.tsx | 4 ++-- drb-frontend/lib/types.ts | 2 +- 7 files changed, 19 insertions(+), 15 deletions(-) diff --git a/drb-c2-core/app/config.py b/drb-c2-core/app/config.py index 0525a32..63f81cc 100644 --- a/drb-c2-core/app/config.py +++ b/drb-c2-core/app/config.py @@ -122,17 +122,17 @@ class Settings(BaseSettings): # NWS alerts cache TTL; the issue asks 60-120 s (floored at 30 s in code). weather_alerts_ttl_s: int = 90 - # Gemini (intelligence extraction, embeddings, incident summaries) + # Gemini: only used if CORR_CHEAP_MODEL / TRANSCRIPT_CORRECTION_MODEL is set + # to a gemini-* ID (app/internal/llm.py). OpenAI serves both by default. gemini_api_key: Optional[str] = None # Correlation LLM (server-26#299). The rules engine decides every confident # case; this model is called at most ONCE per scene, and only when the rules # flag the decision as ambiguous (incident_correlator._ambiguity). There is # no second "smart" tier any more: the cheap/smart consensus tiebreak ran on # ~15% of scenes and resolved nearly every disagreement as "new" (#115). - # The model ID was once retired by Google and 404'd on every call for days; - # a failed LLM call falls back to the rules decision, so nothing broke - # loudly. Verify against https://ai.google.dev/gemini-api/docs/models - # before changing. + # A model ID was once retired by its provider and 404'd on every call for + # days; a failed LLM call falls back to the rules decision, so nothing broke + # loudly. Verify the ID against the provider's model list before changing. corr_cheap_model: str = "gpt-4o-mini" # OpenAI since 2026-11-08 (was gemini-3.6-flash); a gemini-* ID still works # Transcript correction (server-26#36). Runs inside transcription, once per # transcribed call above MIN_WORDS_FOR_CORRECTION, so it is priced like STT diff --git a/drb-c2-core/app/internal/llm.py b/drb-c2-core/app/internal/llm.py index c08e7b6..2490d47 100644 --- a/drb-c2-core/app/internal/llm.py +++ b/drb-c2-core/app/internal/llm.py @@ -79,11 +79,15 @@ def generate_json(model: str, prompt: str, *, purpose: str, from app.internal.ai_clients import openai_client # JSON mode requires the word "JSON" in the prompt; both prompts say # "Return JSON" / "Respond with JSON only". + kwargs = {} + # Reasoning models (o-series, gpt-5) reject a temperature; chat models get 0. + if not model.startswith(("o1", "o3", "o4", "gpt-5")): + kwargs["temperature"] = 0 resp = openai_client().chat.completions.create( model=model, messages=[{"role": "user", "content": prompt}], response_format={"type": "json_object"}, - temperature=0, + **kwargs, ) usage = getattr(resp, "usage", None) record_usage(purpose, model, diff --git a/drb-c2-core/app/internal/llm_correlator.py b/drb-c2-core/app/internal/llm_correlator.py index 1f0aa37..cc374c5 100644 --- a/drb-c2-core/app/internal/llm_correlator.py +++ b/drb-c2-core/app/internal/llm_correlator.py @@ -16,7 +16,7 @@ It returns a decision dict compatible with incident_correlator._apply_decision: "reasoning": str} or None — no API key, nothing to choose from, an answer that fails -validation, or any Gemini failure — and the caller then commits the rules +validation, or any LLM failure — and the caller then commits the rules decision unchanged, so the pipeline never stalls on the LLM. """ import asyncio @@ -196,7 +196,7 @@ def _sync_gemini(model_name: str, prompt: str) -> dict: def _parse_response(raw: dict, ctx: dict, candidates: list[dict]) -> Optional[dict]: """ - Convert raw Gemini JSON to a decision dict, or None when the answer is + Convert the model's raw JSON to a decision dict, or None when the answer is unusable and the rules decision should stand. A "link" must name one of the candidates the model was shown — not any diff --git a/drb-c2-core/app/internal/recorrelation_sweep.py b/drb-c2-core/app/internal/recorrelation_sweep.py index 94b9912..e93b9c9 100644 --- a/drb-c2-core/app/internal/recorrelation_sweep.py +++ b/drb-c2-core/app/internal/recorrelation_sweep.py @@ -84,7 +84,7 @@ def _derive_min_minutes() -> int: return max(_FLOOR_MINUTES, math.ceil(pipeline_worst_case_seconds() / 60)) -# 21 with the defaults (stt 120s, openai 60s x2 attempts, gemini 60s, 3 scenes). +# 22 with the defaults (stt 120s, openai 60s x2 attempts for every call, 3 scenes). MIN_MINUTES_SINCE_PIPELINE_START = _derive_min_minutes() diff --git a/drb-c2-core/app/internal/replay.py b/drb-c2-core/app/internal/replay.py index 6acde7a..de033fb 100644 --- a/drb-c2-core/app/internal/replay.py +++ b/drb-c2-core/app/internal/replay.py @@ -58,9 +58,9 @@ PREFETCH = 6 # Estimates, not a bill — nothing in DRB reads a real invoice (server-26#45). USD_WHISPER_PER_MIN = 0.006 USD_PER_EXTRACTION = 0.0005 # gpt-4o-mini scene extraction + embedding -USD_PER_CORRECTION = 0.0003 # Gemini flash transcript correction +USD_PER_CORRECTION = 0.0004 # gpt-4o-mini transcript correction (~1.5k in / 300 out tokens) USD_PER_GEOCODE = 0.005 # Google geocode, roughly one per located scene -USD_PER_LLM_CORRELATE = 0.0005 # one Gemini flash correlation decision (ambiguous scenes only) +USD_PER_LLM_CORRELATE = 0.0004 # one gpt-4o-mini correlation decision (~2k in / 100 out; ambiguous scenes only) _LLM_CONSENSUS = ("llm", "rules_guard", "agreed", "tiebreak", "llm_orphan_gate") # Fields the pipeline writes onto a call doc. Stripped when a call is copied @@ -569,7 +569,7 @@ async def _run(run_id: str, org_id: str, calls: list[dict], mode: str, metrics = compute_metrics(incidents, sb_calls) metrics["est_cost_usd"] = _running_cost(progress, metrics, mode) metrics["ai_failures"] = dict(Counter(f"{f['tier']}: {f['problem']}" for f in ai_failures)) - metrics["gemini_usage"] = usage + metrics["llm_usage"] = usage except Exception as e: status = "failed" errors.append(f"run: {type(e).__name__}: {e}"[:300]) diff --git a/drb-frontend/components/admin/ReplayTab.tsx b/drb-frontend/components/admin/ReplayTab.tsx index a94ba55..dcf038a 100644 --- a/drb-frontend/components/admin/ReplayTab.tsx +++ b/drb-frontend/components/admin/ReplayTab.tsx @@ -361,10 +361,10 @@ function RunDetail({ run }: { run: ReplayRun }) { paths: {Object.entries(m.corr_path).map(([k, v]) => `${k} ${v}`).join(" · ")}

)} - {m?.gemini_usage && Object.keys(m.gemini_usage).length > 0 && ( + {m?.llm_usage && Object.keys(m.llm_usage).length > 0 && (

Gemini tokens:{" "} - {Object.entries(m.gemini_usage) + {Object.entries(m.llm_usage) .map(([k, u]) => `${k} ${u.calls} calls, in ${u.in.toLocaleString()} / out ${u.out.toLocaleString()} / thinking ${u.thinking.toLocaleString()}`) .join(" · ")}

diff --git a/drb-frontend/lib/types.ts b/drb-frontend/lib/types.ts index a5ba7a5..630f486 100644 --- a/drb-frontend/lib/types.ts +++ b/drb-frontend/lib/types.ts @@ -669,7 +669,7 @@ export interface ReplayMetrics { llm_decisions: number; est_cost_usd: number; ai_failures?: Record; - gemini_usage?: Record; + llm_usage?: Record; } export interface ReplayRun {