AI providers: review follow-ups to 0060ccd
Build & Deploy / Build & push images (push) Successful in 4m59s
Build & Deploy / Deploy Firestore rules & indexes (push) Successful in 27s
Build & Deploy / Deploy to VM (push) Successful in 2m48s
Build & Deploy / Report a failed deploy (push) Skipped

Replay usage key gemini_usage -> llm_usage (backend + ReplayTab), re-price the
correction/correlation estimates for gpt-4o-mini, no temperature for o-series
or gpt-5 models, and refresh comments still naming Gemini.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Logan Cusano
2026-11-08 01:27:16 -05:00
co-authored by Claude Opus 5.5
parent 0060ccde75
commit 4fab959ddb
7 changed files with 19 additions and 15 deletions
+5 -5
View File
@@ -122,17 +122,17 @@ class Settings(BaseSettings):
# NWS alerts cache TTL; the issue asks 60-120 s (floored at 30 s in code). # NWS alerts cache TTL; the issue asks 60-120 s (floored at 30 s in code).
weather_alerts_ttl_s: int = 90 weather_alerts_ttl_s: int = 90
# Gemini (intelligence extraction, embeddings, incident summaries) # Gemini: only used if CORR_CHEAP_MODEL / TRANSCRIPT_CORRECTION_MODEL is set
# to a gemini-* ID (app/internal/llm.py). OpenAI serves both by default.
gemini_api_key: Optional[str] = None gemini_api_key: Optional[str] = None
# Correlation LLM (server-26#299). The rules engine decides every confident # Correlation LLM (server-26#299). The rules engine decides every confident
# case; this model is called at most ONCE per scene, and only when the rules # case; this model is called at most ONCE per scene, and only when the rules
# flag the decision as ambiguous (incident_correlator._ambiguity). There is # flag the decision as ambiguous (incident_correlator._ambiguity). There is
# no second "smart" tier any more: the cheap/smart consensus tiebreak ran on # no second "smart" tier any more: the cheap/smart consensus tiebreak ran on
# ~15% of scenes and resolved nearly every disagreement as "new" (#115). # ~15% of scenes and resolved nearly every disagreement as "new" (#115).
# The model ID was once retired by Google and 404'd on every call for days; # A model ID was once retired by its provider and 404'd on every call for
# a failed LLM call falls back to the rules decision, so nothing broke # days; a failed LLM call falls back to the rules decision, so nothing broke
# loudly. Verify against https://ai.google.dev/gemini-api/docs/models # loudly. Verify the ID against the provider's model list before changing.
# before changing.
corr_cheap_model: str = "gpt-4o-mini" # OpenAI since 2026-11-08 (was gemini-3.6-flash); a gemini-* ID still works corr_cheap_model: str = "gpt-4o-mini" # OpenAI since 2026-11-08 (was gemini-3.6-flash); a gemini-* ID still works
# Transcript correction (server-26#36). Runs inside transcription, once per # Transcript correction (server-26#36). Runs inside transcription, once per
# transcribed call above MIN_WORDS_FOR_CORRECTION, so it is priced like STT # transcribed call above MIN_WORDS_FOR_CORRECTION, so it is priced like STT
+5 -1
View File
@@ -79,11 +79,15 @@ def generate_json(model: str, prompt: str, *, purpose: str,
from app.internal.ai_clients import openai_client from app.internal.ai_clients import openai_client
# JSON mode requires the word "JSON" in the prompt; both prompts say # JSON mode requires the word "JSON" in the prompt; both prompts say
# "Return JSON" / "Respond with JSON only". # "Return JSON" / "Respond with JSON only".
kwargs = {}
# Reasoning models (o-series, gpt-5) reject a temperature; chat models get 0.
if not model.startswith(("o1", "o3", "o4", "gpt-5")):
kwargs["temperature"] = 0
resp = openai_client().chat.completions.create( resp = openai_client().chat.completions.create(
model=model, model=model,
messages=[{"role": "user", "content": prompt}], messages=[{"role": "user", "content": prompt}],
response_format={"type": "json_object"}, response_format={"type": "json_object"},
temperature=0, **kwargs,
) )
usage = getattr(resp, "usage", None) usage = getattr(resp, "usage", None)
record_usage(purpose, model, record_usage(purpose, model,
+2 -2
View File
@@ -16,7 +16,7 @@ It returns a decision dict compatible with incident_correlator._apply_decision:
"reasoning": str} "reasoning": str}
or None — no API key, nothing to choose from, an answer that fails or None — no API key, nothing to choose from, an answer that fails
validation, or any Gemini failure — and the caller then commits the rules validation, or any LLM failure — and the caller then commits the rules
decision unchanged, so the pipeline never stalls on the LLM. decision unchanged, so the pipeline never stalls on the LLM.
""" """
import asyncio import asyncio
@@ -196,7 +196,7 @@ def _sync_gemini(model_name: str, prompt: str) -> dict:
def _parse_response(raw: dict, ctx: dict, candidates: list[dict]) -> Optional[dict]: def _parse_response(raw: dict, ctx: dict, candidates: list[dict]) -> Optional[dict]:
""" """
Convert raw Gemini JSON to a decision dict, or None when the answer is Convert the model's raw JSON to a decision dict, or None when the answer is
unusable and the rules decision should stand. unusable and the rules decision should stand.
A "link" must name one of the candidates the model was shown — not any A "link" must name one of the candidates the model was shown — not any
@@ -84,7 +84,7 @@ def _derive_min_minutes() -> int:
return max(_FLOOR_MINUTES, math.ceil(pipeline_worst_case_seconds() / 60)) return max(_FLOOR_MINUTES, math.ceil(pipeline_worst_case_seconds() / 60))
# 21 with the defaults (stt 120s, openai 60s x2 attempts, gemini 60s, 3 scenes). # 22 with the defaults (stt 120s, openai 60s x2 attempts for every call, 3 scenes).
MIN_MINUTES_SINCE_PIPELINE_START = _derive_min_minutes() MIN_MINUTES_SINCE_PIPELINE_START = _derive_min_minutes()
+3 -3
View File
@@ -58,9 +58,9 @@ PREFETCH = 6
# Estimates, not a bill — nothing in DRB reads a real invoice (server-26#45). # Estimates, not a bill — nothing in DRB reads a real invoice (server-26#45).
USD_WHISPER_PER_MIN = 0.006 USD_WHISPER_PER_MIN = 0.006
USD_PER_EXTRACTION = 0.0005 # gpt-4o-mini scene extraction + embedding USD_PER_EXTRACTION = 0.0005 # gpt-4o-mini scene extraction + embedding
USD_PER_CORRECTION = 0.0003 # Gemini flash transcript correction USD_PER_CORRECTION = 0.0004 # gpt-4o-mini transcript correction (~1.5k in / 300 out tokens)
USD_PER_GEOCODE = 0.005 # Google geocode, roughly one per located scene USD_PER_GEOCODE = 0.005 # Google geocode, roughly one per located scene
USD_PER_LLM_CORRELATE = 0.0005 # one Gemini flash correlation decision (ambiguous scenes only) USD_PER_LLM_CORRELATE = 0.0004 # one gpt-4o-mini correlation decision (~2k in / 100 out; ambiguous scenes only)
_LLM_CONSENSUS = ("llm", "rules_guard", "agreed", "tiebreak", "llm_orphan_gate") _LLM_CONSENSUS = ("llm", "rules_guard", "agreed", "tiebreak", "llm_orphan_gate")
# Fields the pipeline writes onto a call doc. Stripped when a call is copied # Fields the pipeline writes onto a call doc. Stripped when a call is copied
@@ -569,7 +569,7 @@ async def _run(run_id: str, org_id: str, calls: list[dict], mode: str,
metrics = compute_metrics(incidents, sb_calls) metrics = compute_metrics(incidents, sb_calls)
metrics["est_cost_usd"] = _running_cost(progress, metrics, mode) metrics["est_cost_usd"] = _running_cost(progress, metrics, mode)
metrics["ai_failures"] = dict(Counter(f"{f['tier']}: {f['problem']}" for f in ai_failures)) metrics["ai_failures"] = dict(Counter(f"{f['tier']}: {f['problem']}" for f in ai_failures))
metrics["gemini_usage"] = usage metrics["llm_usage"] = usage
except Exception as e: except Exception as e:
status = "failed" status = "failed"
errors.append(f"run: {type(e).__name__}: {e}"[:300]) errors.append(f"run: {type(e).__name__}: {e}"[:300])
+2 -2
View File
@@ -361,10 +361,10 @@ function RunDetail({ run }: { run: ReplayRun }) {
paths: {Object.entries(m.corr_path).map(([k, v]) => `${k} ${v}`).join(" · ")} paths: {Object.entries(m.corr_path).map(([k, v]) => `${k} ${v}`).join(" · ")}
</p> </p>
)} )}
{m?.gemini_usage && Object.keys(m.gemini_usage).length > 0 && ( {m?.llm_usage && Object.keys(m.llm_usage).length > 0 && (
<p className="text-xs font-mono text-ink-muted"> <p className="text-xs font-mono text-ink-muted">
Gemini tokens:{" "} Gemini tokens:{" "}
{Object.entries(m.gemini_usage) {Object.entries(m.llm_usage)
.map(([k, u]) => `${k} ${u.calls} calls, in ${u.in.toLocaleString()} / out ${u.out.toLocaleString()} / thinking ${u.thinking.toLocaleString()}`) .map(([k, u]) => `${k} ${u.calls} calls, in ${u.in.toLocaleString()} / out ${u.out.toLocaleString()} / thinking ${u.thinking.toLocaleString()}`)
.join(" · ")} .join(" · ")}
</p> </p>
+1 -1
View File
@@ -669,7 +669,7 @@ export interface ReplayMetrics {
llm_decisions: number; llm_decisions: number;
est_cost_usd: number; est_cost_usd: number;
ai_failures?: Record<string, number>; ai_failures?: Record<string, number>;
gemini_usage?: Record<string, { calls: number; in: number; out: number; thinking: number }>; llm_usage?: Record<string, { calls: number; in: number; out: number; thinking: number }>;
} }
export interface ReplayRun { export interface ReplayRun {