diff --git a/drb-c2-core/app/internal/weather.py b/drb-c2-core/app/internal/weather.py index aa7aae5..3d2e012 100644 --- a/drb-c2-core/app/internal/weather.py +++ b/drb-c2-core/app/internal/weather.py @@ -48,6 +48,11 @@ ZONE_CONCURRENCY = 6 # the next refresh (the zone cache fills across refreshes); meanwhile the alert # shows in zone_only or with the zones resolved so far. MAX_ZONE_FETCHES_PER_REFRESH = 80 +# Wall-clock cap on one refresh's zone fetches. The refresh runs under the feed +# lock, so without it a slow zone endpoint (20 s timeout x 80 fetches / 6 wide) +# could hold every /weather/alerts request for minutes. Unfinished zones are +# simply not cached and are retried next refresh. +ZONE_BUDGET_S = 15.0 WIND_TTL_S = 5 * 60 WIND_CELL_DEG = 0.5 # wind cache cell; one upstream call per cell per TTL @@ -61,7 +66,7 @@ _warned_ua = False def _headers(accept: str) -> Dict[str, str]: global _warned_ua ua = settings.nws_user_agent or DEFAULT_USER_AGENT - if ua == DEFAULT_USER_AGENT and not _warned_ua: + if (ua == DEFAULT_USER_AGENT or "example.com" in ua) and not _warned_ua: _warned_ua = True logger.warning("NWS_USER_AGENT is the placeholder -- set it to 'app (contact email)'; NWS may block us") return {"User-Agent": ua, "Accept": accept} @@ -193,7 +198,12 @@ async def _resolve_zones(client: "httpx.AsyncClient", alerts: List[Dict[str, Any wanted.append(z) if wanted: sem = asyncio.Semaphore(ZONE_CONCURRENCY) - await asyncio.gather(*(_fetch_zone(client, z, sem) for z in wanted[:MAX_ZONE_FETCHES_PER_REFRESH])) + try: + await asyncio.wait_for( + asyncio.gather(*(_fetch_zone(client, z, sem) for z in wanted[:MAX_ZONE_FETCHES_PER_REFRESH])), + timeout=ZONE_BUDGET_S) + except asyncio.TimeoutError: + logger.warning(f"NWS zone fetches over {ZONE_BUDGET_S:.0f}s budget; the rest resolve next refresh") for a in alerts: if a["geometry"] is not None: continue diff --git a/drb-c2-core/tests/test_weather.py b/drb-c2-core/tests/test_weather.py index 478f634..a55f9c8 100644 --- a/drb-c2-core/tests/test_weather.py +++ b/drb-c2-core/tests/test_weather.py @@ -216,3 +216,24 @@ def test_wind_unavailable_reasons(): with patch.object(weather.httpx, "AsyncClient", FakeHttp({weather.METAR_URL: Resp(status=503)})): body = client.get("/weather/wind", params={"lat": 41.03, "lon": -73.76}).json() assert body["available"] is False and "unavailable" in body["reason"] and "503" in body["error"] + + +def test_slow_zone_endpoint_cannot_hold_the_feed_lock(): + """Zone fetches run under the feed lock; a hung zone endpoint must not stall + every /weather/alerts request -- the alert lands in zone_only instead.""" + import time + + class SlowZones(FakeHttp): + async def get(self, url, params=None, headers=None): + if url.startswith(weather.NWS_ZONE_PREFIX): + await asyncio.sleep(5) + return await super().get(url, params, headers) + + http = SlowZones({weather.NWS_ALERTS_URL: Resp({"features": [_alert("z1", "Flood Watch", None, [ZONE_A])]}), + ZONE_A: Resp({"geometry": ZONE_POLY})}) + with patch.object(weather, "ZONE_BUDGET_S", 0.2), patch.object(weather.httpx, "AsyncClient", http): + t0 = time.time() + feed = asyncio.run(weather.alerts.get()) + assert time.time() - t0 < 2 + assert feed.error is None and [z["id"] for z in feed.zone_only] == ["z1"] + assert weather.zones.get(ZONE_A) == (False, None) # not cached as a failure; retried next refresh diff --git a/infra/ansible/roles/deploy/templates/c2-core.env.j2 b/infra/ansible/roles/deploy/templates/c2-core.env.j2 index f1ba13f..8291115 100644 --- a/infra/ansible/roles/deploy/templates/c2-core.env.j2 +++ b/infra/ansible/roles/deploy/templates/c2-core.env.j2 @@ -44,3 +44,8 @@ AGENT_SERVICE_KEY={{ vault_agent_service_key | default('') }} # frontend call to the API would have failed CORS. If the frontend ever moves # to app.{{ domain }}, change this at the same time. CORS_ORIGINS=["https://{{ domain }}"] + +# NWS weather alerts / METAR wind (server-26#237). api.weather.gov requires a +# User-Agent naming the app and a contact; default to the site URL so a vault +# that predates this key still sends an identifiable agent, not the placeholder. +NWS_USER_AGENT={{ vault_nws_user_agent | default('DRB scanner map (https://' ~ domain ~ ')', true) }} diff --git a/infra/ansible/vault.yml.example b/infra/ansible/vault.yml.example index 15d7a00..eacd784 100644 --- a/infra/ansible/vault.yml.example +++ b/infra/ansible/vault.yml.example @@ -60,3 +60,5 @@ vault_firebase_app_id: "" # No GCP key needed — the VM uses Application Default Credentials via the # GCE metadata server. Terraform grants the required IAM roles at apply time. + +vault_nws_user_agent: "" # optional; e.g. "DRB scanner map (ops@yourdomain)" -- blank/absent uses the site URL