Outage providers: licence-gated registry, ODIN adapter, /outages API, admin tab (Refs #184)

Part 1 of 2 for server-26#184 (map layer + incident cross-match follow in batch 3). - app/internal/outages/: same gate as the camera registry -- settings doc config/outage_providers, every provider OFF by default, unreadable settings fail closed, stubs never enable, gated licences need acknowledge_licence stored with who/when plus an audit_log write as a precondition, and the acknowledgement is bound to the licence status it was given for. Reuses cameras.base Licence/LicenceInfo/get_json/safe_error/require_parsed. - ODIN adapter (Licence.UNVERIFIED, gated): public county-level API verified first-hand against https://odin.ornl.gov/v3/api-docs -- /odi?format=JSON (counts, start, ETR), /odi/map?scale=COUNTY (county name, bbox, label point), /odi/status (utility freshness). Schema canary on all three. metersServed is utility-wide, so customers_served stays None. - Stubs: poweroutage_us (paid), coned_oru / nyseg_rge / central_hudson storm centers and eagle_i_live (agreement required). - GET /outages (signed-in) returns areas + per-provider credit/health and a status (ok / none_enabled / no_source / error) so an empty layer is never ambiguous. /admin/outage-providers GET + PUT, admin only, 409 with requires_acknowledgement like cameras. - Frontend: c2api getOutages/listOutageProviders/setOutageProvider, types, Admin -> Outage providers tab (deep link /admin#outages). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-05 02:38:51 -04:00
parent a75149b2eb
commit ebeabe342e
12 changed files with 4660 additions and 4 deletions
@@ -0,0 +1,305 @@
+"""Power-outage providers (server-26#184): registry, licence gate, admin routes, ODIN adapter.
+
+What is held here (mirrors tests/test_camera_providers.py):
+  * every provider defaults OFF, and an off provider is never fetched;
+  * stubs can never be enabled (409);
+  * a gated licence needs ``acknowledge_licence``, which is stored with who/when
+    and audited -- and an audit failure refuses the enable (fail closed);
+  * an acknowledgement given for a different licence status no longer counts;
+  * the admin routes are admin-only;
+  * the ODIN parser reads the recorded public payloads, and an all-unparseable
+    payload is an error, not an empty layer;
+  * an area no serving provider covers says so instead of returning a bare [].
+"""
+import asyncio
+import json
+from pathlib import Path
+from unittest.mock import AsyncMock, patch
+
+import pytest
+from fastapi import HTTPException
+from fastapi.testclient import TestClient
+
+from app.internal.auth import require_admin_token, require_service_or_firebase_token
+from app.internal.outages import base, providers, registry
+from app.internal.outages.base import Licence
+from app.main import app
+
+client = TestClient(app)
+ADMIN = {"uid": "admin-1", "email": "admin@example.com", "role": "admin"}
+FIXTURE = json.loads((Path(__file__).parent / "fixtures" / "odin_public_2026-10-05.json").read_text())
+STUBS = ("poweroutage_us", "coned_oru_stormcenter", "nyseg_rge_stormcenter", "central_hudson_stormcenter", "eagle_i_live")
+NY_BBOX = (42.5, -78.5, 43.5, -75.5)   # Rochester .. Syracuse
+
+
+class FakeStore:
+    def __init__(self):
+        self.docs = {}
+        self.audit = []
+
+    async def doc_get(self, col, doc):
+        return self.docs.get((col, doc))
+
+    async def doc_set(self, col, doc, data, merge=True):
+        cur = self.docs.setdefault((col, doc), {})
+        if col == "audit_log":
+            self.audit.append(data)
+            return
+        for k, v in data.items():   # one-level deep merge is all registry needs
+            if merge and isinstance(v, dict) and isinstance(cur.get(k), dict):
+                cur[k] = {**cur[k], **v}
+            else:
+                cur[k] = v
+
+
+@pytest.fixture(autouse=True)
+def store():
+    s = FakeStore()
+    registry.invalidate_cache()
+    for feed in registry.FEEDS.values():
+        feed.items, feed.fetched_at, feed.error, feed._next_attempt = [], None, None, 0.0
+    with patch.object(registry.fstore, "doc_get", s.doc_get), patch.object(registry.fstore, "doc_set", s.doc_set), \
+            patch("app.internal.audit.fstore.doc_set", s.doc_set):
+        yield s
+    registry.invalidate_cache()
+    app.dependency_overrides.clear()
+
+
+def run(coro):
+    return asyncio.run(coro)
+
+
+def _odin_areas():
+    return providers.parse_odin(FIXTURE["odi_format_json"], FIXTURE["odi_map_scale_county"], FIXTURE["odi_status"])[0]
+
+
+# ── registry / gate ──────────────────────────────────────────────────────────
+
+def test_every_catalogued_source_is_registered_and_defaults_off():
+    listed = run(registry.list_providers())
+    assert {p["id"] for p in listed} == {"odin", *STUBS}
+    assert all(p["enabled"] is False and p["serving"] is False for p in listed)
+    assert all(p["licence"]["label"] for p in listed)
+    odin = next(p for p in listed if p["id"] == "odin")
+    assert odin["licence"]["status"] == "unverified" and odin["licence"]["gated"] is True
+    assert odin["licence"]["terms_url"] == "https://odin.ornl.gov/v3/api-docs"
+    assert odin["not_serving_reason"] == "disabled (default)"
+
+
+def test_catalogued_licence_statuses_match_the_source_terms():
+    lic = {pid: p.meta.licence.status for pid, p in registry.PROVIDERS.items()}
+    assert lic["poweroutage_us"] == Licence.PAID
+    for pid in ("coned_oru_stormcenter", "nyseg_rge_stormcenter", "central_hudson_stormcenter", "eagle_i_live"):
+        assert lic[pid] == Licence.AGREEMENT_REQUIRED, pid
+    assert all(not registry.PROVIDERS[pid].meta.implemented for pid in STUBS)
+
+
+def test_warning_is_worded_for_outage_data():
+    w = registry.PROVIDERS["odin"].meta.licence.warning()
+    assert "outage data" in w and "imagery" not in w
+
+
+def test_off_provider_is_never_fetched():
+    fetch = AsyncMock(return_value=[])
+    with patch.object(registry.PROVIDERS["odin"], "fetch", fetch):
+        out = run(registry.outages_in_bbox(NY_BBOX))
+    assert out["areas"] == [] and out["providers"] == [] and fetch.await_count == 0
+    assert out["serving_count"] == 0 and out["status"] == "none_enabled"
+
+
+def test_stubs_cannot_be_enabled():
+    for pid in STUBS:
+        with pytest.raises(registry.ProviderError) as e:
+            run(registry.set_enabled(pid, True, ("u", "e"), acknowledge_licence=True))
+        assert e.value.code == 409, pid
+
+
+def test_gated_licence_requires_acknowledgement(store):
+    with pytest.raises(registry.ProviderError) as e:
+        run(registry.set_enabled("odin", True, ("u1", "a@x")))
+    assert e.value.code == 409 and e.value.extra["requires_acknowledgement"] is True
+    assert "Licence unverified" in e.value.extra["warning"]
+    assert not store.audit and not run(registry.is_serving("odin"))
+
+    out = run(registry.set_enabled("odin", True, ("u1", "a@x"), acknowledge_licence=True))
+    assert out["serving"] is True
+    ack = out["acknowledgement"]
+    assert ack["by_uid"] == "u1" and ack["by_email"] == "a@x" and ack["at"]
+    assert ack["licence_status"] == Licence.UNVERIFIED.value
+    assert store.audit[-1]["action"] == "outage_provider.enable_acknowledged"
+    assert store.audit[-1]["details"]["provider_id"] == "odin"
+    assert store.docs[("config", "outage_providers")]["providers"]["odin"]["enabled"] is True
+
+
+def test_gated_enable_refused_when_audit_cannot_be_written(store):
+    with patch.object(registry, "_audit", AsyncMock(side_effect=RuntimeError("firestore down"))):
+        with pytest.raises(registry.ProviderError) as e:
+            run(registry.set_enabled("odin", True, ("u", "e"), acknowledge_licence=True))
+    assert e.value.code == 503
+    assert not run(registry.is_serving("odin"))
+    assert ("config", "outage_providers") not in store.docs
+
+
+def test_disable_clears_acknowledgement_and_is_audited(store):
+    run(registry.set_enabled("odin", True, ("u", "e"), acknowledge_licence=True))
+    out = run(registry.set_enabled("odin", False, ("u", "e")))
+    assert out["serving"] is False and out["acknowledgement"] is None
+    assert store.audit[-1]["action"] == "outage_provider.disable"
+
+
+def test_stale_acknowledgement_stops_serving():
+    p = registry.PROVIDERS["odin"]
+    entry = {"enabled": True, "acknowledgement": {"licence_status": "paid"}}
+    assert registry.effective(p, entry) == (False, "licence acknowledgement missing or out of date")
+    entry["acknowledgement"]["licence_status"] = p.meta.licence.status.value
+    assert registry.effective(p, entry) == (True, None)
+
+
+def test_unreadable_settings_fail_closed():
+    with patch.object(registry.fstore, "doc_get", AsyncMock(side_effect=RuntimeError("boom"))):
+        registry.invalidate_cache()
+        assert run(registry.get_settings()) == {}
+        assert not run(registry.is_serving("odin"))
+
+
+# ── bbox query ───────────────────────────────────────────────────────────────
+
+def test_query_returns_areas_in_bbox_with_credit(store):
+    run(registry.set_enabled("odin", True, ("u", "e"), acknowledge_licence=True))
+    with patch.object(registry.PROVIDERS["odin"], "fetch", AsyncMock(return_value=_odin_areas())):
+        out = run(registry.outages_in_bbox(NY_BBOX))
+    assert out["status"] == "ok" and out["serving_count"] == 1
+    assert {a["fips"] for a in out["areas"]} == {"36055", "36067"}   # not Albany, not King County WA
+    prov = out["providers"][0]
+    assert prov["attribution"].startswith("Outage data: ODIN") and prov["disclaimer"]
+    assert prov["total_in_bbox"] == 2 and prov["fetched_at"] and prov["error"] is None
+    assert "Only utilities that report to ODIN appear" in prov["coverage_note"]
+
+
+def test_area_no_serving_provider_covers_says_so(store):
+    run(registry.set_enabled("odin", True, ("u", "e"), acknowledge_licence=True))
+    fetch = AsyncMock(return_value=_odin_areas())
+    with patch.object(registry.PROVIDERS["odin"], "fetch", fetch):
+        out = run(registry.outages_in_bbox((51.4, -0.2, 51.6, 0.1)))   # London
+    assert fetch.await_count == 0   # outside ODIN coverage -> never fetched
+    assert out["areas"] == [] and out["status"] == "no_source"
+    assert out["message"] == "No licensed outage source covers this area."
+
+
+def test_dead_feed_is_an_error_not_an_empty_layer(store):
+    run(registry.set_enabled("odin", True, ("u", "e"), acknowledge_licence=True))
+    with patch.object(registry.PROVIDERS["odin"], "fetch", AsyncMock(side_effect=ValueError("schema change?"))):
+        out = run(registry.outages_in_bbox(NY_BBOX))
+    assert out["status"] == "error" and "schema change" in out["providers"][0]["error"]
+
+
+def test_failed_refresh_keeps_last_good_feed():
+    feed = registry.FEEDS["odin"]
+    with patch.object(feed.provider, "fetch", AsyncMock(return_value=_odin_areas())):
+        run(feed.get())
+    feed._next_attempt = 0.0
+    with patch.object(feed.provider, "fetch", AsyncMock(side_effect=RuntimeError("down"))):
+        run(feed.get())
+    assert len(feed.items) == 4 and "down" in feed.error and feed.fetched_at
+
+
+# ── admin routes ─────────────────────────────────────────────────────────────
+
+def test_admin_routes_require_admin():
+    def deny():
+        raise HTTPException(403, "Admin access required")
+    app.dependency_overrides[require_admin_token] = deny
+    assert client.get("/admin/outage-providers").status_code == 403
+    assert client.put("/admin/outage-providers/odin", json={"enabled": True, "acknowledge_licence": True}).status_code == 403
+
+
+def test_admin_route_toggle_and_gate(store):
+    app.dependency_overrides[require_admin_token] = lambda: ADMIN
+    r = client.put("/admin/outage-providers/odin", json={"enabled": True})
+    assert r.status_code == 409 and r.json()["requires_acknowledgement"] is True
+    assert r.json()["licence"]["status"] == "unverified"
+    r = client.put("/admin/outage-providers/odin", json={"enabled": True, "acknowledge_licence": True})
+    assert r.status_code == 200 and r.json()["acknowledgement"]["by_email"] == "admin@example.com"
+    assert client.put("/admin/outage-providers/poweroutage_us",
+                      json={"enabled": True, "acknowledge_licence": True}).status_code == 409
+    assert client.put("/admin/outage-providers/nope", json={"enabled": True}).status_code == 404
+    listed = client.get("/admin/outage-providers").json()["providers"]
+    assert next(p for p in listed if p["id"] == "odin")["serving"] is True
+
+
+def test_outages_route_rejects_inverted_bbox_and_reports_status():
+    app.dependency_overrides[require_service_or_firebase_token] = lambda: {"uid": "v"}
+    assert client.get("/outages", params={"south": 2, "west": 0, "north": 1, "east": 1}).status_code == 400
+    r = client.get("/outages", params={"south": 40.9, "west": -74.0, "north": 41.4, "east": -73.4})
+    assert r.status_code == 200 and r.json()["areas"] == [] and r.json()["status"] == "none_enabled"
+
+
+def test_outages_route_requires_sign_in():
+    def deny():
+        raise HTTPException(401, "Not signed in")
+    app.dependency_overrides[require_service_or_firebase_token] = deny
+    assert client.get("/outages", params={"south": 40, "west": -74, "north": 41, "east": -73}).status_code == 401
+
+
+# ── ODIN adapter (recorded public payloads, 2026-10-05) ──────────────────────
+
+def test_odin_parser_reads_recorded_payload():
+    areas, info = providers.parse_odin(FIXTURE["odi_format_json"], FIXTURE["odi_map_scale_county"], FIXTURE["odi_status"])
+    by = {a["fips"]: a for a in areas}
+    assert set(by) == {"36001", "36067", "36055", "53033"}
+    onondaga = by["36067"]
+    assert onondaga["id"] == "odin:36067" and onondaga["provider"] == "odin"
+    assert onondaga["county"] == "Onondaga" and onondaga["state"] == "New York"
+    assert onondaga["customers_out"] == 11 and onondaga["customers_served"] is None
+    assert onondaga["etr"] == "2026-10-05T08:00:00Z" and onondaga["since"] == "2026-10-04T18:35:40Z"
+    assert onondaga["updated_at"].startswith("2026-10-05T06:20:47")
+    assert onondaga["utilities"] == [{"name": "NATIONAL GRID", "eia_id": "13573", "customers_out": 11}]
+    s, w, n, e = onondaga["bbox"]
+    assert s < onondaga["lat"] < n and w < onondaga["lon"] < e
+    assert 42.7 < onondaga["lat"] < 43.3 and -76.6 < onondaga["lon"] < -75.8
+    # King County WA: several outage events from two utilities, summed per county
+    king = by["53033"]
+    assert king["customers_out"] == sum(r["metersOut"] for r in FIXTURE["odi_map_scale_county"] if r["id"] == "53033")
+    assert len(king["utilities"]) == 2
+    assert info["reporting_utilities"] == len(FIXTURE["odi_status"]) and info["utilities_with_error"] == 1
+
+
+def test_odin_fetch_uses_the_three_public_endpoints():
+    calls = []
+
+    async def fake_get_json(url, params=None, headers=None):
+        calls.append((url, params))
+        return {"https://odin.ornl.gov/odi": FIXTURE["odi_format_json"],
+                "https://odin.ornl.gov/odi/map": FIXTURE["odi_map_scale_county"],
+                "https://odin.ornl.gov/odi/status": FIXTURE["odi_status"]}[url]
+
+    p = providers.OdinProvider()
+    with patch.object(providers, "get_json", fake_get_json):
+        areas = run(p.fetch())
+    assert len(areas) == 4
+    assert ("https://odin.ornl.gov/odi", {"format": "JSON"}) in calls
+    assert ("https://odin.ornl.gov/odi/map", {"scale": "COUNTY"}) in calls
+    assert p.reporting_utilities == len(FIXTURE["odi_status"])
+    assert "utilities reporting nationwide" in p.coverage_note()
+
+
+def test_odin_schema_canary_trips_on_unparseable_payload():
+    odi, cmap, status = FIXTURE["odi_format_json"], FIXTURE["odi_map_scale_county"], FIXTURE["odi_status"]
+    renamed_odi = {"outage": [{"county": "36067", "count": 11}]}   # fields renamed upstream
+    with pytest.raises(ValueError, match="schema"):
+        providers.parse_odin(renamed_odi, cmap, status)
+    with pytest.raises(ValueError, match="schema"):
+        providers.parse_odin(odi, [{"fips": "36067"}], status)
+    with pytest.raises(ValueError, match="schema"):
+        providers.parse_odin(odi, cmap, [{"utility": "x"}])
+    with pytest.raises(ValueError, match="schema"):
+        providers.parse_odin([], cmap, status)                   # wrapper changed
+    # A genuinely quiet day is NOT an error: nothing upstream, nothing parsed.
+    assert providers.parse_odin({"outage": []}, [], status)[0] == []
+
+
+def test_area_in_bbox_uses_county_extent():
+    a = base.make_area("odin", "36067", county="Onondaga", state="New York", customers_out=1,
+                       lat=43.0, lon=-76.2, bbox=(42.77, -76.5, 43.27, -75.9))
+    assert base.area_in_bbox(a, (43.2, -76.0, 43.5, -75.5))     # overlaps the county's corner only
+    assert not base.area_in_bbox(a, (44.0, -76.0, 44.5, -75.5))