Merge pull request 'Outage providers: licence-gated registry, ODIN adapter, /outages API, admin tab (Refs #184)' (#255) from feat/184-outage-providers into main
This commit was merged in pull request #255.
This commit is contained in:
+3384
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,329 @@
|
||||
"""Power-outage providers (server-26#184): registry, licence gate, admin routes, ODIN adapter.
|
||||
|
||||
What is held here (mirrors tests/test_camera_providers.py):
|
||||
* every provider defaults OFF, and an off provider is never fetched;
|
||||
* stubs can never be enabled (409);
|
||||
* a gated licence needs ``acknowledge_licence``, which is stored with who/when
|
||||
and audited -- and an audit failure refuses the enable (fail closed);
|
||||
* an acknowledgement given for a different licence status no longer counts;
|
||||
* the admin routes are admin-only;
|
||||
* the ODIN parser reads the recorded public payloads, and an all-unparseable
|
||||
payload is an error, not an empty layer;
|
||||
* an area no serving provider covers says so instead of returning a bare [].
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.internal.auth import require_admin_token, require_service_or_firebase_token
|
||||
from app.internal.outages import base, providers, registry
|
||||
from app.internal.outages.base import Licence
|
||||
from app.main import app
|
||||
|
||||
client = TestClient(app)
|
||||
ADMIN = {"uid": "admin-1", "email": "admin@example.com", "role": "admin"}
|
||||
FIXTURE = json.loads((Path(__file__).parent / "fixtures" / "odin_public_2026-10-05.json").read_text())
|
||||
STUBS = ("poweroutage_us", "coned_oru_stormcenter", "nyseg_rge_stormcenter", "central_hudson_stormcenter", "eagle_i_live")
|
||||
NY_BBOX = (42.5, -78.5, 43.5, -75.5) # Rochester .. Syracuse
|
||||
RECORDED_AT = datetime(2026, 10, 5, 6, 45, tzinfo=timezone.utc) # when the fixture was captured
|
||||
|
||||
|
||||
class FakeStore:
|
||||
def __init__(self):
|
||||
self.docs = {}
|
||||
self.audit = []
|
||||
|
||||
async def doc_get(self, col, doc):
|
||||
return self.docs.get((col, doc))
|
||||
|
||||
async def doc_set(self, col, doc, data, merge=True):
|
||||
cur = self.docs.setdefault((col, doc), {})
|
||||
if col == "audit_log":
|
||||
self.audit.append(data)
|
||||
return
|
||||
for k, v in data.items(): # one-level deep merge is all registry needs
|
||||
if merge and isinstance(v, dict) and isinstance(cur.get(k), dict):
|
||||
cur[k] = {**cur[k], **v}
|
||||
else:
|
||||
cur[k] = v
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def store():
|
||||
s = FakeStore()
|
||||
registry.invalidate_cache()
|
||||
for feed in registry.FEEDS.values():
|
||||
feed.items, feed.fetched_at, feed.error, feed._next_attempt = [], None, None, 0.0
|
||||
with patch.object(registry.fstore, "doc_get", s.doc_get), patch.object(registry.fstore, "doc_set", s.doc_set), \
|
||||
patch("app.internal.audit.fstore.doc_set", s.doc_set):
|
||||
yield s
|
||||
registry.invalidate_cache()
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def run(coro):
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def _odin_areas():
|
||||
return providers.parse_odin(FIXTURE["odi_format_json"], FIXTURE["odi_map_scale_county"], FIXTURE["odi_status"],
|
||||
now=RECORDED_AT)[0]
|
||||
|
||||
|
||||
# ── registry / gate ──────────────────────────────────────────────────────────
|
||||
|
||||
def test_every_catalogued_source_is_registered_and_defaults_off():
|
||||
listed = run(registry.list_providers())
|
||||
assert {p["id"] for p in listed} == {"odin", *STUBS}
|
||||
assert all(p["enabled"] is False and p["serving"] is False for p in listed)
|
||||
assert all(p["licence"]["label"] for p in listed)
|
||||
odin = next(p for p in listed if p["id"] == "odin")
|
||||
assert odin["licence"]["status"] == "unverified" and odin["licence"]["gated"] is True
|
||||
assert odin["licence"]["terms_url"] == "https://odin.ornl.gov/v3/api-docs"
|
||||
assert odin["not_serving_reason"] == "disabled (default)"
|
||||
|
||||
|
||||
def test_catalogued_licence_statuses_match_the_source_terms():
|
||||
lic = {pid: p.meta.licence.status for pid, p in registry.PROVIDERS.items()}
|
||||
assert lic["poweroutage_us"] == Licence.PAID
|
||||
for pid in ("coned_oru_stormcenter", "nyseg_rge_stormcenter", "central_hudson_stormcenter", "eagle_i_live"):
|
||||
assert lic[pid] == Licence.AGREEMENT_REQUIRED, pid
|
||||
assert all(not registry.PROVIDERS[pid].meta.implemented for pid in STUBS)
|
||||
|
||||
|
||||
def test_warning_is_worded_for_outage_data():
|
||||
w = registry.PROVIDERS["odin"].meta.licence.warning()
|
||||
assert "outage data" in w and "imagery" not in w
|
||||
|
||||
|
||||
def test_off_provider_is_never_fetched():
|
||||
fetch = AsyncMock(return_value=[])
|
||||
with patch.object(registry.PROVIDERS["odin"], "fetch", fetch):
|
||||
out = run(registry.outages_in_bbox(NY_BBOX))
|
||||
assert out["areas"] == [] and out["providers"] == [] and fetch.await_count == 0
|
||||
assert out["serving_count"] == 0 and out["status"] == "none_enabled"
|
||||
|
||||
|
||||
def test_stubs_cannot_be_enabled():
|
||||
for pid in STUBS:
|
||||
with pytest.raises(registry.ProviderError) as e:
|
||||
run(registry.set_enabled(pid, True, ("u", "e"), acknowledge_licence=True))
|
||||
assert e.value.code == 409, pid
|
||||
|
||||
|
||||
def test_gated_licence_requires_acknowledgement(store):
|
||||
with pytest.raises(registry.ProviderError) as e:
|
||||
run(registry.set_enabled("odin", True, ("u1", "a@x")))
|
||||
assert e.value.code == 409 and e.value.extra["requires_acknowledgement"] is True
|
||||
assert "Licence unverified" in e.value.extra["warning"]
|
||||
assert not store.audit and not run(registry.is_serving("odin"))
|
||||
|
||||
out = run(registry.set_enabled("odin", True, ("u1", "a@x"), acknowledge_licence=True))
|
||||
assert out["serving"] is True
|
||||
ack = out["acknowledgement"]
|
||||
assert ack["by_uid"] == "u1" and ack["by_email"] == "a@x" and ack["at"]
|
||||
assert ack["licence_status"] == Licence.UNVERIFIED.value
|
||||
assert store.audit[-1]["action"] == "outage_provider.enable_acknowledged"
|
||||
assert store.audit[-1]["details"]["provider_id"] == "odin"
|
||||
assert store.docs[("config", "outage_providers")]["providers"]["odin"]["enabled"] is True
|
||||
|
||||
|
||||
def test_gated_enable_refused_when_audit_cannot_be_written(store):
|
||||
with patch.object(registry, "_audit", AsyncMock(side_effect=RuntimeError("firestore down"))):
|
||||
with pytest.raises(registry.ProviderError) as e:
|
||||
run(registry.set_enabled("odin", True, ("u", "e"), acknowledge_licence=True))
|
||||
assert e.value.code == 503
|
||||
assert not run(registry.is_serving("odin"))
|
||||
assert ("config", "outage_providers") not in store.docs
|
||||
|
||||
|
||||
def test_disable_clears_acknowledgement_and_is_audited(store):
|
||||
run(registry.set_enabled("odin", True, ("u", "e"), acknowledge_licence=True))
|
||||
out = run(registry.set_enabled("odin", False, ("u", "e")))
|
||||
assert out["serving"] is False and out["acknowledgement"] is None
|
||||
assert store.audit[-1]["action"] == "outage_provider.disable"
|
||||
|
||||
|
||||
def test_stale_acknowledgement_stops_serving():
|
||||
p = registry.PROVIDERS["odin"]
|
||||
entry = {"enabled": True, "acknowledgement": {"licence_status": "paid"}}
|
||||
assert registry.effective(p, entry) == (False, "licence acknowledgement missing or out of date")
|
||||
entry["acknowledgement"]["licence_status"] = p.meta.licence.status.value
|
||||
assert registry.effective(p, entry) == (True, None)
|
||||
|
||||
|
||||
def test_unreadable_settings_fail_closed():
|
||||
with patch.object(registry.fstore, "doc_get", AsyncMock(side_effect=RuntimeError("boom"))):
|
||||
registry.invalidate_cache()
|
||||
assert run(registry.get_settings()) == {}
|
||||
assert not run(registry.is_serving("odin"))
|
||||
|
||||
|
||||
# ── bbox query ───────────────────────────────────────────────────────────────
|
||||
|
||||
def test_query_returns_areas_in_bbox_with_credit(store):
|
||||
run(registry.set_enabled("odin", True, ("u", "e"), acknowledge_licence=True))
|
||||
with patch.object(registry.PROVIDERS["odin"], "fetch", AsyncMock(return_value=_odin_areas())):
|
||||
out = run(registry.outages_in_bbox(NY_BBOX))
|
||||
assert out["status"] == "ok" and out["serving_count"] == 1
|
||||
assert {a["fips"] for a in out["areas"]} == {"36055", "36067"} # not Albany, not King County WA
|
||||
prov = out["providers"][0]
|
||||
assert prov["attribution"].startswith("Outage data: ODIN") and prov["disclaimer"]
|
||||
assert prov["total_in_bbox"] == 2 and prov["fetched_at"] and prov["error"] is None
|
||||
assert "Only utilities that report to ODIN appear" in prov["coverage_note"]
|
||||
|
||||
|
||||
def test_area_no_serving_provider_covers_says_so(store):
|
||||
run(registry.set_enabled("odin", True, ("u", "e"), acknowledge_licence=True))
|
||||
fetch = AsyncMock(return_value=_odin_areas())
|
||||
with patch.object(registry.PROVIDERS["odin"], "fetch", fetch):
|
||||
out = run(registry.outages_in_bbox((51.4, -0.2, 51.6, 0.1))) # London
|
||||
assert fetch.await_count == 0 # outside ODIN coverage -> never fetched
|
||||
assert out["areas"] == [] and out["status"] == "no_source"
|
||||
assert out["message"] == "No licensed outage source covers this area."
|
||||
|
||||
|
||||
def test_dead_feed_is_an_error_not_an_empty_layer(store):
|
||||
run(registry.set_enabled("odin", True, ("u", "e"), acknowledge_licence=True))
|
||||
with patch.object(registry.PROVIDERS["odin"], "fetch", AsyncMock(side_effect=ValueError("schema change?"))):
|
||||
out = run(registry.outages_in_bbox(NY_BBOX))
|
||||
assert out["status"] == "error" and "schema change" in out["providers"][0]["error"]
|
||||
|
||||
|
||||
def test_failed_refresh_keeps_last_good_feed():
|
||||
feed = registry.FEEDS["odin"]
|
||||
with patch.object(feed.provider, "fetch", AsyncMock(return_value=_odin_areas())):
|
||||
run(feed.get())
|
||||
feed._next_attempt = 0.0
|
||||
with patch.object(feed.provider, "fetch", AsyncMock(side_effect=RuntimeError("down"))):
|
||||
run(feed.get())
|
||||
assert len(feed.items) == 4 and "down" in feed.error and feed.fetched_at
|
||||
|
||||
|
||||
# ── admin routes ─────────────────────────────────────────────────────────────
|
||||
|
||||
def test_admin_routes_require_admin():
|
||||
def deny():
|
||||
raise HTTPException(403, "Admin access required")
|
||||
app.dependency_overrides[require_admin_token] = deny
|
||||
assert client.get("/admin/outage-providers").status_code == 403
|
||||
assert client.put("/admin/outage-providers/odin", json={"enabled": True, "acknowledge_licence": True}).status_code == 403
|
||||
|
||||
|
||||
def test_admin_route_toggle_and_gate(store):
|
||||
app.dependency_overrides[require_admin_token] = lambda: ADMIN
|
||||
r = client.put("/admin/outage-providers/odin", json={"enabled": True})
|
||||
assert r.status_code == 409 and r.json()["requires_acknowledgement"] is True
|
||||
assert r.json()["licence"]["status"] == "unverified"
|
||||
r = client.put("/admin/outage-providers/odin", json={"enabled": True, "acknowledge_licence": True})
|
||||
assert r.status_code == 200 and r.json()["acknowledgement"]["by_email"] == "admin@example.com"
|
||||
assert client.put("/admin/outage-providers/poweroutage_us",
|
||||
json={"enabled": True, "acknowledge_licence": True}).status_code == 409
|
||||
assert client.put("/admin/outage-providers/nope", json={"enabled": True}).status_code == 404
|
||||
listed = client.get("/admin/outage-providers").json()["providers"]
|
||||
assert next(p for p in listed if p["id"] == "odin")["serving"] is True
|
||||
|
||||
|
||||
def test_outages_route_rejects_inverted_bbox_and_reports_status():
|
||||
app.dependency_overrides[require_service_or_firebase_token] = lambda: {"uid": "v"}
|
||||
assert client.get("/outages", params={"south": 2, "west": 0, "north": 1, "east": 1}).status_code == 400
|
||||
r = client.get("/outages", params={"south": 40.9, "west": -74.0, "north": 41.4, "east": -73.4})
|
||||
assert r.status_code == 200 and r.json()["areas"] == [] and r.json()["status"] == "none_enabled"
|
||||
|
||||
|
||||
def test_outages_route_requires_sign_in():
|
||||
def deny():
|
||||
raise HTTPException(401, "Not signed in")
|
||||
app.dependency_overrides[require_service_or_firebase_token] = deny
|
||||
assert client.get("/outages", params={"south": 40, "west": -74, "north": 41, "east": -73}).status_code == 401
|
||||
|
||||
|
||||
# ── ODIN adapter (recorded public payloads, 2026-10-05) ──────────────────────
|
||||
|
||||
def test_odin_parser_reads_recorded_payload():
|
||||
areas, info = providers.parse_odin(FIXTURE["odi_format_json"], FIXTURE["odi_map_scale_county"], FIXTURE["odi_status"],
|
||||
now=RECORDED_AT)
|
||||
by = {a["fips"]: a for a in areas}
|
||||
assert set(by) == {"36001", "36067", "36055", "53033"}
|
||||
onondaga = by["36067"]
|
||||
assert onondaga["id"] == "odin:36067" and onondaga["provider"] == "odin"
|
||||
assert onondaga["county"] == "Onondaga" and onondaga["state"] == "New York"
|
||||
assert onondaga["customers_out"] == 11 and onondaga["customers_served"] is None
|
||||
assert onondaga["etr"] == "2026-10-05T08:00:00Z" and onondaga["since"] == "2026-10-04T18:35:40Z"
|
||||
assert onondaga["updated_at"].startswith("2026-10-05T06:20:47")
|
||||
assert onondaga["utilities"] == [{"name": "NATIONAL GRID", "eia_id": "13573", "customers_out": 11,
|
||||
"stale_customers_out": 0}]
|
||||
assert onondaga["stale"] is False and onondaga["stale_customers_out"] == 0
|
||||
s, w, n, e = onondaga["bbox"]
|
||||
assert s < onondaga["lat"] < n and w < onondaga["lon"] < e
|
||||
assert 42.7 < onondaga["lat"] < 43.3 and -76.6 < onondaga["lon"] < -75.8
|
||||
# King County WA: several outage events from two utilities, summed per county
|
||||
king = by["53033"]
|
||||
assert king["customers_out"] == sum(r["metersOut"] for r in FIXTURE["odi_map_scale_county"] if r["id"] == "53033")
|
||||
assert len(king["utilities"]) == 2
|
||||
# Puget Sound Energy's "0001-01-01T00:00:00Z" means no ETR -- not a restoration time, not stale.
|
||||
assert king["etr"] is None and king["stale_customers_out"] == 0
|
||||
assert info["reporting_utilities"] == len(FIXTURE["odi_status"]) and info["utilities_with_error"] == 1
|
||||
|
||||
|
||||
def test_odin_events_long_past_their_etr_are_stale_not_live():
|
||||
# ODIN still listed Avangrid's Monroe County row from April 2026 (ETR
|
||||
# 2026-04-10) on 2026-10-05. It must not read as 18 customers out right now.
|
||||
by = {a["fips"]: a for a in _odin_areas()}
|
||||
monroe = by["36055"]
|
||||
assert monroe["customers_out"] == 0 and monroe["stale_customers_out"] == 18 and monroe["stale"] is True
|
||||
assert monroe["since"].startswith("2026-04-10") # still shown, so the stale row is explainable
|
||||
# Same payload, captured the day the outage was live: not stale.
|
||||
live = {a["fips"]: a for a in providers.parse_odin(
|
||||
FIXTURE["odi_format_json"], FIXTURE["odi_map_scale_county"], FIXTURE["odi_status"],
|
||||
now=datetime(2026, 4, 10, 20, 0, tzinfo=timezone.utc))[0]}
|
||||
assert live["36055"]["customers_out"] == 18 and live["36055"]["stale"] is False
|
||||
# Live counties sort ahead of stale ones.
|
||||
assert [a["stale"] for a in _odin_areas()] == sorted(a["stale"] for a in _odin_areas())
|
||||
|
||||
|
||||
def test_odin_fetch_uses_the_three_public_endpoints():
|
||||
calls = []
|
||||
|
||||
async def fake_get_json(url, params=None, headers=None):
|
||||
calls.append((url, params))
|
||||
return {"https://odin.ornl.gov/odi": FIXTURE["odi_format_json"],
|
||||
"https://odin.ornl.gov/odi/map": FIXTURE["odi_map_scale_county"],
|
||||
"https://odin.ornl.gov/odi/status": FIXTURE["odi_status"]}[url]
|
||||
|
||||
p = providers.OdinProvider()
|
||||
with patch.object(providers, "get_json", fake_get_json):
|
||||
areas = run(p.fetch())
|
||||
assert len(areas) == 4
|
||||
assert ("https://odin.ornl.gov/odi", {"format": "JSON"}) in calls
|
||||
assert ("https://odin.ornl.gov/odi/map", {"scale": "COUNTY"}) in calls
|
||||
assert p.reporting_utilities == len(FIXTURE["odi_status"])
|
||||
assert "utilities reporting nationwide" in p.coverage_note()
|
||||
|
||||
|
||||
def test_odin_schema_canary_trips_on_unparseable_payload():
|
||||
odi, cmap, status = FIXTURE["odi_format_json"], FIXTURE["odi_map_scale_county"], FIXTURE["odi_status"]
|
||||
renamed_odi = {"outage": [{"county": "36067", "count": 11}]} # fields renamed upstream
|
||||
with pytest.raises(ValueError, match="schema"):
|
||||
providers.parse_odin(renamed_odi, cmap, status)
|
||||
with pytest.raises(ValueError, match="schema"):
|
||||
providers.parse_odin(odi, [{"fips": "36067"}], status)
|
||||
with pytest.raises(ValueError, match="schema"):
|
||||
providers.parse_odin(odi, cmap, [{"utility": "x"}])
|
||||
with pytest.raises(ValueError, match="schema"):
|
||||
providers.parse_odin([], cmap, status) # wrapper changed
|
||||
# A genuinely quiet day is NOT an error: nothing upstream, nothing parsed.
|
||||
assert providers.parse_odin({"outage": []}, [], status)[0] == []
|
||||
|
||||
|
||||
def test_area_in_bbox_uses_county_extent():
|
||||
a = base.make_area("odin", "36067", county="Onondaga", state="New York", customers_out=1,
|
||||
lat=43.0, lon=-76.2, bbox=(42.77, -76.5, 43.27, -75.9))
|
||||
assert base.area_in_bbox(a, (43.2, -76.0, 43.5, -75.5)) # overlaps the county's corner only
|
||||
assert not base.area_in_bbox(a, (44.0, -76.0, 44.5, -75.5))
|
||||
Reference in New Issue
Block a user