Outage providers: licence-gated registry, ODIN adapter, /outages API, admin tab (Refs #184) #255

Merged
logan merged 2 commits from feat/184-outage-providers into main 2026-10-05 02:47:16 -04:00
12 changed files with 4715 additions and 4 deletions
@@ -0,0 +1,8 @@
"""Pluggable power-outage providers (map layer, server-26#184).
Same shape as ``app.internal.cameras``: every source is one adapter carrying
its catalogued licence status, every provider defaults OFF, and a licence-gated
one serves only after an admin acknowledgement recorded in audit_log
(``registry``). Scraped utility storm-center JSON is never shipped -- those
sources are stubs until a data-sharing agreement exists.
"""
+201
View File
@@ -0,0 +1,201 @@
"""Outage provider interface, normalised record and the per-provider feed cache.
The licence model (``Licence``, ``LicenceInfo``, the GATED set) is the camera
layer's, imported -- one catalogue vocabulary for every licence-gated source.
Only the acknowledgement wording differs (outage data, not imagery), so
``OutageLicenceInfo`` overrides ``warning()`` and nothing else.
Outage feeds are public and identical for every org, so -- like the camera
catalogs -- they are cached in memory, fetched lazily on request, and never
written to Firestore. A failed refresh keeps serving the last good feed and
reports the error and its age: a dead feed must show as an error, never as an
empty layer (the AI-silent-failures lesson).
"""
import asyncio
import time
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Tuple
from app.internal.cameras.base import ( # noqa: F401 (re-exported for adapters)
BBox, Licence, LicenceInfo, bbox_intersects, finite, get_json, require_parsed, safe_error,
)
from app.internal.logger import logger
RETRY_AFTER_FAILURE_S = 60
DEFAULT_FEED_TTL_S = 5 * 60 # ODIN asks utilities for updates every 5-15 min
USER_AGENT = "DRB-c2-core/1.0 (+public power-outage data; contact via odin.ornl.gov)"
OutageArea = Dict[str, Any]
@dataclass(frozen=True)
class OutageLicenceInfo(LicenceInfo):
"""``LicenceInfo`` with the acknowledgement worded for outage data."""
def warning(self) -> Optional[str]:
if self.status == Licence.PROHIBITED:
return f"{self.label}. This source's terms forbid this use; it cannot be enabled."
if not self.gated:
return None
return (f"{self.label}. {self.terms} Enabling it means DRB shows this source's outage data "
f"to every signed-in user. Only confirm if DRB actually holds the right to do so.")
@dataclass(frozen=True)
class OutageProviderMeta:
id: str
name: str
region: str
coverage: BBox # rough bbox; a query outside it never triggers a fetch
licence: LicenceInfo
granularity: str # "county", "point/polygon", ...
implemented: bool
notes: str # what is verified vs unverified about this adapter
homepage: Optional[str] = None
ttl_s: int = DEFAULT_FEED_TTL_S
class OutageProvider:
"""Base adapter. Subclasses set ``meta`` and implement ``fetch``."""
meta: OutageProviderMeta
async def fetch(self) -> List[OutageArea]:
raise NotImplementedError(f"{self.meta.id}: adapter is a stub ({self.meta.notes})")
def coverage_note(self) -> Optional[str]:
"""What this source can and cannot see, shown next to its data."""
return None
class StubProvider(OutageProvider):
"""A catalogued source with no adapter -- listed so Admin shows its licence
status, but it can never be enabled."""
def __init__(self, meta: OutageProviderMeta):
assert not meta.implemented
self.meta = meta
class Feed:
"""In-memory cache of one provider's normalised outage areas."""
def __init__(self, provider: OutageProvider):
self.provider = provider
self.items: List[OutageArea] = []
self.fetched_at: Optional[float] = None # epoch s of last SUCCESSFUL fetch
self.last_attempt_at: Optional[float] = None
self.error: Optional[str] = None
self._next_attempt = 0.0
self._lock = asyncio.Lock()
async def get(self) -> "Feed":
if time.time() < self._next_attempt:
return self
async with self._lock:
if time.time() < self._next_attempt:
return self
ttl = self.provider.meta.ttl_s
self.last_attempt_at = time.time()
try:
items = await self.provider.fetch()
self.items = items
self.fetched_at = time.time()
self.error = None
self._next_attempt = time.time() + ttl
except Exception as e:
self._next_attempt = time.time() + min(ttl, RETRY_AFTER_FAILURE_S)
self.error = safe_error(e)
logger.warning(f"outages/{self.provider.meta.id}: refresh failed, serving {len(self.items)} cached: {self.error}")
return self
def status(self) -> dict:
return {"fetched_at": self.fetched_at, "last_attempt_at": self.last_attempt_at,
"error": self.error, "count": len(self.items)}
# ── helpers shared by adapters ───────────────────────────────────────────────
def make_area(provider_id: str, fips: str, *, county: Optional[str], state: Optional[str], customers_out: int,
customers_served: Optional[int] = None, etr: Optional[str] = None, since: Optional[str] = None,
updated_at: Optional[str] = None, lat: Optional[float] = None, lon: Optional[float] = None,
bbox: Optional[BBox] = None, utilities: Optional[List[dict]] = None,
stale_customers_out: int = 0) -> OutageArea:
"""One normalised county outage record. ``id`` is provider-prefixed.
``customers_out`` counts only live events. ``stale_customers_out`` counts
events the source still lists whose own restoration estimate passed long
ago (ODIN carries April rows in October) -- kept visible, never summed in.
``stale`` is True when every event in the county is stale."""
return {
"id": f"{provider_id}:{fips}",
"provider": provider_id,
"fips": fips, "county": county, "state": state,
"customers_out": customers_out,
"stale_customers_out": stale_customers_out,
"stale": customers_out == 0 and stale_customers_out > 0,
"customers_served": customers_served,
"etr": etr, # latest estimated restoration across the county's outages, if any
"since": since, # earliest reported start across the county's outages
"updated_at": updated_at, # newest time a reporting utility in this county last sent data
# Approximate: the centre of the county's bounding box, for placing a label.
"lat": lat, "lon": lon,
"bbox": list(bbox) if bbox else None, # south, west, north, east
"utilities": utilities or [],
}
def area_in_bbox(area: OutageArea, bbox: BBox) -> bool:
"""County bbox intersects the query; falls back to the label point."""
b = area.get("bbox")
if b and len(b) == 4:
return bbox_intersects((b[0], b[1], b[2], b[3]), bbox)
lat, lon = area.get("lat"), area.get("lon")
if lat is None or lon is None:
return False
s, w, n, e = bbox
return s <= lat <= n and w <= lon <= e
def parse_iso(v: Any) -> Optional[datetime]:
"""ISO-8601 (with Z or offset) -> aware UTC datetime; None when unparseable."""
if not isinstance(v, str) or not v:
return None
try:
d = datetime.fromisoformat(v.replace("Z", "+00:00"))
except ValueError:
return None
return (d if d.tzinfo else d.replace(tzinfo=timezone.utc)).astimezone(timezone.utc)
def iso(d: Optional[datetime]) -> Optional[str]:
return d.isoformat().replace("+00:00", "Z") if d else None
def geometry_bbox(geom: Any) -> Optional[BBox]:
"""Bounding box (south, west, north, east) of a GeoJSON Polygon/MultiPolygon."""
if not isinstance(geom, dict):
return None
lats: List[float] = []
lons: List[float] = []
def walk(c: Any) -> None:
if isinstance(c, list) and len(c) >= 2 and not isinstance(c[0], list):
lon, lat = finite(c[0]), finite(c[1])
if lon is not None and lat is not None and -180 <= lon <= 180 and -90 <= lat <= 90:
lons.append(lon)
lats.append(lat)
elif isinstance(c, list):
for x in c:
walk(x)
walk(geom.get("coordinates"))
if not lats:
return None
return (min(lats), min(lons), max(lats), max(lons))
def bbox_centre(b: Optional[BBox]) -> Tuple[Optional[float], Optional[float]]:
if not b:
return None, None
return (round((b[0] + b[2]) / 2, 5), round((b[1] + b[3]) / 2, 5))
@@ -0,0 +1,273 @@
"""Outage sources -- server-26#184 (source-terms table in its 2026-10-05 comment).
Only ODIN has an adapter. Every other source is listed so Admin shows its
licence status, and is a stub because DRB holds no right to use it: the utility
storm-center maps are undocumented JSON with no reuse licence (the panel ruling
stands -- do not ship scraped utility data commercially), PowerOutage.us
licenses its API for personal/non-commercial use only, and EAGLE-I's live feed
is a DOE partner system. Do not "fix" a stub by scraping.
ODIN, verified first-hand 2026-10-05 against https://odin.ornl.gov/v3/api-docs
(the OpenAPI document the ODIN developer guide points at) and live responses:
GET /odi?format=JSON public, no token. ``{"outage": [CIM Outage, ...]}``,
one row per outage event, every row aggregated to a
county: ``communityDescriptor`` = 5-digit county FIPS,
``metersAffected``, ``reportedStartTime``,
``estimatedRestorationTime.ert``, ``names`` carrying
``UtilityName`` / ``UtilityId`` (EIA id).
GET /odi/map?scale=COUNTY public. One row per county with active outages:
``id`` (FIPS), ``countyName``, ``stateName``,
``metersOut`` (= sum of /odi metersAffected, checked),
``feature`` (county Polygon/MultiPolygon).
GET /odi/status public. One row per reporting utility: ``eiaId``,
``name``, ``receivedDate``, ``hasError``.
``metersServed`` on /odi/map is the utility's system-wide total (National Grid
NY carries 1,686,884 on every county), so it is NOT a county denominator and
``customers_served`` stays None.
"""
import asyncio
import re
from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional, Tuple
from app.internal.outages.base import (
Licence, OutageArea, OutageLicenceInfo, OutageProvider, OutageProviderMeta, StubProvider, USER_AGENT,
bbox_centre, finite, geometry_bbox, get_json, iso, make_area, parse_iso, require_parsed,
)
US_BBOX = (17.5, -180.0, 72.0, -64.0) # CONUS + AK + HI + PR, rough
NY_BBOX = (40.4, -80.0, 45.1, -71.7)
ODIN_BASE = "https://odin.ornl.gov"
ODIN_OPENAPI = "https://odin.ornl.gov/v3/api-docs"
_FIPS = re.compile(r"^\d{5}$")
# An event whose own estimated restoration passed this long ago is stale: ODIN
# still listed Avangrid rows from April 2026 (ETR April/May) on 2026-10-05.
# Those are not live outages and must not be summed into one or set "since".
STALE_AFTER_ETR = timedelta(hours=24)
class OdinProvider(OutageProvider):
meta = OutageProviderMeta(
id="odin",
name="ODIN (Oak Ridge National Laboratory / DOE)",
region="United States -- participating utilities only (NY: National Grid, Central Hudson, Avangrid reporting 2026-10-05)",
coverage=US_BBOX,
licence=OutageLicenceInfo(
status=Licence.UNVERIFIED,
label="Licence unverified -- public county-level API, but no terms of use or reuse licence found",
terms=("ODIN documents GET /odi, /odi/map and /odi/status as public, non-authenticated endpoints "
"returning current outages aggregated to county FIPS (OpenAPI read first-hand 2026-10-05). "
"That document carries no licence or terms-of-service field, and no terms-of-use page was "
"found on odin.ornl.gov. ORNL is operated for DOE by a contractor and the data is "
"utility-reported, so federal public-domain status cannot be assumed. Confirm reuse terms "
"with ODIN (odin.ornl.gov/pages/contact.html) before relying on it commercially."),
terms_url=ODIN_OPENAPI,
attribution="Outage data: ODIN, Oak Ridge National Laboratory (utility-reported)",
attribution_url="https://odin.ornl.gov",
disclaimer=("County-level counts of meters reported out by utilities that participate in ODIN. "
"A utility that does not report to ODIN shows no outages here, even during one. "
"Restoration times are the utility's estimate, when given."),
issue="server-26#184",
),
granularity="county (FIPS)",
implemented=True,
notes=("Public API, verified first-hand 2026-10-05: /odi?format=JSON (counts, start, ETR, utility EIA id), "
"/odi/map?scale=COUNTY (county name, state, county polygon -> bbox and label point) and /odi/status "
"(utility last-received time). Counts are meters, used as customers. Con Edison / O&R are on ODIN's "
"participant list but were NOT reporting on 2026-10-05, so NYC/Westchester show nothing from ODIN "
"until they do. Schema canary: rows upstream but none parseable is an error, never an empty layer."),
homepage="https://odin.ornl.gov",
)
def __init__(self) -> None:
self.reporting_utilities: Optional[int] = None
self.utilities_with_error: Optional[int] = None
async def fetch(self) -> List[OutageArea]:
h = {"User-Agent": USER_AGENT}
odi, county_map, status = await asyncio.gather(
get_json(f"{ODIN_BASE}/odi", params={"format": "JSON"}, headers=h),
get_json(f"{ODIN_BASE}/odi/map", params={"scale": "COUNTY"}, headers=h),
get_json(f"{ODIN_BASE}/odi/status", headers=h),
)
areas, info = parse_odin(odi, county_map, status)
self.reporting_utilities = info["reporting_utilities"]
self.utilities_with_error = info["utilities_with_error"]
return areas
def coverage_note(self) -> Optional[str]:
n = self.reporting_utilities
base = "Only utilities that report to ODIN appear; one outside ODIN shows no outages here even during one."
if n is None:
return base
err = f", {self.utilities_with_error} with a feed error" if self.utilities_with_error else ""
return f"{base} {n} utilities reporting nationwide{err}."
def _utility(names: object) -> Tuple[Optional[str], Optional[str]]:
name = eia = None
for n in names if isinstance(names, list) else []:
if not isinstance(n, dict):
continue
if n.get("nameType") == "UtilityName" and isinstance(n.get("name"), str):
name = n["name"]
elif n.get("nameType") == "UtilityId" and n.get("name") is not None:
eia = str(n["name"])
return name, eia
def parse_odin(odi: object, county_map: object, status: object,
now: Optional[datetime] = None) -> Tuple[List[OutageArea], dict]:
"""The three public ODIN responses -> one normalised record per county.
Events whose estimated restoration is more than ``STALE_AFTER_ETR`` before
``now`` are counted separately (``stale_customers_out``), not as live.
Raises ValueError when a payload has the wrong top-level shape or when rows
came back but none could be read -- an upstream schema change must surface
as a feed error, not as an empty map."""
if not isinstance(odi, dict) or not isinstance(odi.get("outage"), list):
raise ValueError("ODIN /odi: expected {'outage': [...]} -- upstream schema change?")
if not isinstance(county_map, list):
raise ValueError("ODIN /odi/map: expected a list -- upstream schema change?")
if not isinstance(status, list):
raise ValueError("ODIN /odi/status: expected a list -- upstream schema change?")
# Utility freshness, by EIA id.
received: Dict[str, str] = {}
errors = 0
for u in status:
if not isinstance(u, dict):
continue
errors += 1 if u.get("hasError") is True else 0
eia, d = u.get("eiaId"), parse_iso(u.get("receivedDate"))
if eia is not None and d is not None:
received[str(eia)] = iso(d)
require_parsed(len(status), list(received), "ODIN /odi/status")
# County geography.
geo: Dict[str, dict] = {}
for row in county_map:
if not isinstance(row, dict):
continue
fips = str(row.get("id") or "")
if not _FIPS.match(fips):
continue
b = geometry_bbox(row.get("feature"))
geo[fips] = {"county": row.get("countyName") if isinstance(row.get("countyName"), str) else None,
"state": row.get("stateName") if isinstance(row.get("stateName"), str) else None,
"bbox": b}
require_parsed(len(county_map), list(geo), "ODIN /odi/map county")
# Outage events, aggregated per county.
stale_before = (now or datetime.now(timezone.utc)) - STALE_AFTER_ETR
agg: Dict[str, dict] = {}
parsed = stale_events = 0
for o in odi["outage"]:
if not isinstance(o, dict):
continue
fips = str(o.get("communityDescriptor") or "")
meters = finite(o.get("metersAffected"))
if not _FIPS.match(fips) or meters is None or meters < 0:
continue
parsed += 1
area = o.get("outageArea") if isinstance(o.get("outageArea"), dict) else {}
start = parse_iso(o.get("reportedStartTime")) or parse_iso(area.get("earliestReportedTime"))
ert_obj = o.get("estimatedRestorationTime") if isinstance(o.get("estimatedRestorationTime"), dict) else {}
ert = parse_iso(ert_obj.get("ert"))
if ert is not None and ert.year < 2000: # Puget Sound Energy sends 0001-01-01T00:00:00Z for "no ETR"
ert = None
stale = ert is not None and ert < stale_before
stale_events += stale
a = agg.setdefault(fips, {"out": 0, "stale_out": 0, "utilities": {},
"live": {"since": None, "etr": None}, "stale": {"since": None, "etr": None}})
a["stale_out" if stale else "out"] += int(meters)
t = a["stale" if stale else "live"] # since/etr come from live events whenever there are any
if start and (t["since"] is None or start < t["since"]):
t["since"] = start
if ert and (t["etr"] is None or ert > t["etr"]):
t["etr"] = ert
name, eia = _utility(o.get("names"))
key = eia or name or "unknown"
u = a["utilities"].setdefault(key, {"name": name, "eia_id": eia, "customers_out": 0,
"stale_customers_out": 0})
u["stale_customers_out" if stale else "customers_out"] += int(meters)
require_parsed(len(odi["outage"]), [None] * parsed, "ODIN /odi outage")
areas: List[OutageArea] = []
for fips, a in agg.items():
g = geo.get(fips, {})
utilities = sorted(a["utilities"].values(), key=lambda u: (-u["customers_out"], -u["stale_customers_out"]))
stamps = [received[u["eia_id"]] for u in utilities if u["eia_id"] in received]
lat, lon = bbox_centre(g.get("bbox"))
t = a["live"] if a["out"] else a["stale"]
areas.append(make_area(
"odin", fips, county=g.get("county"), state=g.get("state"), customers_out=a["out"],
stale_customers_out=a["stale_out"], customers_served=None, etr=iso(t["etr"]), since=iso(t["since"]),
updated_at=max(stamps) if stamps else None, lat=lat, lon=lon, bbox=g.get("bbox"),
utilities=utilities,
))
areas.sort(key=lambda r: (-r["customers_out"], -r["stale_customers_out"]))
return areas, {"reporting_utilities": len(status), "utilities_with_error": errors, "stale_events": stale_events}
# ── stubs: catalogued, never enable-able ─────────────────────────────────────
_STORM_CENTER_TERMS = ("The utility's public outage map is backed by undocumented JSON (vendor storm-center "
"endpoints) with no published licence for automated reuse; the endpoints can change "
"without notice. Panel ruling (server-26#184): do not ship scraped utility data "
"commercially. A direct data-sharing agreement is the licensed path (also a sales lead).")
def _stub(id: str, name: str, region: str, coverage, status: Licence, label: str, terms: str,
terms_url: Optional[str], granularity: str, notes: str, homepage: Optional[str]) -> StubProvider:
return StubProvider(OutageProviderMeta(
id=id, name=name, region=region, coverage=coverage,
licence=OutageLicenceInfo(status=status, label=label, terms=terms, terms_url=terms_url,
issue="server-26#184"),
granularity=granularity, implemented=False, notes=notes, homepage=homepage,
))
PROVIDERS: List[OutageProvider] = [
OdinProvider(),
_stub("poweroutage_us", "PowerOutage.us (FE Bluefire LLC)", "United States", US_BBOX, Licence.PAID,
"Paid API -- licence is personal/non-commercial (news media excepted); commercial use needs an agreement DRB does not hold",
("API & Content Terms: the licence is for \"personal and noncommercial use (except for News Media "
"Organizations)\"; commercial use needs a separate agreement; pricing by contact. Quoted from a "
"search summary -- the terms page returned 403 to a direct fetch, so re-read it first-hand before "
"any acknowledgement."),
"https://poweroutage.us/legal/apitermsofuse", "county / utility",
"No adapter: needs a commercial contract and its API documentation.", "https://poweroutage.us"),
_stub("coned_oru_stormcenter", "Con Edison / Orange & Rockland outage map", "NYC, Westchester, Orange & Rockland",
NY_BBOX, Licence.AGREEMENT_REQUIRED,
"Requires a data-sharing agreement -- undocumented storm-center JSON, no reuse licence",
_STORM_CENTER_TERMS, "https://www.coned.com/en/services-and-outages/outage-map-faq", "point / polygon + ETR",
("No adapter by design. Con Edison and O&R are on ODIN's participant list but were not reporting to "
"ODIN on 2026-10-05; if they start, the ODIN adapter covers them at county level."),
"https://outagemap.coned.com"),
_stub("nyseg_rge_stormcenter", "NYSEG / RG&E outage map (Avangrid)", "Upstate NY", NY_BBOX,
Licence.AGREEMENT_REQUIRED,
"Requires a data-sharing agreement -- undocumented storm-center JSON, no reuse licence",
_STORM_CENTER_TERMS, None, "point / polygon + ETR",
("No adapter by design. Avangrid (NYSEG/RG&E) reports to ODIN, so county-level data arrives through "
"the ODIN adapter."),
"https://www.nyseg.com"),
_stub("central_hudson_stormcenter", "Central Hudson outage map", "Hudson Valley, NY", NY_BBOX,
Licence.AGREEMENT_REQUIRED,
"Requires a data-sharing agreement -- undocumented storm-center JSON, no reuse licence",
_STORM_CENTER_TERMS, None, "point / polygon + ETR",
("No adapter by design. Central Hudson reports to ODIN, so county-level data arrives through the "
"ODIN adapter."),
"https://www.cenhud.com"),
_stub("eagle_i_live", "EAGLE-I live feed (ORNL / DOE)", "United States", US_BBOX, Licence.AGREEMENT_REQUIRED,
"Live feed is a DOE / government-partner system -- DRB has no access agreement",
("EAGLE-I's live 15-minute county outage feed is restricted to DOE and government partners. The public "
"ORNL Constellation dataset is historic only (no terms stated on the dataset) and is no use for a "
"live map."),
"https://doi.ccs.ornl.gov/dataset/d26451af-7ea4-577c-8285-78b72cd8a2dc", "county",
"No adapter: needs a DOE partner agreement.", "https://eagle-i.doe.gov"),
]
@@ -0,0 +1,216 @@
"""Outage provider registry: settings, licence gate, and bbox queries.
Mirrors ``app.internal.cameras.registry`` rule for rule. Settings live in
Firestore ``config/outage_providers``::
{"providers": {"<id>": {"enabled": bool, "updated_at": iso, "updated_by": {...},
"acknowledgement": {"by_uid", "by_email", "at", "licence_status", "warning"} | None}}}
Rules (owner decision #247 on server-26#184):
* every provider defaults OFF -- a missing doc/entry means off, and an
unreadable settings doc means everything is off (fail closed);
* a stub (no adapter) or a PROHIBITED source can never be enabled;
* a GATED licence (agreement required / non-commercial / unverified / paid)
can only be enabled with an explicit acknowledgement, recorded with who and
when in the settings doc AND audit_log. The audit write is a precondition
for a gated enable: no record, no enable;
* an acknowledgement is bound to the licence status it was given for. If a
provider's catalogued status changes in code, the old acknowledgement no
longer counts and the provider stops serving until re-acknowledged.
"""
import asyncio
import time
from datetime import datetime, timezone
from typing import Dict, List, Optional, Tuple
from app.internal import firestore as fstore
from app.internal.logger import logger
from app.internal.outages import providers as sources
from app.internal.outages.base import BBox, Feed, Licence, OutageArea, OutageProvider, area_in_bbox, bbox_intersects
COLLECTION, DOC_ID = "config", "outage_providers"
_TTL_S = 30.0
MAX_AREAS = 3000 # per response; ODIN's whole country is ~100-300 counties on a normal day
PROVIDERS: Dict[str, OutageProvider] = {p.meta.id: p for p in sources.PROVIDERS}
FEEDS: Dict[str, Feed] = {pid: Feed(p) for pid, p in PROVIDERS.items() if p.meta.implemented}
NO_SOURCE_MESSAGE = "No licensed outage source covers this area."
_cache: Dict[str, dict] = {}
_cache_ts = 0.0
class ProviderError(Exception):
"""A refused settings change. ``code`` is the HTTP status the router returns."""
def __init__(self, code: int, message: str, **extra):
super().__init__(message)
self.code, self.message, self.extra = code, message, extra
def invalidate_cache() -> None:
global _cache_ts
_cache_ts = 0.0
async def get_settings() -> Dict[str, dict]:
"""Per-provider settings. Unreadable Firestore means everything is OFF (fail closed)."""
global _cache, _cache_ts
if _cache_ts and time.monotonic() - _cache_ts < _TTL_S:
return _cache
try:
doc = await fstore.doc_get(COLLECTION, DOC_ID) or {}
provs = doc.get("providers") if isinstance(doc.get("providers"), dict) else {}
_cache = {k: v for k, v in provs.items() if isinstance(v, dict)}
except Exception as e:
logger.warning(f"outage providers: settings unreadable ({e}); treating every provider as OFF")
_cache = {}
_cache_ts = time.monotonic()
return _cache
def effective(provider: OutageProvider, entry: Optional[dict]) -> Tuple[bool, Optional[str]]:
"""(serving?, reason-if-not) for a provider given its settings entry."""
meta = provider.meta
if not meta.implemented:
return False, "adapter is a stub"
if meta.licence.status == Licence.PROHIBITED:
return False, "source terms forbid this use"
if not entry or not entry.get("enabled"):
return False, "disabled (default)"
if meta.licence.gated:
ack = entry.get("acknowledgement") or {}
if ack.get("licence_status") != meta.licence.status.value:
return False, "licence acknowledgement missing or out of date"
return True, None
async def is_serving(provider_id: str) -> bool:
p = PROVIDERS.get(provider_id)
if p is None:
return False
return effective(p, (await get_settings()).get(provider_id))[0]
async def set_enabled(provider_id: str, enabled: bool, actor: Tuple[str, str],
acknowledge_licence: bool = False) -> dict:
p = PROVIDERS.get(provider_id)
if p is None:
raise ProviderError(404, f"unknown outage provider '{provider_id}'")
meta, lic = p.meta, p.meta.licence
uid, email = actor
now = datetime.now(timezone.utc).isoformat()
entry: dict = {"enabled": bool(enabled), "updated_at": now, "updated_by": {"uid": uid, "email": email}}
if enabled:
if not meta.implemented:
raise ProviderError(409, f"{meta.name}: no working adapter -- {meta.notes}")
if lic.status == Licence.PROHIBITED:
raise ProviderError(409, lic.warning() or "prohibited")
if lic.gated:
if not acknowledge_licence:
raise ProviderError(409, f"{meta.name}: licence acknowledgement required",
requires_acknowledgement=True, warning=lic.warning(),
licence=lic.as_dict())
entry["acknowledgement"] = {"by_uid": uid, "by_email": email, "at": now,
"licence_status": lic.status.value, "warning": lic.warning()}
# Precondition, not a side effect: a gated enable with no audit
# record must not happen. A failure here aborts before the write.
try:
await _audit(actor, "outage_provider.enable_acknowledged", provider_id,
{"licence_status": lic.status.value, "licence_label": lic.label,
"warning": lic.warning(), "acknowledged_at": now})
except Exception as e:
logger.error(f"outage providers: audit write failed, refusing gated enable of {provider_id} ({e})")
raise ProviderError(503, "audit log unavailable -- a licence-gated provider cannot be enabled "
"without an audit record; try again")
else:
entry["acknowledgement"] = None # re-enabling a gated source needs a fresh acknowledgement
before = (await get_settings()).get(provider_id) or {}
await fstore.doc_set(COLLECTION, DOC_ID, {"providers": {provider_id: entry}}, merge=True)
invalidate_cache()
if not (enabled and lic.gated):
try:
await _audit(actor, "outage_provider.enable" if enabled else "outage_provider.disable", provider_id,
{"from": bool(before.get("enabled")), "to": bool(enabled),
"licence_status": lic.status.value})
except Exception as e: # the toggle already happened; never 500 a completed write
logger.error(f"outage providers: audit write failed for {provider_id} ({e})")
logger.info(f"outage provider {provider_id} -> enabled={enabled} by {email or uid}")
return await describe(provider_id)
async def _audit(actor: Tuple[str, str], action: str, provider_id: str, details: dict) -> None:
from app.internal import audit
await audit.write_audit(actor_uid=actor[0], actor_email=actor[1], action=action,
details={"provider_id": provider_id, **details})
async def describe(provider_id: str) -> dict:
p = PROVIDERS[provider_id]
entry = (await get_settings()).get(provider_id) or {}
serving, reason = effective(p, entry)
m = p.meta
feed = FEEDS.get(provider_id)
return {
"id": m.id, "name": m.name, "region": m.region, "granularity": m.granularity,
"implemented": m.implemented, "notes": m.notes, "homepage": m.homepage,
"licence": m.licence.as_dict(),
"enabled": bool(entry.get("enabled")),
"serving": serving, "not_serving_reason": reason,
"acknowledgement": entry.get("acknowledgement"),
"updated_at": entry.get("updated_at"), "updated_by": entry.get("updated_by"),
"coverage_note": p.coverage_note(),
"health": feed.status() if feed else None,
}
async def list_providers() -> List[dict]:
await get_settings()
return [await describe(pid) for pid in PROVIDERS]
def credit(provider_id: str) -> dict:
lic = PROVIDERS[provider_id].meta.licence
return {"attribution": lic.attribution, "attribution_url": lic.attribution_url, "disclaimer": lic.disclaimer}
async def outages_in_bbox(bbox: BBox) -> dict:
"""Outage areas from every SERVING provider whose coverage meets ``bbox``.
Providers that are off are never fetched.
``status`` says why the list is what it is, so an empty layer is never
ambiguous: ``none_enabled`` (no provider is on), ``no_source`` (none of the
serving providers covers this area), ``error`` (every covering provider
failed and has nothing cached), else ``ok`` -- which may legitimately
carry zero areas when nothing is out."""
conf = await get_settings()
serving = [pid for pid in FEEDS if effective(PROVIDERS[pid], conf.get(pid))[0]]
live = [pid for pid in serving if bbox_intersects(PROVIDERS[pid].meta.coverage, bbox)]
await asyncio.gather(*(FEEDS[pid].get() for pid in live))
areas: List[OutageArea] = []
providers = []
for pid in serving:
feed = FEEDS[pid]
hits = [a for a in feed.items if area_in_bbox(a, bbox)] if pid in live else []
room = max(0, MAX_AREAS - len(areas))
areas.extend(hits[:room])
providers.append({"id": pid, "name": PROVIDERS[pid].meta.name, **credit(pid),
"granularity": PROVIDERS[pid].meta.granularity,
"coverage_note": PROVIDERS[pid].coverage_note(),
"covers_bbox": pid in live,
"fetched_at": feed.fetched_at, "error": feed.error,
"total_in_bbox": len(hits), "truncated": len(hits) > room})
if not serving:
status, message = "none_enabled", "No outage source is enabled. An admin can enable one under Admin -> Outage providers."
elif not live:
status, message = "no_source", NO_SOURCE_MESSAGE
elif all(FEEDS[pid].error and FEEDS[pid].fetched_at is None for pid in live):
status, message = "error", "Every outage source covering this area failed to load; see each provider's error."
else:
status, message = "ok", None
return {"areas": areas, "providers": providers, "serving_count": len(serving),
"status": status, "message": message}
+3
View File
@@ -19,6 +19,7 @@ from app.internal.auth import (
) )
from app.routers import nodes, systems, calls, upload, tokens, incidents, alerts, admin, trips, places, links, users from app.routers import nodes, systems, calls, upload, tokens, incidents, alerts, admin, trips, places, links, users
from app.routers import enrollment, media, org, waitlist, telemetry, replay, traffic, cameras from app.routers import enrollment, media, org, waitlist, telemetry, replay, traffic, cameras
from app.routers import outages
from app.routers import discord as discord_router from app.routers import discord as discord_router
from app.routers import discord_link as discord_link_router from app.routers import discord_link as discord_link_router
from app.internal import dynsec from app.internal import dynsec
@@ -146,6 +147,8 @@ app.include_router(places.router, dependencies=[Depends(require_service_or_fi
app.include_router(traffic.router, dependencies=[Depends(require_service_or_firebase_token)]) app.include_router(traffic.router, dependencies=[Depends(require_service_or_firebase_token)])
app.include_router(cameras.router, dependencies=[Depends(require_service_or_firebase_token)]) app.include_router(cameras.router, dependencies=[Depends(require_service_or_firebase_token)])
app.include_router(cameras.admin_router) # auth: admin only, per endpoint (needs the principal for audit) app.include_router(cameras.admin_router) # auth: admin only, per endpoint (needs the principal for audit)
app.include_router(outages.router, dependencies=[Depends(require_service_or_firebase_token)])
app.include_router(outages.admin_router) # auth: admin only, per endpoint (needs the principal for audit)
app.include_router(upload.router) # auth is per-node, handled inline app.include_router(upload.router) # auth is per-node, handled inline
app.include_router(admin.router) # auth is per-endpoint (read: firebase, write: admin) app.include_router(admin.router) # auth is per-endpoint (read: firebase, write: admin)
app.include_router(replay.router) # auth: admin only (every route spends or reads a replay run) app.include_router(replay.router) # auth: admin only (every route spends or reads a replay run)
+52
View File
@@ -0,0 +1,52 @@
"""Power-outage endpoints (map layer, server-26#184).
``GET /outages`` -- any signed-in user (auth applied at include time in main.py):
outage areas inside a bbox from every provider an admin has enabled, each with
the provider's attribution, disclaimer, coverage note and feed freshness, plus
a ``status`` that says why an empty list is empty.
``/admin/outage-providers`` -- platform admin only: list every catalogued
provider with its licence status and health, and toggle it. Gated licences
need ``acknowledge_licence: true``; that acknowledgement is stored with who and
when and written to audit_log (see app/internal/outages/registry.py).
"""
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi.responses import JSONResponse
from pydantic import BaseModel
from app.internal.auth import describe_actor, require_admin_token
from app.internal.outages import registry
router = APIRouter(prefix="/outages", tags=["outages"])
admin_router = APIRouter(prefix="/admin/outage-providers", tags=["admin"])
@router.get("")
async def get_outages(
south: float = Query(..., ge=-90, le=90),
west: float = Query(..., ge=-180, le=180),
north: float = Query(..., ge=-90, le=90),
east: float = Query(..., ge=-180, le=180),
):
if south > north or west > east:
raise HTTPException(400, "bbox must satisfy south<=north and west<=east")
return await registry.outages_in_bbox((south, west, north, east))
class ProviderToggle(BaseModel):
enabled: bool
acknowledge_licence: bool = False
@admin_router.get("")
async def list_outage_providers(_=Depends(require_admin_token)):
return {"providers": await registry.list_providers()}
@admin_router.put("/{provider_id}")
async def set_outage_provider(provider_id: str, body: ProviderToggle, principal: dict = Depends(require_admin_token)):
try:
return await registry.set_enabled(provider_id, body.enabled, describe_actor(principal),
acknowledge_licence=body.acknowledge_licence)
except registry.ProviderError as e:
return JSONResponse(status_code=e.code, content={"detail": e.message, **e.extra})
File diff suppressed because it is too large Load Diff
+329
View File
@@ -0,0 +1,329 @@
"""Power-outage providers (server-26#184): registry, licence gate, admin routes, ODIN adapter.
What is held here (mirrors tests/test_camera_providers.py):
* every provider defaults OFF, and an off provider is never fetched;
* stubs can never be enabled (409);
* a gated licence needs ``acknowledge_licence``, which is stored with who/when
and audited -- and an audit failure refuses the enable (fail closed);
* an acknowledgement given for a different licence status no longer counts;
* the admin routes are admin-only;
* the ODIN parser reads the recorded public payloads, and an all-unparseable
payload is an error, not an empty layer;
* an area no serving provider covers says so instead of returning a bare [].
"""
import asyncio
import json
from datetime import datetime, timezone
from pathlib import Path
from unittest.mock import AsyncMock, patch
import pytest
from fastapi import HTTPException
from fastapi.testclient import TestClient
from app.internal.auth import require_admin_token, require_service_or_firebase_token
from app.internal.outages import base, providers, registry
from app.internal.outages.base import Licence
from app.main import app
client = TestClient(app)
ADMIN = {"uid": "admin-1", "email": "admin@example.com", "role": "admin"}
FIXTURE = json.loads((Path(__file__).parent / "fixtures" / "odin_public_2026-10-05.json").read_text())
STUBS = ("poweroutage_us", "coned_oru_stormcenter", "nyseg_rge_stormcenter", "central_hudson_stormcenter", "eagle_i_live")
NY_BBOX = (42.5, -78.5, 43.5, -75.5) # Rochester .. Syracuse
RECORDED_AT = datetime(2026, 10, 5, 6, 45, tzinfo=timezone.utc) # when the fixture was captured
class FakeStore:
def __init__(self):
self.docs = {}
self.audit = []
async def doc_get(self, col, doc):
return self.docs.get((col, doc))
async def doc_set(self, col, doc, data, merge=True):
cur = self.docs.setdefault((col, doc), {})
if col == "audit_log":
self.audit.append(data)
return
for k, v in data.items(): # one-level deep merge is all registry needs
if merge and isinstance(v, dict) and isinstance(cur.get(k), dict):
cur[k] = {**cur[k], **v}
else:
cur[k] = v
@pytest.fixture(autouse=True)
def store():
s = FakeStore()
registry.invalidate_cache()
for feed in registry.FEEDS.values():
feed.items, feed.fetched_at, feed.error, feed._next_attempt = [], None, None, 0.0
with patch.object(registry.fstore, "doc_get", s.doc_get), patch.object(registry.fstore, "doc_set", s.doc_set), \
patch("app.internal.audit.fstore.doc_set", s.doc_set):
yield s
registry.invalidate_cache()
app.dependency_overrides.clear()
def run(coro):
return asyncio.run(coro)
def _odin_areas():
return providers.parse_odin(FIXTURE["odi_format_json"], FIXTURE["odi_map_scale_county"], FIXTURE["odi_status"],
now=RECORDED_AT)[0]
# ── registry / gate ──────────────────────────────────────────────────────────
def test_every_catalogued_source_is_registered_and_defaults_off():
listed = run(registry.list_providers())
assert {p["id"] for p in listed} == {"odin", *STUBS}
assert all(p["enabled"] is False and p["serving"] is False for p in listed)
assert all(p["licence"]["label"] for p in listed)
odin = next(p for p in listed if p["id"] == "odin")
assert odin["licence"]["status"] == "unverified" and odin["licence"]["gated"] is True
assert odin["licence"]["terms_url"] == "https://odin.ornl.gov/v3/api-docs"
assert odin["not_serving_reason"] == "disabled (default)"
def test_catalogued_licence_statuses_match_the_source_terms():
lic = {pid: p.meta.licence.status for pid, p in registry.PROVIDERS.items()}
assert lic["poweroutage_us"] == Licence.PAID
for pid in ("coned_oru_stormcenter", "nyseg_rge_stormcenter", "central_hudson_stormcenter", "eagle_i_live"):
assert lic[pid] == Licence.AGREEMENT_REQUIRED, pid
assert all(not registry.PROVIDERS[pid].meta.implemented for pid in STUBS)
def test_warning_is_worded_for_outage_data():
w = registry.PROVIDERS["odin"].meta.licence.warning()
assert "outage data" in w and "imagery" not in w
def test_off_provider_is_never_fetched():
fetch = AsyncMock(return_value=[])
with patch.object(registry.PROVIDERS["odin"], "fetch", fetch):
out = run(registry.outages_in_bbox(NY_BBOX))
assert out["areas"] == [] and out["providers"] == [] and fetch.await_count == 0
assert out["serving_count"] == 0 and out["status"] == "none_enabled"
def test_stubs_cannot_be_enabled():
for pid in STUBS:
with pytest.raises(registry.ProviderError) as e:
run(registry.set_enabled(pid, True, ("u", "e"), acknowledge_licence=True))
assert e.value.code == 409, pid
def test_gated_licence_requires_acknowledgement(store):
with pytest.raises(registry.ProviderError) as e:
run(registry.set_enabled("odin", True, ("u1", "a@x")))
assert e.value.code == 409 and e.value.extra["requires_acknowledgement"] is True
assert "Licence unverified" in e.value.extra["warning"]
assert not store.audit and not run(registry.is_serving("odin"))
out = run(registry.set_enabled("odin", True, ("u1", "a@x"), acknowledge_licence=True))
assert out["serving"] is True
ack = out["acknowledgement"]
assert ack["by_uid"] == "u1" and ack["by_email"] == "a@x" and ack["at"]
assert ack["licence_status"] == Licence.UNVERIFIED.value
assert store.audit[-1]["action"] == "outage_provider.enable_acknowledged"
assert store.audit[-1]["details"]["provider_id"] == "odin"
assert store.docs[("config", "outage_providers")]["providers"]["odin"]["enabled"] is True
def test_gated_enable_refused_when_audit_cannot_be_written(store):
with patch.object(registry, "_audit", AsyncMock(side_effect=RuntimeError("firestore down"))):
with pytest.raises(registry.ProviderError) as e:
run(registry.set_enabled("odin", True, ("u", "e"), acknowledge_licence=True))
assert e.value.code == 503
assert not run(registry.is_serving("odin"))
assert ("config", "outage_providers") not in store.docs
def test_disable_clears_acknowledgement_and_is_audited(store):
run(registry.set_enabled("odin", True, ("u", "e"), acknowledge_licence=True))
out = run(registry.set_enabled("odin", False, ("u", "e")))
assert out["serving"] is False and out["acknowledgement"] is None
assert store.audit[-1]["action"] == "outage_provider.disable"
def test_stale_acknowledgement_stops_serving():
p = registry.PROVIDERS["odin"]
entry = {"enabled": True, "acknowledgement": {"licence_status": "paid"}}
assert registry.effective(p, entry) == (False, "licence acknowledgement missing or out of date")
entry["acknowledgement"]["licence_status"] = p.meta.licence.status.value
assert registry.effective(p, entry) == (True, None)
def test_unreadable_settings_fail_closed():
with patch.object(registry.fstore, "doc_get", AsyncMock(side_effect=RuntimeError("boom"))):
registry.invalidate_cache()
assert run(registry.get_settings()) == {}
assert not run(registry.is_serving("odin"))
# ── bbox query ───────────────────────────────────────────────────────────────
def test_query_returns_areas_in_bbox_with_credit(store):
run(registry.set_enabled("odin", True, ("u", "e"), acknowledge_licence=True))
with patch.object(registry.PROVIDERS["odin"], "fetch", AsyncMock(return_value=_odin_areas())):
out = run(registry.outages_in_bbox(NY_BBOX))
assert out["status"] == "ok" and out["serving_count"] == 1
assert {a["fips"] for a in out["areas"]} == {"36055", "36067"} # not Albany, not King County WA
prov = out["providers"][0]
assert prov["attribution"].startswith("Outage data: ODIN") and prov["disclaimer"]
assert prov["total_in_bbox"] == 2 and prov["fetched_at"] and prov["error"] is None
assert "Only utilities that report to ODIN appear" in prov["coverage_note"]
def test_area_no_serving_provider_covers_says_so(store):
run(registry.set_enabled("odin", True, ("u", "e"), acknowledge_licence=True))
fetch = AsyncMock(return_value=_odin_areas())
with patch.object(registry.PROVIDERS["odin"], "fetch", fetch):
out = run(registry.outages_in_bbox((51.4, -0.2, 51.6, 0.1))) # London
assert fetch.await_count == 0 # outside ODIN coverage -> never fetched
assert out["areas"] == [] and out["status"] == "no_source"
assert out["message"] == "No licensed outage source covers this area."
def test_dead_feed_is_an_error_not_an_empty_layer(store):
run(registry.set_enabled("odin", True, ("u", "e"), acknowledge_licence=True))
with patch.object(registry.PROVIDERS["odin"], "fetch", AsyncMock(side_effect=ValueError("schema change?"))):
out = run(registry.outages_in_bbox(NY_BBOX))
assert out["status"] == "error" and "schema change" in out["providers"][0]["error"]
def test_failed_refresh_keeps_last_good_feed():
feed = registry.FEEDS["odin"]
with patch.object(feed.provider, "fetch", AsyncMock(return_value=_odin_areas())):
run(feed.get())
feed._next_attempt = 0.0
with patch.object(feed.provider, "fetch", AsyncMock(side_effect=RuntimeError("down"))):
run(feed.get())
assert len(feed.items) == 4 and "down" in feed.error and feed.fetched_at
# ── admin routes ─────────────────────────────────────────────────────────────
def test_admin_routes_require_admin():
def deny():
raise HTTPException(403, "Admin access required")
app.dependency_overrides[require_admin_token] = deny
assert client.get("/admin/outage-providers").status_code == 403
assert client.put("/admin/outage-providers/odin", json={"enabled": True, "acknowledge_licence": True}).status_code == 403
def test_admin_route_toggle_and_gate(store):
app.dependency_overrides[require_admin_token] = lambda: ADMIN
r = client.put("/admin/outage-providers/odin", json={"enabled": True})
assert r.status_code == 409 and r.json()["requires_acknowledgement"] is True
assert r.json()["licence"]["status"] == "unverified"
r = client.put("/admin/outage-providers/odin", json={"enabled": True, "acknowledge_licence": True})
assert r.status_code == 200 and r.json()["acknowledgement"]["by_email"] == "admin@example.com"
assert client.put("/admin/outage-providers/poweroutage_us",
json={"enabled": True, "acknowledge_licence": True}).status_code == 409
assert client.put("/admin/outage-providers/nope", json={"enabled": True}).status_code == 404
listed = client.get("/admin/outage-providers").json()["providers"]
assert next(p for p in listed if p["id"] == "odin")["serving"] is True
def test_outages_route_rejects_inverted_bbox_and_reports_status():
app.dependency_overrides[require_service_or_firebase_token] = lambda: {"uid": "v"}
assert client.get("/outages", params={"south": 2, "west": 0, "north": 1, "east": 1}).status_code == 400
r = client.get("/outages", params={"south": 40.9, "west": -74.0, "north": 41.4, "east": -73.4})
assert r.status_code == 200 and r.json()["areas"] == [] and r.json()["status"] == "none_enabled"
def test_outages_route_requires_sign_in():
def deny():
raise HTTPException(401, "Not signed in")
app.dependency_overrides[require_service_or_firebase_token] = deny
assert client.get("/outages", params={"south": 40, "west": -74, "north": 41, "east": -73}).status_code == 401
# ── ODIN adapter (recorded public payloads, 2026-10-05) ──────────────────────
def test_odin_parser_reads_recorded_payload():
areas, info = providers.parse_odin(FIXTURE["odi_format_json"], FIXTURE["odi_map_scale_county"], FIXTURE["odi_status"],
now=RECORDED_AT)
by = {a["fips"]: a for a in areas}
assert set(by) == {"36001", "36067", "36055", "53033"}
onondaga = by["36067"]
assert onondaga["id"] == "odin:36067" and onondaga["provider"] == "odin"
assert onondaga["county"] == "Onondaga" and onondaga["state"] == "New York"
assert onondaga["customers_out"] == 11 and onondaga["customers_served"] is None
assert onondaga["etr"] == "2026-10-05T08:00:00Z" and onondaga["since"] == "2026-10-04T18:35:40Z"
assert onondaga["updated_at"].startswith("2026-10-05T06:20:47")
assert onondaga["utilities"] == [{"name": "NATIONAL GRID", "eia_id": "13573", "customers_out": 11,
"stale_customers_out": 0}]
assert onondaga["stale"] is False and onondaga["stale_customers_out"] == 0
s, w, n, e = onondaga["bbox"]
assert s < onondaga["lat"] < n and w < onondaga["lon"] < e
assert 42.7 < onondaga["lat"] < 43.3 and -76.6 < onondaga["lon"] < -75.8
# King County WA: several outage events from two utilities, summed per county
king = by["53033"]
assert king["customers_out"] == sum(r["metersOut"] for r in FIXTURE["odi_map_scale_county"] if r["id"] == "53033")
assert len(king["utilities"]) == 2
# Puget Sound Energy's "0001-01-01T00:00:00Z" means no ETR -- not a restoration time, not stale.
assert king["etr"] is None and king["stale_customers_out"] == 0
assert info["reporting_utilities"] == len(FIXTURE["odi_status"]) and info["utilities_with_error"] == 1
def test_odin_events_long_past_their_etr_are_stale_not_live():
# ODIN still listed Avangrid's Monroe County row from April 2026 (ETR
# 2026-04-10) on 2026-10-05. It must not read as 18 customers out right now.
by = {a["fips"]: a for a in _odin_areas()}
monroe = by["36055"]
assert monroe["customers_out"] == 0 and monroe["stale_customers_out"] == 18 and monroe["stale"] is True
assert monroe["since"].startswith("2026-04-10") # still shown, so the stale row is explainable
# Same payload, captured the day the outage was live: not stale.
live = {a["fips"]: a for a in providers.parse_odin(
FIXTURE["odi_format_json"], FIXTURE["odi_map_scale_county"], FIXTURE["odi_status"],
now=datetime(2026, 4, 10, 20, 0, tzinfo=timezone.utc))[0]}
assert live["36055"]["customers_out"] == 18 and live["36055"]["stale"] is False
# Live counties sort ahead of stale ones.
assert [a["stale"] for a in _odin_areas()] == sorted(a["stale"] for a in _odin_areas())
def test_odin_fetch_uses_the_three_public_endpoints():
calls = []
async def fake_get_json(url, params=None, headers=None):
calls.append((url, params))
return {"https://odin.ornl.gov/odi": FIXTURE["odi_format_json"],
"https://odin.ornl.gov/odi/map": FIXTURE["odi_map_scale_county"],
"https://odin.ornl.gov/odi/status": FIXTURE["odi_status"]}[url]
p = providers.OdinProvider()
with patch.object(providers, "get_json", fake_get_json):
areas = run(p.fetch())
assert len(areas) == 4
assert ("https://odin.ornl.gov/odi", {"format": "JSON"}) in calls
assert ("https://odin.ornl.gov/odi/map", {"scale": "COUNTY"}) in calls
assert p.reporting_utilities == len(FIXTURE["odi_status"])
assert "utilities reporting nationwide" in p.coverage_note()
def test_odin_schema_canary_trips_on_unparseable_payload():
odi, cmap, status = FIXTURE["odi_format_json"], FIXTURE["odi_map_scale_county"], FIXTURE["odi_status"]
renamed_odi = {"outage": [{"county": "36067", "count": 11}]} # fields renamed upstream
with pytest.raises(ValueError, match="schema"):
providers.parse_odin(renamed_odi, cmap, status)
with pytest.raises(ValueError, match="schema"):
providers.parse_odin(odi, [{"fips": "36067"}], status)
with pytest.raises(ValueError, match="schema"):
providers.parse_odin(odi, cmap, [{"utility": "x"}])
with pytest.raises(ValueError, match="schema"):
providers.parse_odin([], cmap, status) # wrapper changed
# A genuinely quiet day is NOT an error: nothing upstream, nothing parsed.
assert providers.parse_odin({"outage": []}, [], status)[0] == []
def test_area_in_bbox_uses_county_extent():
a = base.make_area("odin", "36067", county="Onondaga", state="New York", customers_out=1,
lat=43.0, lon=-76.2, bbox=(42.77, -76.5, 43.27, -75.9))
assert base.area_in_bbox(a, (43.2, -76.0, 43.5, -75.5)) # overlaps the county's corner only
assert not base.area_in_bbox(a, (44.0, -76.0, 44.5, -75.5))
+6 -3
View File
@@ -4,6 +4,7 @@ import { useAuth } from "@/components/AuthProvider";
import { c2api } from "@/lib/c2api"; import { c2api } from "@/lib/c2api";
import { ReplayTab } from "@/components/admin/ReplayTab"; import { ReplayTab } from "@/components/admin/ReplayTab";
import { CameraProvidersTab } from "@/components/admin/CameraProvidersTab"; import { CameraProvidersTab } from "@/components/admin/CameraProvidersTab";
import { OutageProvidersTab } from "@/components/admin/OutageProvidersTab";
import { useEffect, useState, useRef, useCallback } from "react"; import { useEffect, useState, useRef, useCallback } from "react";
import { useRouter } from "next/navigation"; import { useRouter } from "next/navigation";
import type { UserRecord, AuditEntry, UserRole, CallRecord } from "@/lib/types"; import type { UserRecord, AuditEntry, UserRole, CallRecord } from "@/lib/types";
@@ -1268,11 +1269,12 @@ function SttEvalTab() {
// Main admin page // Main admin page
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
type AdminTab = "features" | "cameras" | "correlation" | "replay" | "users" | "audit" | "calls" | "eval"; type AdminTab = "features" | "cameras" | "outages" | "correlation" | "replay" | "users" | "audit" | "calls" | "eval";
const TAB_LABELS: { key: AdminTab; label: string }[] = [ const TAB_LABELS: { key: AdminTab; label: string }[] = [
{ key: "features", label: "AI Features" }, { key: "features", label: "AI Features" },
{ key: "cameras", label: "Camera providers" }, { key: "cameras", label: "Camera providers" },
{ key: "outages", label: "Outage providers" },
{ key: "correlation", label: "Correlation Debug" }, { key: "correlation", label: "Correlation Debug" },
{ key: "replay", label: "Replay" }, { key: "replay", label: "Replay" },
{ key: "calls", label: "Calls" }, { key: "calls", label: "Calls" },
@@ -1285,7 +1287,7 @@ export default function AdminPage() {
const { user, isAdmin, loading: authLoading } = useAuth(); const { user, isAdmin, loading: authLoading } = useAuth();
const router = useRouter(); const router = useRouter();
const [tab, setTab] = useState<AdminTab>("features"); const [tab, setTab] = useState<AdminTab>("features");
// Deep link to a tab (/admin#cameras), e.g. from the live wall's empty state. // Deep link to a tab (/admin#cameras, /admin#outages), e.g. from a map layer's empty state.
useEffect(() => { useEffect(() => {
const h = window.location.hash.slice(1); const h = window.location.hash.slice(1);
if (TAB_LABELS.some((t) => t.key === h)) setTab(h as AdminTab); if (TAB_LABELS.some((t) => t.key === h)) setTab(h as AdminTab);
@@ -1304,7 +1306,7 @@ export default function AdminPage() {
if (!isAdmin) return null; if (!isAdmin) return null;
// Users/Audit tabs benefit from full width; everything else is narrow // Users/Audit tabs benefit from full width; everything else is narrow
const wide = tab === "users" || tab === "audit" || tab === "replay" || tab === "cameras"; const wide = tab === "users" || tab === "audit" || tab === "replay" || tab === "cameras" || tab === "outages";
return ( return (
<div className={`space-y-6 ${wide ? "" : "max-w-2xl"}`}> <div className={`space-y-6 ${wide ? "" : "max-w-2xl"}`}>
@@ -1326,6 +1328,7 @@ export default function AdminPage() {
{tab === "features" && <FeaturesTab />} {tab === "features" && <FeaturesTab />}
{tab === "cameras" && <CameraProvidersTab />} {tab === "cameras" && <CameraProvidersTab />}
{tab === "outages" && <OutageProvidersTab />}
{tab === "correlation" && <CorrelationDebugTab />} {tab === "correlation" && <CorrelationDebugTab />}
{tab === "replay" && <ReplayTab />} {tab === "replay" && <ReplayTab />}
{tab === "calls" && <StaleCallsTab />} {tab === "calls" && <StaleCallsTab />}
@@ -0,0 +1,156 @@
"use client";
/**
* Admin → Outage providers (server-26#184). Same licence gate as Camera
* providers: every source is listed with its licence status shown first, all
* default OFF. A licence-gated source (unverified / agreement required / paid)
* shows a blocking warning and needs an explicit confirmation, which c2-core
* records with who/when in config/outage_providers and audit_log. Stubs —
* utility storm-center maps, PowerOutage.us, EAGLE-I live — cannot be switched
* on at all: DRB holds no right to that data.
*/
import { useEffect, useState } from "react";
import { c2api } from "@/lib/c2api";
import type { LicenceStatus, OutageProviderAdmin } from "@/lib/types";
const LICENCE_STYLE: Record<LicenceStatus, { label: string; cls: string }> = {
commercial_ok: { label: "Commercial OK", cls: "bg-green-900 text-green-300 border-green-700" },
attribution_required: { label: "Attribution required", cls: "bg-sky-950 text-sky-300 border-sky-700" },
agreement_required: { label: "Requires agreement — not signed", cls: "bg-red-950 text-red-300 border-red-700" },
non_commercial: { label: "Non-commercial only", cls: "bg-red-950 text-red-300 border-red-700" },
unverified: { label: "Licence unverified", cls: "bg-amber-950 text-amber-300 border-amber-700" },
paid: { label: "Paid licence — no contract", cls: "bg-amber-950 text-amber-300 border-amber-700" },
prohibited: { label: "Terms forbid use", cls: "bg-red-950 text-red-300 border-red-700" },
};
function fmtEpoch(s: number | null | undefined): string {
return s ? new Date(s * 1000).toLocaleString([], { month: "short", day: "numeric", hour: "numeric", minute: "2-digit" }) : "—";
}
export function OutageProvidersTab() {
const [providers, setProviders] = useState<OutageProviderAdmin[] | null>(null);
const [error, setError] = useState<string | null>(null);
const [busy, setBusy] = useState<string | null>(null);
const [confirming, setConfirming] = useState<OutageProviderAdmin | null>(null);
const [ackChecked, setAckChecked] = useState(false);
const load = () =>
c2api.listOutageProviders().then((r) => setProviders(r.providers)).catch((e) => setError(String(e)));
useEffect(() => { load(); }, []);
async function toggle(p: OutageProviderAdmin, enabled: boolean, acknowledge = false) {
if (enabled && p.licence.gated && !acknowledge) {
setError(null);
setAckChecked(false);
setConfirming(p);
return;
}
setBusy(p.id);
setError(null);
try {
const updated = await c2api.setOutageProvider(p.id, enabled, acknowledge);
setProviders((cur) => cur?.map((x) => (x.id === p.id ? updated : x)) ?? null);
setConfirming(null);
} catch (e) {
setError(String(e));
} finally {
setBusy(null);
}
}
return (
<section className="space-y-3">
<p className="text-gray-400 text-xs leading-snug">
Power-outage sources for the map. Every provider is <b>off</b> until enabled here. Licence status comes from the
source-terms research on server-26#184 (2026-10-05); a red or amber licence means DRB does not hold a confirmed right
to show that data. Scraped utility outage maps are never shipped — those sources stay locked until a data-sharing
agreement exists.
</p>
{error && <div className="bg-red-950 border border-red-800 rounded-lg p-3 text-red-400 text-sm font-mono break-words">{error}</div>}
{!providers ? (
<p className="text-gray-500 text-sm font-mono">Loading…</p>
) : (
<div className="bg-gray-900 border border-gray-800 rounded-xl divide-y divide-gray-800">
{providers.map((p) => {
const lic = LICENCE_STYLE[p.licence.status];
const lockReason = !p.implemented ? "No working adapter (stub)" : p.licence.status === "prohibited" ? "Source terms forbid this use" : null;
return (
<div key={p.id} className="px-5 py-4 space-y-2">
<div className="flex items-start justify-between gap-4">
<div className="min-w-0 space-y-1">
<span className={`inline-block text-xs font-semibold px-2 py-0.5 rounded border ${lic.cls}`}>{lic.label}</span>
<p className="text-white text-sm font-semibold">{p.name}</p>
<p className="text-gray-400 text-xs">{p.licence.label}</p>
<p className="text-gray-500 text-xs">{p.region} · {p.granularity}</p>
</div>
<div className="shrink-0 text-right space-y-1">
<button
type="button"
role="switch"
aria-checked={p.enabled}
aria-label={`${p.enabled ? "Disable" : "Enable"} ${p.name}`}
disabled={busy === p.id || (!!lockReason && !p.enabled)}
title={lockReason ?? undefined}
onClick={() => toggle(p, !p.enabled)}
className={`relative inline-flex h-6 w-11 items-center rounded-full transition-colors disabled:opacity-40 ${p.enabled ? "bg-indigo-600" : "bg-gray-700"}`}
>
<span className={`inline-block h-4 w-4 rounded-full bg-white shadow transition-transform ${p.enabled ? "translate-x-6" : "translate-x-1"}`} />
</button>
<p className={`text-[11px] font-mono ${p.serving ? "text-green-400" : "text-gray-500"}`}>
{p.serving ? "serving" : lockReason ?? p.not_serving_reason ?? "off"}
</p>
</div>
</div>
{p.implemented && (
<div className="grid grid-cols-2 md:grid-cols-3 gap-2 text-[11px] font-mono text-gray-400">
<span>counties out: {p.health ? p.health.count : "—"}</span>
<span>last fetch: {fmtEpoch(p.health?.fetched_at)}</span>
<span className={p.health?.error ? "text-red-400" : ""}>health: {p.health ? (p.health.error ? "error" : p.health.fetched_at ? "ok" : "not fetched yet") : "n/a"}</span>
</div>
)}
{p.health?.error && <p className="text-[11px] font-mono text-red-400 break-words">{p.health.error}</p>}
{p.coverage_note && <p className="text-[11px] text-gray-400">{p.coverage_note}</p>}
{p.licence.attribution && <p className="text-[11px] text-gray-400">Attribution shown with the data: “{p.licence.attribution}”</p>}
{p.acknowledgement && (
<p className="text-[11px] text-amber-300">
Licence warning acknowledged by {p.acknowledgement.by_email || p.acknowledgement.by_uid} on {new Date(p.acknowledgement.at).toLocaleString()}
</p>
)}
<details className="text-[11px] text-gray-500">
<summary className="cursor-pointer select-none">Terms & adapter notes</summary>
<p className="mt-1">{p.licence.terms}</p>
<p className="mt-1">{p.notes}</p>
<p className="mt-1 space-x-2">
{p.licence.terms_url && <a className="underline" href={p.licence.terms_url} target="_blank" rel="noopener noreferrer">terms</a>}
{p.homepage && <a className="underline" href={p.homepage} target="_blank" rel="noopener noreferrer">source</a>}
{p.licence.issue && <span>tracking: {p.licence.issue}</span>}
</p>
</details>
</div>
);
})}
</div>
)}
{confirming && (
<div className="fixed inset-0 z-50 bg-black/70 flex items-center justify-center p-4" role="alertdialog" aria-modal="true" aria-labelledby="outage-ack-title">
<div className="bg-gray-900 border border-red-700 rounded-xl max-w-lg w-full p-5 space-y-3">
<h2 id="outage-ack-title" className="text-white font-semibold">Enable {confirming.name}?</h2>
<p className="text-red-300 text-sm font-semibold">{LICENCE_STYLE[confirming.licence.status].label}</p>
<p className="text-gray-300 text-sm leading-snug">{confirming.licence.warning}</p>
{error && <p className="text-red-400 text-xs font-mono break-words">{error}</p>}
<label className="flex items-start gap-2 text-sm text-gray-200">
<input type="checkbox" checked={ackChecked} onChange={(e) => setAckChecked(e.target.checked)} className="mt-1" />
<span>I confirm DRB holds the right to show this source&apos;s outage data. This confirmation is recorded with my account and the time in the audit log.</span>
</label>
<div className="flex justify-end gap-2">
<button type="button" onClick={() => setConfirming(null)} className="text-sm px-3 py-1.5 rounded border border-gray-700 text-gray-300">Cancel</button>
<button type="button" disabled={!ackChecked || busy === confirming.id} onClick={() => toggle(confirming, true, true)}
className="text-sm px-3 py-1.5 rounded bg-red-700 text-white disabled:opacity-40">Enable and record</button>
</div>
</div>
</div>
)}
</section>
);
}
+19 -1
View File
@@ -1,5 +1,7 @@
import { auth } from "@/lib/firebase"; import { auth } from "@/lib/firebase";
import type { AreaContext, CameraProviderAdmin, CamerasResponse, Ny511Response, TalkgroupPending } from "@/lib/types"; import type {
AreaContext, CameraProviderAdmin, CamerasResponse, Ny511Response, OutageProviderAdmin, OutagesResponse, TalkgroupPending,
} from "@/lib/types";
import type { DiscordAdminOverview, DiscordStream, RadioBot, RadioBotAddResult } from "@/lib/discord"; import type { DiscordAdminOverview, DiscordStream, RadioBot, RadioBotAddResult } from "@/lib/discord";
const BASE = process.env.NEXT_PUBLIC_C2_URL ?? "http://localhost:8000"; const BASE = process.env.NEXT_PUBLIC_C2_URL ?? "http://localhost:8000";
@@ -498,6 +500,22 @@ export const c2api = {
body: JSON.stringify({ enabled, acknowledge_licence: acknowledgeLicence }), body: JSON.stringify({ enabled, acknowledge_licence: acknowledgeLicence }),
}), }),
// Power outages from every admin-enabled provider inside a bbox (server-26#184).
// `status` explains an empty list: none_enabled / no_source / error / ok.
getOutages: (bbox: { south: number; west: number; north: number; east: number }) =>
request<OutagesResponse>(`/outages?${new URLSearchParams({
south: String(bbox.south), west: String(bbox.west), north: String(bbox.north), east: String(bbox.east),
})}`),
// Admin → Outage providers. Same licence gate as cameras: a gated provider
// answers 409 with `requires_acknowledgement` until `acknowledge_licence` is sent.
listOutageProviders: () => request<{ providers: OutageProviderAdmin[] }>("/admin/outage-providers"),
setOutageProvider: (id: string, enabled: boolean, acknowledgeLicence = false) =>
request<OutageProviderAdmin>(`/admin/outage-providers/${encodeURIComponent(id)}`, {
method: "PUT",
body: JSON.stringify({ enabled, acknowledge_licence: acknowledgeLicence }),
}),
// Public waitlist — no auth, see routers/waitlist.py // Public waitlist — no auth, see routers/waitlist.py
joinWaitlist: (body: { email: string; org_name?: string; note?: string }) => joinWaitlist: (body: { email: string; org_name?: string; note?: string }) =>
request<{ ok: boolean }>("/waitlist", { method: "POST", body: JSON.stringify(body) }), request<{ ok: boolean }>("/waitlist", { method: "POST", body: JSON.stringify(body) }),
+68
View File
@@ -217,6 +217,74 @@ export interface CameraProviderAdmin {
health: { fetched_at: number | null; last_attempt_at: number | null; error: string | null; count: number } | null; health: { fetched_at: number | null; last_attempt_at: number | null; error: string | null; count: number } | null;
} }
// Power outages (server-26#184) — GET /outages, cached server-side, not Firestore.
/**
* One county with active outages from an enabled provider. `id` is
* provider-prefixed ("odin:36067"). Counts are utility-reported meters, used
* as customers. `lat`/`lon` is the centre of the county's bounding box — a
* label point, not an outage location. Times are ISO-8601 UTC.
*/
export interface OutageArea {
id: string;
provider: string;
fips: string;
county: string | null;
state: string | null;
customers_out: number; // live events only
/** Meters on events the source still lists whose own ETR passed >24h ago (ODIN keeps months-old rows). */
stale_customers_out: number;
stale: boolean; // every event in the county is stale: dim or hide, never show as live
customers_served: number | null; // null: no county-level denominator from the source
etr: string | null; // latest estimated restoration across the county's outages
since: string | null; // earliest reported start
updated_at: string | null; // newest time a reporting utility here last sent data
lat: number | null;
lon: number | null;
bbox: [number, number, number, number] | null; // south, west, north, east
utilities: { name: string | null; eia_id: string | null; customers_out: number; stale_customers_out: number }[];
}
export interface OutageProviderFeed extends CameraCredit {
id: string;
name: string;
granularity: string;
coverage_note: string | null;
covers_bbox: boolean;
fetched_at: number | null; // epoch s of the provider's last successful fetch
error: string | null;
total_in_bbox: number;
truncated: boolean;
}
export interface OutagesResponse {
areas: OutageArea[];
providers: OutageProviderFeed[];
serving_count: number;
/** Why the list is what it is — an empty layer is never ambiguous. */
status: "ok" | "none_enabled" | "no_source" | "error";
message: string | null;
}
/** Admin → Outage providers (GET /admin/outage-providers). */
export interface OutageProviderAdmin {
id: string;
name: string;
region: string;
granularity: string;
implemented: boolean;
notes: string;
homepage: string | null;
licence: CameraProviderAdmin["licence"];
enabled: boolean;
serving: boolean;
not_serving_reason: string | null;
acknowledgement: CameraProviderAdmin["acknowledgement"];
updated_at: string | null;
updated_by: { uid: string; email: string } | null;
coverage_note: string | null;
health: CameraProviderAdmin["health"];
}
export interface Ny511Event { export interface Ny511Event {
id: string; id: string;
lat: number; lat: number;