Merge pull request 'Outage providers: licence-gated registry, ODIN adapter, /outages API, admin tab (Refs #184)' (#255) from feat/184-outage-providers into main
This commit was merged in pull request #255.
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
"""Pluggable power-outage providers (map layer, server-26#184).
|
||||
|
||||
Same shape as ``app.internal.cameras``: every source is one adapter carrying
|
||||
its catalogued licence status, every provider defaults OFF, and a licence-gated
|
||||
one serves only after an admin acknowledgement recorded in audit_log
|
||||
(``registry``). Scraped utility storm-center JSON is never shipped -- those
|
||||
sources are stubs until a data-sharing agreement exists.
|
||||
"""
|
||||
@@ -0,0 +1,201 @@
|
||||
"""Outage provider interface, normalised record and the per-provider feed cache.
|
||||
|
||||
The licence model (``Licence``, ``LicenceInfo``, the GATED set) is the camera
|
||||
layer's, imported -- one catalogue vocabulary for every licence-gated source.
|
||||
Only the acknowledgement wording differs (outage data, not imagery), so
|
||||
``OutageLicenceInfo`` overrides ``warning()`` and nothing else.
|
||||
|
||||
Outage feeds are public and identical for every org, so -- like the camera
|
||||
catalogs -- they are cached in memory, fetched lazily on request, and never
|
||||
written to Firestore. A failed refresh keeps serving the last good feed and
|
||||
reports the error and its age: a dead feed must show as an error, never as an
|
||||
empty layer (the AI-silent-failures lesson).
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from app.internal.cameras.base import ( # noqa: F401 (re-exported for adapters)
|
||||
BBox, Licence, LicenceInfo, bbox_intersects, finite, get_json, require_parsed, safe_error,
|
||||
)
|
||||
from app.internal.logger import logger
|
||||
|
||||
RETRY_AFTER_FAILURE_S = 60
|
||||
DEFAULT_FEED_TTL_S = 5 * 60 # ODIN asks utilities for updates every 5-15 min
|
||||
USER_AGENT = "DRB-c2-core/1.0 (+public power-outage data; contact via odin.ornl.gov)"
|
||||
|
||||
OutageArea = Dict[str, Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OutageLicenceInfo(LicenceInfo):
|
||||
"""``LicenceInfo`` with the acknowledgement worded for outage data."""
|
||||
|
||||
def warning(self) -> Optional[str]:
|
||||
if self.status == Licence.PROHIBITED:
|
||||
return f"{self.label}. This source's terms forbid this use; it cannot be enabled."
|
||||
if not self.gated:
|
||||
return None
|
||||
return (f"{self.label}. {self.terms} Enabling it means DRB shows this source's outage data "
|
||||
f"to every signed-in user. Only confirm if DRB actually holds the right to do so.")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OutageProviderMeta:
|
||||
id: str
|
||||
name: str
|
||||
region: str
|
||||
coverage: BBox # rough bbox; a query outside it never triggers a fetch
|
||||
licence: LicenceInfo
|
||||
granularity: str # "county", "point/polygon", ...
|
||||
implemented: bool
|
||||
notes: str # what is verified vs unverified about this adapter
|
||||
homepage: Optional[str] = None
|
||||
ttl_s: int = DEFAULT_FEED_TTL_S
|
||||
|
||||
|
||||
class OutageProvider:
|
||||
"""Base adapter. Subclasses set ``meta`` and implement ``fetch``."""
|
||||
meta: OutageProviderMeta
|
||||
|
||||
async def fetch(self) -> List[OutageArea]:
|
||||
raise NotImplementedError(f"{self.meta.id}: adapter is a stub ({self.meta.notes})")
|
||||
|
||||
def coverage_note(self) -> Optional[str]:
|
||||
"""What this source can and cannot see, shown next to its data."""
|
||||
return None
|
||||
|
||||
|
||||
class StubProvider(OutageProvider):
|
||||
"""A catalogued source with no adapter -- listed so Admin shows its licence
|
||||
status, but it can never be enabled."""
|
||||
|
||||
def __init__(self, meta: OutageProviderMeta):
|
||||
assert not meta.implemented
|
||||
self.meta = meta
|
||||
|
||||
|
||||
class Feed:
|
||||
"""In-memory cache of one provider's normalised outage areas."""
|
||||
|
||||
def __init__(self, provider: OutageProvider):
|
||||
self.provider = provider
|
||||
self.items: List[OutageArea] = []
|
||||
self.fetched_at: Optional[float] = None # epoch s of last SUCCESSFUL fetch
|
||||
self.last_attempt_at: Optional[float] = None
|
||||
self.error: Optional[str] = None
|
||||
self._next_attempt = 0.0
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def get(self) -> "Feed":
|
||||
if time.time() < self._next_attempt:
|
||||
return self
|
||||
async with self._lock:
|
||||
if time.time() < self._next_attempt:
|
||||
return self
|
||||
ttl = self.provider.meta.ttl_s
|
||||
self.last_attempt_at = time.time()
|
||||
try:
|
||||
items = await self.provider.fetch()
|
||||
self.items = items
|
||||
self.fetched_at = time.time()
|
||||
self.error = None
|
||||
self._next_attempt = time.time() + ttl
|
||||
except Exception as e:
|
||||
self._next_attempt = time.time() + min(ttl, RETRY_AFTER_FAILURE_S)
|
||||
self.error = safe_error(e)
|
||||
logger.warning(f"outages/{self.provider.meta.id}: refresh failed, serving {len(self.items)} cached: {self.error}")
|
||||
return self
|
||||
|
||||
def status(self) -> dict:
|
||||
return {"fetched_at": self.fetched_at, "last_attempt_at": self.last_attempt_at,
|
||||
"error": self.error, "count": len(self.items)}
|
||||
|
||||
|
||||
# ── helpers shared by adapters ───────────────────────────────────────────────
|
||||
|
||||
def make_area(provider_id: str, fips: str, *, county: Optional[str], state: Optional[str], customers_out: int,
|
||||
customers_served: Optional[int] = None, etr: Optional[str] = None, since: Optional[str] = None,
|
||||
updated_at: Optional[str] = None, lat: Optional[float] = None, lon: Optional[float] = None,
|
||||
bbox: Optional[BBox] = None, utilities: Optional[List[dict]] = None,
|
||||
stale_customers_out: int = 0) -> OutageArea:
|
||||
"""One normalised county outage record. ``id`` is provider-prefixed.
|
||||
|
||||
``customers_out`` counts only live events. ``stale_customers_out`` counts
|
||||
events the source still lists whose own restoration estimate passed long
|
||||
ago (ODIN carries April rows in October) -- kept visible, never summed in.
|
||||
``stale`` is True when every event in the county is stale."""
|
||||
return {
|
||||
"id": f"{provider_id}:{fips}",
|
||||
"provider": provider_id,
|
||||
"fips": fips, "county": county, "state": state,
|
||||
"customers_out": customers_out,
|
||||
"stale_customers_out": stale_customers_out,
|
||||
"stale": customers_out == 0 and stale_customers_out > 0,
|
||||
"customers_served": customers_served,
|
||||
"etr": etr, # latest estimated restoration across the county's outages, if any
|
||||
"since": since, # earliest reported start across the county's outages
|
||||
"updated_at": updated_at, # newest time a reporting utility in this county last sent data
|
||||
# Approximate: the centre of the county's bounding box, for placing a label.
|
||||
"lat": lat, "lon": lon,
|
||||
"bbox": list(bbox) if bbox else None, # south, west, north, east
|
||||
"utilities": utilities or [],
|
||||
}
|
||||
|
||||
|
||||
def area_in_bbox(area: OutageArea, bbox: BBox) -> bool:
|
||||
"""County bbox intersects the query; falls back to the label point."""
|
||||
b = area.get("bbox")
|
||||
if b and len(b) == 4:
|
||||
return bbox_intersects((b[0], b[1], b[2], b[3]), bbox)
|
||||
lat, lon = area.get("lat"), area.get("lon")
|
||||
if lat is None or lon is None:
|
||||
return False
|
||||
s, w, n, e = bbox
|
||||
return s <= lat <= n and w <= lon <= e
|
||||
|
||||
|
||||
def parse_iso(v: Any) -> Optional[datetime]:
|
||||
"""ISO-8601 (with Z or offset) -> aware UTC datetime; None when unparseable."""
|
||||
if not isinstance(v, str) or not v:
|
||||
return None
|
||||
try:
|
||||
d = datetime.fromisoformat(v.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
return (d if d.tzinfo else d.replace(tzinfo=timezone.utc)).astimezone(timezone.utc)
|
||||
|
||||
|
||||
def iso(d: Optional[datetime]) -> Optional[str]:
|
||||
return d.isoformat().replace("+00:00", "Z") if d else None
|
||||
|
||||
|
||||
def geometry_bbox(geom: Any) -> Optional[BBox]:
|
||||
"""Bounding box (south, west, north, east) of a GeoJSON Polygon/MultiPolygon."""
|
||||
if not isinstance(geom, dict):
|
||||
return None
|
||||
lats: List[float] = []
|
||||
lons: List[float] = []
|
||||
|
||||
def walk(c: Any) -> None:
|
||||
if isinstance(c, list) and len(c) >= 2 and not isinstance(c[0], list):
|
||||
lon, lat = finite(c[0]), finite(c[1])
|
||||
if lon is not None and lat is not None and -180 <= lon <= 180 and -90 <= lat <= 90:
|
||||
lons.append(lon)
|
||||
lats.append(lat)
|
||||
elif isinstance(c, list):
|
||||
for x in c:
|
||||
walk(x)
|
||||
|
||||
walk(geom.get("coordinates"))
|
||||
if not lats:
|
||||
return None
|
||||
return (min(lats), min(lons), max(lats), max(lons))
|
||||
|
||||
|
||||
def bbox_centre(b: Optional[BBox]) -> Tuple[Optional[float], Optional[float]]:
|
||||
if not b:
|
||||
return None, None
|
||||
return (round((b[0] + b[2]) / 2, 5), round((b[1] + b[3]) / 2, 5))
|
||||
@@ -0,0 +1,273 @@
|
||||
"""Outage sources -- server-26#184 (source-terms table in its 2026-10-05 comment).
|
||||
|
||||
Only ODIN has an adapter. Every other source is listed so Admin shows its
|
||||
licence status, and is a stub because DRB holds no right to use it: the utility
|
||||
storm-center maps are undocumented JSON with no reuse licence (the panel ruling
|
||||
stands -- do not ship scraped utility data commercially), PowerOutage.us
|
||||
licenses its API for personal/non-commercial use only, and EAGLE-I's live feed
|
||||
is a DOE partner system. Do not "fix" a stub by scraping.
|
||||
|
||||
ODIN, verified first-hand 2026-10-05 against https://odin.ornl.gov/v3/api-docs
|
||||
(the OpenAPI document the ODIN developer guide points at) and live responses:
|
||||
|
||||
GET /odi?format=JSON public, no token. ``{"outage": [CIM Outage, ...]}``,
|
||||
one row per outage event, every row aggregated to a
|
||||
county: ``communityDescriptor`` = 5-digit county FIPS,
|
||||
``metersAffected``, ``reportedStartTime``,
|
||||
``estimatedRestorationTime.ert``, ``names`` carrying
|
||||
``UtilityName`` / ``UtilityId`` (EIA id).
|
||||
GET /odi/map?scale=COUNTY public. One row per county with active outages:
|
||||
``id`` (FIPS), ``countyName``, ``stateName``,
|
||||
``metersOut`` (= sum of /odi metersAffected, checked),
|
||||
``feature`` (county Polygon/MultiPolygon).
|
||||
GET /odi/status public. One row per reporting utility: ``eiaId``,
|
||||
``name``, ``receivedDate``, ``hasError``.
|
||||
|
||||
``metersServed`` on /odi/map is the utility's system-wide total (National Grid
|
||||
NY carries 1,686,884 on every county), so it is NOT a county denominator and
|
||||
``customers_served`` stays None.
|
||||
"""
|
||||
import asyncio
|
||||
import re
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from app.internal.outages.base import (
|
||||
Licence, OutageArea, OutageLicenceInfo, OutageProvider, OutageProviderMeta, StubProvider, USER_AGENT,
|
||||
bbox_centre, finite, geometry_bbox, get_json, iso, make_area, parse_iso, require_parsed,
|
||||
)
|
||||
|
||||
US_BBOX = (17.5, -180.0, 72.0, -64.0) # CONUS + AK + HI + PR, rough
|
||||
NY_BBOX = (40.4, -80.0, 45.1, -71.7)
|
||||
|
||||
ODIN_BASE = "https://odin.ornl.gov"
|
||||
ODIN_OPENAPI = "https://odin.ornl.gov/v3/api-docs"
|
||||
_FIPS = re.compile(r"^\d{5}$")
|
||||
# An event whose own estimated restoration passed this long ago is stale: ODIN
|
||||
# still listed Avangrid rows from April 2026 (ETR April/May) on 2026-10-05.
|
||||
# Those are not live outages and must not be summed into one or set "since".
|
||||
STALE_AFTER_ETR = timedelta(hours=24)
|
||||
|
||||
|
||||
class OdinProvider(OutageProvider):
|
||||
meta = OutageProviderMeta(
|
||||
id="odin",
|
||||
name="ODIN (Oak Ridge National Laboratory / DOE)",
|
||||
region="United States -- participating utilities only (NY: National Grid, Central Hudson, Avangrid reporting 2026-10-05)",
|
||||
coverage=US_BBOX,
|
||||
licence=OutageLicenceInfo(
|
||||
status=Licence.UNVERIFIED,
|
||||
label="Licence unverified -- public county-level API, but no terms of use or reuse licence found",
|
||||
terms=("ODIN documents GET /odi, /odi/map and /odi/status as public, non-authenticated endpoints "
|
||||
"returning current outages aggregated to county FIPS (OpenAPI read first-hand 2026-10-05). "
|
||||
"That document carries no licence or terms-of-service field, and no terms-of-use page was "
|
||||
"found on odin.ornl.gov. ORNL is operated for DOE by a contractor and the data is "
|
||||
"utility-reported, so federal public-domain status cannot be assumed. Confirm reuse terms "
|
||||
"with ODIN (odin.ornl.gov/pages/contact.html) before relying on it commercially."),
|
||||
terms_url=ODIN_OPENAPI,
|
||||
attribution="Outage data: ODIN, Oak Ridge National Laboratory (utility-reported)",
|
||||
attribution_url="https://odin.ornl.gov",
|
||||
disclaimer=("County-level counts of meters reported out by utilities that participate in ODIN. "
|
||||
"A utility that does not report to ODIN shows no outages here, even during one. "
|
||||
"Restoration times are the utility's estimate, when given."),
|
||||
issue="server-26#184",
|
||||
),
|
||||
granularity="county (FIPS)",
|
||||
implemented=True,
|
||||
notes=("Public API, verified first-hand 2026-10-05: /odi?format=JSON (counts, start, ETR, utility EIA id), "
|
||||
"/odi/map?scale=COUNTY (county name, state, county polygon -> bbox and label point) and /odi/status "
|
||||
"(utility last-received time). Counts are meters, used as customers. Con Edison / O&R are on ODIN's "
|
||||
"participant list but were NOT reporting on 2026-10-05, so NYC/Westchester show nothing from ODIN "
|
||||
"until they do. Schema canary: rows upstream but none parseable is an error, never an empty layer."),
|
||||
homepage="https://odin.ornl.gov",
|
||||
)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.reporting_utilities: Optional[int] = None
|
||||
self.utilities_with_error: Optional[int] = None
|
||||
|
||||
async def fetch(self) -> List[OutageArea]:
|
||||
h = {"User-Agent": USER_AGENT}
|
||||
odi, county_map, status = await asyncio.gather(
|
||||
get_json(f"{ODIN_BASE}/odi", params={"format": "JSON"}, headers=h),
|
||||
get_json(f"{ODIN_BASE}/odi/map", params={"scale": "COUNTY"}, headers=h),
|
||||
get_json(f"{ODIN_BASE}/odi/status", headers=h),
|
||||
)
|
||||
areas, info = parse_odin(odi, county_map, status)
|
||||
self.reporting_utilities = info["reporting_utilities"]
|
||||
self.utilities_with_error = info["utilities_with_error"]
|
||||
return areas
|
||||
|
||||
def coverage_note(self) -> Optional[str]:
|
||||
n = self.reporting_utilities
|
||||
base = "Only utilities that report to ODIN appear; one outside ODIN shows no outages here even during one."
|
||||
if n is None:
|
||||
return base
|
||||
err = f", {self.utilities_with_error} with a feed error" if self.utilities_with_error else ""
|
||||
return f"{base} {n} utilities reporting nationwide{err}."
|
||||
|
||||
|
||||
def _utility(names: object) -> Tuple[Optional[str], Optional[str]]:
|
||||
name = eia = None
|
||||
for n in names if isinstance(names, list) else []:
|
||||
if not isinstance(n, dict):
|
||||
continue
|
||||
if n.get("nameType") == "UtilityName" and isinstance(n.get("name"), str):
|
||||
name = n["name"]
|
||||
elif n.get("nameType") == "UtilityId" and n.get("name") is not None:
|
||||
eia = str(n["name"])
|
||||
return name, eia
|
||||
|
||||
|
||||
def parse_odin(odi: object, county_map: object, status: object,
|
||||
now: Optional[datetime] = None) -> Tuple[List[OutageArea], dict]:
|
||||
"""The three public ODIN responses -> one normalised record per county.
|
||||
|
||||
Events whose estimated restoration is more than ``STALE_AFTER_ETR`` before
|
||||
``now`` are counted separately (``stale_customers_out``), not as live.
|
||||
|
||||
Raises ValueError when a payload has the wrong top-level shape or when rows
|
||||
came back but none could be read -- an upstream schema change must surface
|
||||
as a feed error, not as an empty map."""
|
||||
if not isinstance(odi, dict) or not isinstance(odi.get("outage"), list):
|
||||
raise ValueError("ODIN /odi: expected {'outage': [...]} -- upstream schema change?")
|
||||
if not isinstance(county_map, list):
|
||||
raise ValueError("ODIN /odi/map: expected a list -- upstream schema change?")
|
||||
if not isinstance(status, list):
|
||||
raise ValueError("ODIN /odi/status: expected a list -- upstream schema change?")
|
||||
|
||||
# Utility freshness, by EIA id.
|
||||
received: Dict[str, str] = {}
|
||||
errors = 0
|
||||
for u in status:
|
||||
if not isinstance(u, dict):
|
||||
continue
|
||||
errors += 1 if u.get("hasError") is True else 0
|
||||
eia, d = u.get("eiaId"), parse_iso(u.get("receivedDate"))
|
||||
if eia is not None and d is not None:
|
||||
received[str(eia)] = iso(d)
|
||||
require_parsed(len(status), list(received), "ODIN /odi/status")
|
||||
|
||||
# County geography.
|
||||
geo: Dict[str, dict] = {}
|
||||
for row in county_map:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
fips = str(row.get("id") or "")
|
||||
if not _FIPS.match(fips):
|
||||
continue
|
||||
b = geometry_bbox(row.get("feature"))
|
||||
geo[fips] = {"county": row.get("countyName") if isinstance(row.get("countyName"), str) else None,
|
||||
"state": row.get("stateName") if isinstance(row.get("stateName"), str) else None,
|
||||
"bbox": b}
|
||||
require_parsed(len(county_map), list(geo), "ODIN /odi/map county")
|
||||
|
||||
# Outage events, aggregated per county.
|
||||
stale_before = (now or datetime.now(timezone.utc)) - STALE_AFTER_ETR
|
||||
agg: Dict[str, dict] = {}
|
||||
parsed = stale_events = 0
|
||||
for o in odi["outage"]:
|
||||
if not isinstance(o, dict):
|
||||
continue
|
||||
fips = str(o.get("communityDescriptor") or "")
|
||||
meters = finite(o.get("metersAffected"))
|
||||
if not _FIPS.match(fips) or meters is None or meters < 0:
|
||||
continue
|
||||
parsed += 1
|
||||
area = o.get("outageArea") if isinstance(o.get("outageArea"), dict) else {}
|
||||
start = parse_iso(o.get("reportedStartTime")) or parse_iso(area.get("earliestReportedTime"))
|
||||
ert_obj = o.get("estimatedRestorationTime") if isinstance(o.get("estimatedRestorationTime"), dict) else {}
|
||||
ert = parse_iso(ert_obj.get("ert"))
|
||||
if ert is not None and ert.year < 2000: # Puget Sound Energy sends 0001-01-01T00:00:00Z for "no ETR"
|
||||
ert = None
|
||||
stale = ert is not None and ert < stale_before
|
||||
stale_events += stale
|
||||
a = agg.setdefault(fips, {"out": 0, "stale_out": 0, "utilities": {},
|
||||
"live": {"since": None, "etr": None}, "stale": {"since": None, "etr": None}})
|
||||
a["stale_out" if stale else "out"] += int(meters)
|
||||
t = a["stale" if stale else "live"] # since/etr come from live events whenever there are any
|
||||
if start and (t["since"] is None or start < t["since"]):
|
||||
t["since"] = start
|
||||
if ert and (t["etr"] is None or ert > t["etr"]):
|
||||
t["etr"] = ert
|
||||
name, eia = _utility(o.get("names"))
|
||||
key = eia or name or "unknown"
|
||||
u = a["utilities"].setdefault(key, {"name": name, "eia_id": eia, "customers_out": 0,
|
||||
"stale_customers_out": 0})
|
||||
u["stale_customers_out" if stale else "customers_out"] += int(meters)
|
||||
require_parsed(len(odi["outage"]), [None] * parsed, "ODIN /odi outage")
|
||||
|
||||
areas: List[OutageArea] = []
|
||||
for fips, a in agg.items():
|
||||
g = geo.get(fips, {})
|
||||
utilities = sorted(a["utilities"].values(), key=lambda u: (-u["customers_out"], -u["stale_customers_out"]))
|
||||
stamps = [received[u["eia_id"]] for u in utilities if u["eia_id"] in received]
|
||||
lat, lon = bbox_centre(g.get("bbox"))
|
||||
t = a["live"] if a["out"] else a["stale"]
|
||||
areas.append(make_area(
|
||||
"odin", fips, county=g.get("county"), state=g.get("state"), customers_out=a["out"],
|
||||
stale_customers_out=a["stale_out"], customers_served=None, etr=iso(t["etr"]), since=iso(t["since"]),
|
||||
updated_at=max(stamps) if stamps else None, lat=lat, lon=lon, bbox=g.get("bbox"),
|
||||
utilities=utilities,
|
||||
))
|
||||
areas.sort(key=lambda r: (-r["customers_out"], -r["stale_customers_out"]))
|
||||
return areas, {"reporting_utilities": len(status), "utilities_with_error": errors, "stale_events": stale_events}
|
||||
|
||||
|
||||
# ── stubs: catalogued, never enable-able ─────────────────────────────────────
|
||||
|
||||
_STORM_CENTER_TERMS = ("The utility's public outage map is backed by undocumented JSON (vendor storm-center "
|
||||
"endpoints) with no published licence for automated reuse; the endpoints can change "
|
||||
"without notice. Panel ruling (server-26#184): do not ship scraped utility data "
|
||||
"commercially. A direct data-sharing agreement is the licensed path (also a sales lead).")
|
||||
|
||||
|
||||
def _stub(id: str, name: str, region: str, coverage, status: Licence, label: str, terms: str,
|
||||
terms_url: Optional[str], granularity: str, notes: str, homepage: Optional[str]) -> StubProvider:
|
||||
return StubProvider(OutageProviderMeta(
|
||||
id=id, name=name, region=region, coverage=coverage,
|
||||
licence=OutageLicenceInfo(status=status, label=label, terms=terms, terms_url=terms_url,
|
||||
issue="server-26#184"),
|
||||
granularity=granularity, implemented=False, notes=notes, homepage=homepage,
|
||||
))
|
||||
|
||||
|
||||
PROVIDERS: List[OutageProvider] = [
|
||||
OdinProvider(),
|
||||
_stub("poweroutage_us", "PowerOutage.us (FE Bluefire LLC)", "United States", US_BBOX, Licence.PAID,
|
||||
"Paid API -- licence is personal/non-commercial (news media excepted); commercial use needs an agreement DRB does not hold",
|
||||
("API & Content Terms: the licence is for \"personal and noncommercial use (except for News Media "
|
||||
"Organizations)\"; commercial use needs a separate agreement; pricing by contact. Quoted from a "
|
||||
"search summary -- the terms page returned 403 to a direct fetch, so re-read it first-hand before "
|
||||
"any acknowledgement."),
|
||||
"https://poweroutage.us/legal/apitermsofuse", "county / utility",
|
||||
"No adapter: needs a commercial contract and its API documentation.", "https://poweroutage.us"),
|
||||
_stub("coned_oru_stormcenter", "Con Edison / Orange & Rockland outage map", "NYC, Westchester, Orange & Rockland",
|
||||
NY_BBOX, Licence.AGREEMENT_REQUIRED,
|
||||
"Requires a data-sharing agreement -- undocumented storm-center JSON, no reuse licence",
|
||||
_STORM_CENTER_TERMS, "https://www.coned.com/en/services-and-outages/outage-map-faq", "point / polygon + ETR",
|
||||
("No adapter by design. Con Edison and O&R are on ODIN's participant list but were not reporting to "
|
||||
"ODIN on 2026-10-05; if they start, the ODIN adapter covers them at county level."),
|
||||
"https://outagemap.coned.com"),
|
||||
_stub("nyseg_rge_stormcenter", "NYSEG / RG&E outage map (Avangrid)", "Upstate NY", NY_BBOX,
|
||||
Licence.AGREEMENT_REQUIRED,
|
||||
"Requires a data-sharing agreement -- undocumented storm-center JSON, no reuse licence",
|
||||
_STORM_CENTER_TERMS, None, "point / polygon + ETR",
|
||||
("No adapter by design. Avangrid (NYSEG/RG&E) reports to ODIN, so county-level data arrives through "
|
||||
"the ODIN adapter."),
|
||||
"https://www.nyseg.com"),
|
||||
_stub("central_hudson_stormcenter", "Central Hudson outage map", "Hudson Valley, NY", NY_BBOX,
|
||||
Licence.AGREEMENT_REQUIRED,
|
||||
"Requires a data-sharing agreement -- undocumented storm-center JSON, no reuse licence",
|
||||
_STORM_CENTER_TERMS, None, "point / polygon + ETR",
|
||||
("No adapter by design. Central Hudson reports to ODIN, so county-level data arrives through the "
|
||||
"ODIN adapter."),
|
||||
"https://www.cenhud.com"),
|
||||
_stub("eagle_i_live", "EAGLE-I live feed (ORNL / DOE)", "United States", US_BBOX, Licence.AGREEMENT_REQUIRED,
|
||||
"Live feed is a DOE / government-partner system -- DRB has no access agreement",
|
||||
("EAGLE-I's live 15-minute county outage feed is restricted to DOE and government partners. The public "
|
||||
"ORNL Constellation dataset is historic only (no terms stated on the dataset) and is no use for a "
|
||||
"live map."),
|
||||
"https://doi.ccs.ornl.gov/dataset/d26451af-7ea4-577c-8285-78b72cd8a2dc", "county",
|
||||
"No adapter: needs a DOE partner agreement.", "https://eagle-i.doe.gov"),
|
||||
]
|
||||
@@ -0,0 +1,216 @@
|
||||
"""Outage provider registry: settings, licence gate, and bbox queries.
|
||||
|
||||
Mirrors ``app.internal.cameras.registry`` rule for rule. Settings live in
|
||||
Firestore ``config/outage_providers``::
|
||||
|
||||
{"providers": {"<id>": {"enabled": bool, "updated_at": iso, "updated_by": {...},
|
||||
"acknowledgement": {"by_uid", "by_email", "at", "licence_status", "warning"} | None}}}
|
||||
|
||||
Rules (owner decision #247 on server-26#184):
|
||||
* every provider defaults OFF -- a missing doc/entry means off, and an
|
||||
unreadable settings doc means everything is off (fail closed);
|
||||
* a stub (no adapter) or a PROHIBITED source can never be enabled;
|
||||
* a GATED licence (agreement required / non-commercial / unverified / paid)
|
||||
can only be enabled with an explicit acknowledgement, recorded with who and
|
||||
when in the settings doc AND audit_log. The audit write is a precondition
|
||||
for a gated enable: no record, no enable;
|
||||
* an acknowledgement is bound to the licence status it was given for. If a
|
||||
provider's catalogued status changes in code, the old acknowledgement no
|
||||
longer counts and the provider stops serving until re-acknowledged.
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from app.internal import firestore as fstore
|
||||
from app.internal.logger import logger
|
||||
from app.internal.outages import providers as sources
|
||||
from app.internal.outages.base import BBox, Feed, Licence, OutageArea, OutageProvider, area_in_bbox, bbox_intersects
|
||||
|
||||
COLLECTION, DOC_ID = "config", "outage_providers"
|
||||
_TTL_S = 30.0
|
||||
MAX_AREAS = 3000 # per response; ODIN's whole country is ~100-300 counties on a normal day
|
||||
|
||||
PROVIDERS: Dict[str, OutageProvider] = {p.meta.id: p for p in sources.PROVIDERS}
|
||||
FEEDS: Dict[str, Feed] = {pid: Feed(p) for pid, p in PROVIDERS.items() if p.meta.implemented}
|
||||
|
||||
NO_SOURCE_MESSAGE = "No licensed outage source covers this area."
|
||||
|
||||
_cache: Dict[str, dict] = {}
|
||||
_cache_ts = 0.0
|
||||
|
||||
|
||||
class ProviderError(Exception):
|
||||
"""A refused settings change. ``code`` is the HTTP status the router returns."""
|
||||
|
||||
def __init__(self, code: int, message: str, **extra):
|
||||
super().__init__(message)
|
||||
self.code, self.message, self.extra = code, message, extra
|
||||
|
||||
|
||||
def invalidate_cache() -> None:
|
||||
global _cache_ts
|
||||
_cache_ts = 0.0
|
||||
|
||||
|
||||
async def get_settings() -> Dict[str, dict]:
|
||||
"""Per-provider settings. Unreadable Firestore means everything is OFF (fail closed)."""
|
||||
global _cache, _cache_ts
|
||||
if _cache_ts and time.monotonic() - _cache_ts < _TTL_S:
|
||||
return _cache
|
||||
try:
|
||||
doc = await fstore.doc_get(COLLECTION, DOC_ID) or {}
|
||||
provs = doc.get("providers") if isinstance(doc.get("providers"), dict) else {}
|
||||
_cache = {k: v for k, v in provs.items() if isinstance(v, dict)}
|
||||
except Exception as e:
|
||||
logger.warning(f"outage providers: settings unreadable ({e}); treating every provider as OFF")
|
||||
_cache = {}
|
||||
_cache_ts = time.monotonic()
|
||||
return _cache
|
||||
|
||||
|
||||
def effective(provider: OutageProvider, entry: Optional[dict]) -> Tuple[bool, Optional[str]]:
|
||||
"""(serving?, reason-if-not) for a provider given its settings entry."""
|
||||
meta = provider.meta
|
||||
if not meta.implemented:
|
||||
return False, "adapter is a stub"
|
||||
if meta.licence.status == Licence.PROHIBITED:
|
||||
return False, "source terms forbid this use"
|
||||
if not entry or not entry.get("enabled"):
|
||||
return False, "disabled (default)"
|
||||
if meta.licence.gated:
|
||||
ack = entry.get("acknowledgement") or {}
|
||||
if ack.get("licence_status") != meta.licence.status.value:
|
||||
return False, "licence acknowledgement missing or out of date"
|
||||
return True, None
|
||||
|
||||
|
||||
async def is_serving(provider_id: str) -> bool:
|
||||
p = PROVIDERS.get(provider_id)
|
||||
if p is None:
|
||||
return False
|
||||
return effective(p, (await get_settings()).get(provider_id))[0]
|
||||
|
||||
|
||||
async def set_enabled(provider_id: str, enabled: bool, actor: Tuple[str, str],
|
||||
acknowledge_licence: bool = False) -> dict:
|
||||
p = PROVIDERS.get(provider_id)
|
||||
if p is None:
|
||||
raise ProviderError(404, f"unknown outage provider '{provider_id}'")
|
||||
meta, lic = p.meta, p.meta.licence
|
||||
uid, email = actor
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
entry: dict = {"enabled": bool(enabled), "updated_at": now, "updated_by": {"uid": uid, "email": email}}
|
||||
|
||||
if enabled:
|
||||
if not meta.implemented:
|
||||
raise ProviderError(409, f"{meta.name}: no working adapter -- {meta.notes}")
|
||||
if lic.status == Licence.PROHIBITED:
|
||||
raise ProviderError(409, lic.warning() or "prohibited")
|
||||
if lic.gated:
|
||||
if not acknowledge_licence:
|
||||
raise ProviderError(409, f"{meta.name}: licence acknowledgement required",
|
||||
requires_acknowledgement=True, warning=lic.warning(),
|
||||
licence=lic.as_dict())
|
||||
entry["acknowledgement"] = {"by_uid": uid, "by_email": email, "at": now,
|
||||
"licence_status": lic.status.value, "warning": lic.warning()}
|
||||
# Precondition, not a side effect: a gated enable with no audit
|
||||
# record must not happen. A failure here aborts before the write.
|
||||
try:
|
||||
await _audit(actor, "outage_provider.enable_acknowledged", provider_id,
|
||||
{"licence_status": lic.status.value, "licence_label": lic.label,
|
||||
"warning": lic.warning(), "acknowledged_at": now})
|
||||
except Exception as e:
|
||||
logger.error(f"outage providers: audit write failed, refusing gated enable of {provider_id} ({e})")
|
||||
raise ProviderError(503, "audit log unavailable -- a licence-gated provider cannot be enabled "
|
||||
"without an audit record; try again")
|
||||
else:
|
||||
entry["acknowledgement"] = None # re-enabling a gated source needs a fresh acknowledgement
|
||||
|
||||
before = (await get_settings()).get(provider_id) or {}
|
||||
await fstore.doc_set(COLLECTION, DOC_ID, {"providers": {provider_id: entry}}, merge=True)
|
||||
invalidate_cache()
|
||||
if not (enabled and lic.gated):
|
||||
try:
|
||||
await _audit(actor, "outage_provider.enable" if enabled else "outage_provider.disable", provider_id,
|
||||
{"from": bool(before.get("enabled")), "to": bool(enabled),
|
||||
"licence_status": lic.status.value})
|
||||
except Exception as e: # the toggle already happened; never 500 a completed write
|
||||
logger.error(f"outage providers: audit write failed for {provider_id} ({e})")
|
||||
logger.info(f"outage provider {provider_id} -> enabled={enabled} by {email or uid}")
|
||||
return await describe(provider_id)
|
||||
|
||||
|
||||
async def _audit(actor: Tuple[str, str], action: str, provider_id: str, details: dict) -> None:
|
||||
from app.internal import audit
|
||||
await audit.write_audit(actor_uid=actor[0], actor_email=actor[1], action=action,
|
||||
details={"provider_id": provider_id, **details})
|
||||
|
||||
|
||||
async def describe(provider_id: str) -> dict:
|
||||
p = PROVIDERS[provider_id]
|
||||
entry = (await get_settings()).get(provider_id) or {}
|
||||
serving, reason = effective(p, entry)
|
||||
m = p.meta
|
||||
feed = FEEDS.get(provider_id)
|
||||
return {
|
||||
"id": m.id, "name": m.name, "region": m.region, "granularity": m.granularity,
|
||||
"implemented": m.implemented, "notes": m.notes, "homepage": m.homepage,
|
||||
"licence": m.licence.as_dict(),
|
||||
"enabled": bool(entry.get("enabled")),
|
||||
"serving": serving, "not_serving_reason": reason,
|
||||
"acknowledgement": entry.get("acknowledgement"),
|
||||
"updated_at": entry.get("updated_at"), "updated_by": entry.get("updated_by"),
|
||||
"coverage_note": p.coverage_note(),
|
||||
"health": feed.status() if feed else None,
|
||||
}
|
||||
|
||||
|
||||
async def list_providers() -> List[dict]:
|
||||
await get_settings()
|
||||
return [await describe(pid) for pid in PROVIDERS]
|
||||
|
||||
|
||||
def credit(provider_id: str) -> dict:
|
||||
lic = PROVIDERS[provider_id].meta.licence
|
||||
return {"attribution": lic.attribution, "attribution_url": lic.attribution_url, "disclaimer": lic.disclaimer}
|
||||
|
||||
|
||||
async def outages_in_bbox(bbox: BBox) -> dict:
|
||||
"""Outage areas from every SERVING provider whose coverage meets ``bbox``.
|
||||
Providers that are off are never fetched.
|
||||
|
||||
``status`` says why the list is what it is, so an empty layer is never
|
||||
ambiguous: ``none_enabled`` (no provider is on), ``no_source`` (none of the
|
||||
serving providers covers this area), ``error`` (every covering provider
|
||||
failed and has nothing cached), else ``ok`` -- which may legitimately
|
||||
carry zero areas when nothing is out."""
|
||||
conf = await get_settings()
|
||||
serving = [pid for pid in FEEDS if effective(PROVIDERS[pid], conf.get(pid))[0]]
|
||||
live = [pid for pid in serving if bbox_intersects(PROVIDERS[pid].meta.coverage, bbox)]
|
||||
await asyncio.gather(*(FEEDS[pid].get() for pid in live))
|
||||
areas: List[OutageArea] = []
|
||||
providers = []
|
||||
for pid in serving:
|
||||
feed = FEEDS[pid]
|
||||
hits = [a for a in feed.items if area_in_bbox(a, bbox)] if pid in live else []
|
||||
room = max(0, MAX_AREAS - len(areas))
|
||||
areas.extend(hits[:room])
|
||||
providers.append({"id": pid, "name": PROVIDERS[pid].meta.name, **credit(pid),
|
||||
"granularity": PROVIDERS[pid].meta.granularity,
|
||||
"coverage_note": PROVIDERS[pid].coverage_note(),
|
||||
"covers_bbox": pid in live,
|
||||
"fetched_at": feed.fetched_at, "error": feed.error,
|
||||
"total_in_bbox": len(hits), "truncated": len(hits) > room})
|
||||
|
||||
if not serving:
|
||||
status, message = "none_enabled", "No outage source is enabled. An admin can enable one under Admin -> Outage providers."
|
||||
elif not live:
|
||||
status, message = "no_source", NO_SOURCE_MESSAGE
|
||||
elif all(FEEDS[pid].error and FEEDS[pid].fetched_at is None for pid in live):
|
||||
status, message = "error", "Every outage source covering this area failed to load; see each provider's error."
|
||||
else:
|
||||
status, message = "ok", None
|
||||
return {"areas": areas, "providers": providers, "serving_count": len(serving),
|
||||
"status": status, "message": message}
|
||||
@@ -19,6 +19,7 @@ from app.internal.auth import (
|
||||
)
|
||||
from app.routers import nodes, systems, calls, upload, tokens, incidents, alerts, admin, trips, places, links, users
|
||||
from app.routers import enrollment, media, org, waitlist, telemetry, replay, traffic, cameras
|
||||
from app.routers import outages
|
||||
from app.routers import discord as discord_router
|
||||
from app.routers import discord_link as discord_link_router
|
||||
from app.internal import dynsec
|
||||
@@ -147,6 +148,8 @@ app.include_router(places.router, dependencies=[Depends(require_service_or_fi
|
||||
app.include_router(traffic.router, dependencies=[Depends(require_service_or_firebase_token)])
|
||||
app.include_router(cameras.router, dependencies=[Depends(require_service_or_firebase_token)])
|
||||
app.include_router(cameras.admin_router) # auth: admin only, per endpoint (needs the principal for audit)
|
||||
app.include_router(outages.router, dependencies=[Depends(require_service_or_firebase_token)])
|
||||
app.include_router(outages.admin_router) # auth: admin only, per endpoint (needs the principal for audit)
|
||||
app.include_router(upload.router) # auth is per-node, handled inline
|
||||
app.include_router(admin.router) # auth is per-endpoint (read: firebase, write: admin)
|
||||
app.include_router(replay.router) # auth: admin only (every route spends or reads a replay run)
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Power-outage endpoints (map layer, server-26#184).
|
||||
|
||||
``GET /outages`` -- any signed-in user (auth applied at include time in main.py):
|
||||
outage areas inside a bbox from every provider an admin has enabled, each with
|
||||
the provider's attribution, disclaimer, coverage note and feed freshness, plus
|
||||
a ``status`` that says why an empty list is empty.
|
||||
|
||||
``/admin/outage-providers`` -- platform admin only: list every catalogued
|
||||
provider with its licence status and health, and toggle it. Gated licences
|
||||
need ``acknowledge_licence: true``; that acknowledgement is stored with who and
|
||||
when and written to audit_log (see app/internal/outages/registry.py).
|
||||
"""
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.internal.auth import describe_actor, require_admin_token
|
||||
from app.internal.outages import registry
|
||||
|
||||
router = APIRouter(prefix="/outages", tags=["outages"])
|
||||
admin_router = APIRouter(prefix="/admin/outage-providers", tags=["admin"])
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def get_outages(
|
||||
south: float = Query(..., ge=-90, le=90),
|
||||
west: float = Query(..., ge=-180, le=180),
|
||||
north: float = Query(..., ge=-90, le=90),
|
||||
east: float = Query(..., ge=-180, le=180),
|
||||
):
|
||||
if south > north or west > east:
|
||||
raise HTTPException(400, "bbox must satisfy south<=north and west<=east")
|
||||
return await registry.outages_in_bbox((south, west, north, east))
|
||||
|
||||
|
||||
class ProviderToggle(BaseModel):
|
||||
enabled: bool
|
||||
acknowledge_licence: bool = False
|
||||
|
||||
|
||||
@admin_router.get("")
|
||||
async def list_outage_providers(_=Depends(require_admin_token)):
|
||||
return {"providers": await registry.list_providers()}
|
||||
|
||||
|
||||
@admin_router.put("/{provider_id}")
|
||||
async def set_outage_provider(provider_id: str, body: ProviderToggle, principal: dict = Depends(require_admin_token)):
|
||||
try:
|
||||
return await registry.set_enabled(provider_id, body.enabled, describe_actor(principal),
|
||||
acknowledge_licence=body.acknowledge_licence)
|
||||
except registry.ProviderError as e:
|
||||
return JSONResponse(status_code=e.code, content={"detail": e.message, **e.extra})
|
||||
Reference in New Issue
Block a user