logan/server-26
1
0
Fork 0
Code Issues 84 Pull Requests Actions Packages Projects Releases Wiki Activity

Review: ODIN stale events and 0001-01-01 no-ETR sentinel (Refs #184)

Browse Source 
ODIN still lists Avangrid rows from April 2026 (ETR April/May) as active on
2026-10-05; parse_odin now counts events whose own ETR passed >24h ago as
stale_customers_out, not customers_out, and since/etr come from live events.
Puget Sound Energy sends 0001-01-01T00:00:00Z for no ETR; treat as None.
Ack dialog now shows a refused enable's error instead of hiding it behind
the overlay.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Logan Cusano
2026-10-05 02:45:25 -04:00
co-authored by Claude Opus 5.5
parent ebeabe342e
commit 0efcd676c3
5 changed files with 76 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
drb-c2-core/app/internal/outages/base.py
+10 -2
View File
@@ -119,13 +119,21 @@ class Feed:
def make_area(provider_id: str, fips: str, *, county: Optional[str], state: Optional[str], customers_out: int,
customers_served: Optional[int] = None, etr: Optional[str] = None, since: Optional[str] = None,
updated_at: Optional[str] = None, lat: Optional[float] = None, lon: Optional[float] = None,
bbox: Optional[BBox] = None, utilities: Optional[List[dict]] = None) -> OutageArea:
"""One normalised county outage record. ``id`` is provider-prefixed."""
bbox: Optional[BBox] = None, utilities: Optional[List[dict]] = None,
stale_customers_out: int = 0) -> OutageArea:
"""One normalised county outage record. ``id`` is provider-prefixed.
``customers_out`` counts only live events. ``stale_customers_out`` counts
events the source still lists whose own restoration estimate passed long
ago (ODIN carries April rows in October) -- kept visible, never summed in.
``stale`` is True when every event in the county is stale."""
return {
"id": f"{provider_id}:{fips}",
"provider": provider_id,
"fips": fips, "county": county, "state": state,
"customers_out": customers_out,
"stale_customers_out": stale_customers_out,
"stale": customers_out == 0 and stale_customers_out > 0,
"customers_served": customers_served,
"etr": etr, # latest estimated restoration across the county's outages, if any
"since": since, # earliest reported start across the county's outages
drb-c2-core/app/internal/outages/providers.py
+32 -14
View File
@@ -29,6 +29,7 @@ NY carries 1,686,884 on every county), so it is NOT a county denominator and
"""
import asyncio
import re
from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional, Tuple
from app.internal.outages.base import (
@@ -42,6 +43,10 @@ NY_BBOX = (40.4, -80.0, 45.1, -71.7)
ODIN_BASE = "https://odin.ornl.gov"
ODIN_OPENAPI = "https://odin.ornl.gov/v3/api-docs"
_FIPS = re.compile(r"^\d{5}$")
# An event whose own estimated restoration passed this long ago is stale: ODIN
# still listed Avangrid rows from April 2026 (ETR April/May) on 2026-10-05.
# Those are not live outages and must not be summed into one or set "since".
STALE_AFTER_ETR = timedelta(hours=24)
class OdinProvider(OutageProvider):
@@ -114,9 +119,13 @@ def _utility(names: object) -> Tuple[Optional[str], Optional[str]]:
return name, eia
def parse_odin(odi: object, county_map: object, status: object) -> Tuple[List[OutageArea], dict]:
def parse_odin(odi: object, county_map: object, status: object,
now: Optional[datetime] = None) -> Tuple[List[OutageArea], dict]:
"""The three public ODIN responses -> one normalised record per county.
Events whose estimated restoration is more than ``STALE_AFTER_ETR`` before
``now`` are counted separately (``stale_customers_out``), not as live.
Raises ValueError when a payload has the wrong top-level shape or when rows
came back but none could be read -- an upstream schema change must surface
as a feed error, not as an empty map."""
@@ -154,8 +163,9 @@ def parse_odin(odi: object, county_map: object, status: object) -> Tuple[List[Ou
require_parsed(len(county_map), list(geo), "ODIN /odi/map county")
# Outage events, aggregated per county.
stale_before = (now or datetime.now(timezone.utc)) - STALE_AFTER_ETR
agg: Dict[str, dict] = {}
parsed = 0
parsed = stale_events = 0
for o in odi["outage"]:
if not isinstance(o, dict):
continue
@@ -164,36 +174,44 @@ def parse_odin(odi: object, county_map: object, status: object) -> Tuple[List[Ou
if not _FIPS.match(fips) or meters is None or meters < 0:
continue
parsed += 1
a = agg.setdefault(fips, {"out": 0, "since": None, "etr": None, "utilities": {}})
a["out"] += int(meters)
area = o.get("outageArea") if isinstance(o.get("outageArea"), dict) else {}
start = parse_iso(o.get("reportedStartTime")) or parse_iso(area.get("earliestReportedTime"))
if start and (a["since"] is None or start < a["since"]):
a["since"] = start
ert_obj = o.get("estimatedRestorationTime") if isinstance(o.get("estimatedRestorationTime"), dict) else {}
ert = parse_iso(ert_obj.get("ert"))
if ert and (a["etr"] is None or ert > a["etr"]):
a["etr"] = ert
if ert is not None and ert.year < 2000: # Puget Sound Energy sends 0001-01-01T00:00:00Z for "no ETR"
ert = None
stale = ert is not None and ert < stale_before
stale_events += stale
a = agg.setdefault(fips, {"out": 0, "stale_out": 0, "utilities": {},
"live": {"since": None, "etr": None}, "stale": {"since": None, "etr": None}})
a["stale_out" if stale else "out"] += int(meters)
t = a["stale" if stale else "live"] # since/etr come from live events whenever there are any
if start and (t["since"] is None or start < t["since"]):
t["since"] = start
if ert and (t["etr"] is None or ert > t["etr"]):
t["etr"] = ert
name, eia = _utility(o.get("names"))
key = eia or name or "unknown"
u = a["utilities"].setdefault(key, {"name": name, "eia_id": eia, "customers_out": 0})
u["customers_out"] += int(meters)
u = a["utilities"].setdefault(key, {"name": name, "eia_id": eia, "customers_out": 0,
"stale_customers_out": 0})
u["stale_customers_out" if stale else "customers_out"] += int(meters)
require_parsed(len(odi["outage"]), [None] * parsed, "ODIN /odi outage")
areas: List[OutageArea] = []
for fips, a in agg.items():
g = geo.get(fips, {})
utilities = sorted(a["utilities"].values(), key=lambda u: -u["customers_out"])
utilities = sorted(a["utilities"].values(), key=lambda u: (-u["customers_out"], -u["stale_customers_out"]))
stamps = [received[u["eia_id"]] for u in utilities if u["eia_id"] in received]
lat, lon = bbox_centre(g.get("bbox"))
t = a["live"] if a["out"] else a["stale"]
areas.append(make_area(
"odin", fips, county=g.get("county"), state=g.get("state"), customers_out=a["out"],
customers_served=None, etr=iso(a["etr"]), since=iso(a["since"]),
stale_customers_out=a["stale_out"], customers_served=None, etr=iso(t["etr"]), since=iso(t["since"]),
updated_at=max(stamps) if stamps else None, lat=lat, lon=lon, bbox=g.get("bbox"),
utilities=utilities,
))
areas.sort(key=lambda r: -r["customers_out"])
return areas, {"reporting_utilities": len(status), "utilities_with_error": errors}
areas.sort(key=lambda r: (-r["customers_out"], -r["stale_customers_out"]))
return areas, {"reporting_utilities": len(status), "utilities_with_error": errors, "stale_events": stale_events}
# ── stubs: catalogued, never enable-able ─────────────────────────────────────
drb-c2-core/tests/test_outage_providers.py
+27 -3
View File
@@ -13,6 +13,7 @@ What is held here (mirrors tests/test_camera_providers.py):
"""
import asyncio
import json
from datetime import datetime, timezone
from pathlib import Path
from unittest.mock import AsyncMock, patch
@@ -30,6 +31,7 @@ ADMIN = {"uid": "admin-1", "email": "admin@example.com", "role": "admin"}
FIXTURE = json.loads((Path(__file__).parent / "fixtures" / "odin_public_2026-10-05.json").read_text())
STUBS = ("poweroutage_us", "coned_oru_stormcenter", "nyseg_rge_stormcenter", "central_hudson_stormcenter", "eagle_i_live")
NY_BBOX = (42.5, -78.5, 43.5, -75.5) # Rochester .. Syracuse
RECORDED_AT = datetime(2026, 10, 5, 6, 45, tzinfo=timezone.utc) # when the fixture was captured
class FakeStore:
@@ -70,7 +72,8 @@ def run(coro):
def _odin_areas():
return providers.parse_odin(FIXTURE["odi_format_json"], FIXTURE["odi_map_scale_county"], FIXTURE["odi_status"])[0]
return providers.parse_odin(FIXTURE["odi_format_json"], FIXTURE["odi_map_scale_county"], FIXTURE["odi_status"],
now=RECORDED_AT)[0]
# ── registry / gate ──────────────────────────────────────────────────────────
@@ -244,7 +247,8 @@ def test_outages_route_requires_sign_in():
# ── ODIN adapter (recorded public payloads, 2026-10-05) ──────────────────────
def test_odin_parser_reads_recorded_payload():
areas, info = providers.parse_odin(FIXTURE["odi_format_json"], FIXTURE["odi_map_scale_county"], FIXTURE["odi_status"])
areas, info = providers.parse_odin(FIXTURE["odi_format_json"], FIXTURE["odi_map_scale_county"], FIXTURE["odi_status"],
now=RECORDED_AT)
by = {a["fips"]: a for a in areas}
assert set(by) == {"36001", "36067", "36055", "53033"}
onondaga = by["36067"]
@@ -253,7 +257,9 @@ def test_odin_parser_reads_recorded_payload():
assert onondaga["customers_out"] == 11 and onondaga["customers_served"] is None
assert onondaga["etr"] == "2026-10-05T08:00:00Z" and onondaga["since"] == "2026-10-04T18:35:40Z"
assert onondaga["updated_at"].startswith("2026-10-05T06:20:47")
assert onondaga["utilities"] == [{"name": "NATIONAL GRID", "eia_id": "13573", "customers_out": 11}]
assert onondaga["utilities"] == [{"name": "NATIONAL GRID", "eia_id": "13573", "customers_out": 11,
"stale_customers_out": 0}]
assert onondaga["stale"] is False and onondaga["stale_customers_out"] == 0
s, w, n, e = onondaga["bbox"]
assert s < onondaga["lat"] < n and w < onondaga["lon"] < e
assert 42.7 < onondaga["lat"] < 43.3 and -76.6 < onondaga["lon"] < -75.8
@@ -261,9 +267,27 @@ def test_odin_parser_reads_recorded_payload():
king = by["53033"]
assert king["customers_out"] == sum(r["metersOut"] for r in FIXTURE["odi_map_scale_county"] if r["id"] == "53033")
assert len(king["utilities"]) == 2
# Puget Sound Energy's "0001-01-01T00:00:00Z" means no ETR -- not a restoration time, not stale.
assert king["etr"] is None and king["stale_customers_out"] == 0
assert info["reporting_utilities"] == len(FIXTURE["odi_status"]) and info["utilities_with_error"] == 1
def test_odin_events_long_past_their_etr_are_stale_not_live():
# ODIN still listed Avangrid's Monroe County row from April 2026 (ETR
# 2026-04-10) on 2026-10-05. It must not read as 18 customers out right now.
by = {a["fips"]: a for a in _odin_areas()}
monroe = by["36055"]
assert monroe["customers_out"] == 0 and monroe["stale_customers_out"] == 18 and monroe["stale"] is True
assert monroe["since"].startswith("2026-04-10") # still shown, so the stale row is explainable
# Same payload, captured the day the outage was live: not stale.
live = {a["fips"]: a for a in providers.parse_odin(
FIXTURE["odi_format_json"], FIXTURE["odi_map_scale_county"], FIXTURE["odi_status"],
now=datetime(2026, 4, 10, 20, 0, tzinfo=timezone.utc))[0]}
assert live["36055"]["customers_out"] == 18 and live["36055"]["stale"] is False
# Live counties sort ahead of stale ones.
assert [a["stale"] for a in _odin_areas()] == sorted(a["stale"] for a in _odin_areas())
def test_odin_fetch_uses_the_three_public_endpoints():
calls = []
drb-frontend/components/admin/OutageProvidersTab.tsx
+2
View File
@@ -40,6 +40,7 @@ export function OutageProvidersTab() {
async function toggle(p: OutageProviderAdmin, enabled: boolean, acknowledge = false) {
if (enabled && p.licence.gated && !acknowledge) {
setError(null);
setAckChecked(false);
setConfirming(p);
return;
@@ -137,6 +138,7 @@ export function OutageProvidersTab() {
<h2 id="outage-ack-title" className="text-white font-semibold">Enable {confirming.name}?</h2>
<p className="text-red-300 text-sm font-semibold">{LICENCE_STYLE[confirming.licence.status].label}</p>
<p className="text-gray-300 text-sm leading-snug">{confirming.licence.warning}</p>
{error && <p className="text-red-400 text-xs font-mono break-words">{error}</p>}
<label className="flex items-start gap-2 text-sm text-gray-200">
<input type="checkbox" checked={ackChecked} onChange={(e) => setAckChecked(e.target.checked)} className="mt-1" />
<span>I confirm DRB holds the right to show this source&apos;s outage data. This confirmation is recorded with my account and the time in the audit log.</span>
drb-frontend/lib/types.ts
+5 -2
View File
@@ -230,7 +230,10 @@ export interface OutageArea {
fips: string;
county: string | null;
state: string | null;
customers_out: number;
customers_out: number; // live events only
/** Meters on events the source still lists whose own ETR passed >24h ago (ODIN keeps months-old rows). */
stale_customers_out: number;
stale: boolean; // every event in the county is stale: dim or hide, never show as live
customers_served: number | null; // null: no county-level denominator from the source
etr: string | null; // latest estimated restoration across the county's outages
since: string | null; // earliest reported start
@@ -238,7 +241,7 @@ export interface OutageArea {
lat: number | null;
lon: number | null;
bbox: [number, number, number, number] | null; // south, west, north, east
utilities: { name: string | null; eia_id: string | null; customers_out: number }[];
utilities: { name: string | null; eia_id: string | null; customers_out: number; stale_customers_out: number }[];
}
export interface OutageProviderFeed extends CameraCredit {