From aa5b71d85316f3481bb10a9f79466e322c72278c Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Sun, 4 Oct 2026 15:55:58 -0400 Subject: [PATCH 1/6] Live ops wall + pluggable camera providers with licence gating (server-26#198) Backend (c2-core): - app/internal/cameras/: provider interface, licence model, per-provider catalog cache; one adapter per source in design-canvas/camera-sources.md. Implemented: 511NY (legacy shape + tolerant XML inventory parser, 511x endpoints UNVERIFIED), Caltrans, DelDOT, Ontario 511, DriveBC, Calgary, TfL, NSW, Fintraffic. Stubs: NYC DOT, Westchester DPW, TRANSCOM, CTroads, 511NJ, Austin, TxDOT, MD CHART, WSDOT, Tallinn, Tarktee, Warendorf, TrafficLand, Windy, EarthCam. - Every provider defaults OFF (config/camera_providers). Gated licences (agreement required / unverified / non-commercial / paid) need an explicit acknowledgement stored with who/when and written to audit_log; the audit write is a precondition. Stubs and prohibited sources cannot be enabled. Ack is bound to the licence status it was given for. - GET /cameras (signed-in), GET/PUT /admin/camera-providers (admin only). - /traffic/511 follows the ny511 provider switch and returns the DAA credit + NYSDOT as-is disclaimer; 511NY event descriptions no longer truncated ("may not alter the substance of a report"). Frontend: - LiveView becomes the ops wall: Region > Incident > Camera drill-down, worst-first rail (MapView), KPIs, transcript ticker, scrubber incident bands, auto-tour, deep links (#inc=&cam=&tour), Esc steps up. - Incident panel: nearest cameras (path-weighted), fused situational context (ADS-B, AIS, 511NY events, NEXRAD tile sample), units, call spine. - MapView keeps every layer; adds Camera Viewsheds (approx.) overlay and a nearest-camera layer for the selected incident. - CameraFeed hygiene: hidden-tab/off-screen gating, 2 concurrent HLS cap, last good frame + stale badge, frozen-frame signature, HLS stall watchdog, provider attribution rendered on the imagery. - Camera/event popups show source credit, update time and the disclaimer. - Admin > Camera providers tab with licence status, health, gated confirm. - Camera detection is not built (blocked on #199/#200); seam only. Co-Authored-By: Claude Opus 5.5 --- drb-c2-core/.env.example | 4 + drb-c2-core/app/config.py | 10 +- drb-c2-core/app/internal/cameras/__init__.py | 6 + drb-c2-core/app/internal/cameras/base.py | 251 ++++++++++++ drb-c2-core/app/internal/cameras/general.py | 368 ++++++++++++++++++ drb-c2-core/app/internal/cameras/regional.py | 164 ++++++++ drb-c2-core/app/internal/cameras/registry.py | 198 ++++++++++ drb-c2-core/app/internal/ny511.py | 106 ++++- drb-c2-core/app/main.py | 4 +- drb-c2-core/app/routers/cameras.py | 51 +++ drb-c2-core/app/routers/traffic.py | 9 + drb-c2-core/tests/test_camera_providers.py | 283 ++++++++++++++ drb-c2-core/tests/test_ny511.py | 45 ++- drb-frontend/app/admin/page.tsx | 7 +- drb-frontend/app/globals.css | 15 + drb-frontend/components/CameraFeed.tsx | 262 +++++++++++-- drb-frontend/components/LiveView.tsx | 348 +++++++++++++++-- drb-frontend/components/MapView.tsx | 258 +++++++++--- drb-frontend/components/SourceCredit.tsx | 51 +++ drb-frontend/components/TimeScrubber.tsx | 47 ++- .../components/admin/CameraProvidersTab.tsx | 150 +++++++ drb-frontend/components/live/CameraDrill.tsx | 63 +++ .../components/live/TranscriptTicker.tsx | 62 +++ drb-frontend/components/live/WallPanels.tsx | 295 ++++++++++++++ drb-frontend/lib/c2api.ts | 17 +- drb-frontend/lib/liveWall.ts | 244 ++++++++++++ drb-frontend/lib/types.ts | 75 +++- drb-frontend/lib/useCameras.ts | 81 ++++ 28 files changed, 3342 insertions(+), 132 deletions(-) create mode 100644 drb-c2-core/app/internal/cameras/__init__.py create mode 100644 drb-c2-core/app/internal/cameras/base.py create mode 100644 drb-c2-core/app/internal/cameras/general.py create mode 100644 drb-c2-core/app/internal/cameras/regional.py create mode 100644 drb-c2-core/app/internal/cameras/registry.py create mode 100644 drb-c2-core/app/routers/cameras.py create mode 100644 drb-c2-core/tests/test_camera_providers.py create mode 100644 drb-frontend/components/SourceCredit.tsx create mode 100644 drb-frontend/components/admin/CameraProvidersTab.tsx create mode 100644 drb-frontend/components/live/CameraDrill.tsx create mode 100644 drb-frontend/components/live/TranscriptTicker.tsx create mode 100644 drb-frontend/components/live/WallPanels.tsx create mode 100644 drb-frontend/lib/liveWall.ts create mode 100644 drb-frontend/lib/useCameras.ts diff --git a/drb-c2-core/.env.example b/drb-c2-core/.env.example index 1faa287..33eb3e0 100644 --- a/drb-c2-core/.env.example +++ b/drb-c2-core/.env.example @@ -27,6 +27,10 @@ NODE_OFFLINE_THRESHOLD=90 # Enable "Geocoding API" in Cloud Console for this key GOOGLE_MAPS_API_KEY= NY511_API_KEY= +# 511NY feed base URL from the DAA grant (default: legacy 511ny.org/api, dead since the 511x cutover) +NY511_API_BASE= +# Ontario 511 developer key (camera provider "ontario511"; providers default OFF) +ONTARIO511_API_KEY= # OpenAI — for transcription (Whisper), intelligence extraction, embeddings, and summaries OPENAI_API_KEY= diff --git a/drb-c2-core/app/config.py b/drb-c2-core/app/config.py index cb5695f..76bc34c 100644 --- a/drb-c2-core/app/config.py +++ b/drb-c2-core/app/config.py @@ -32,9 +32,15 @@ class Settings(BaseSettings): # Google Maps (geocoding) google_maps_api_key: Optional[str] = None - # 511NY developer key (server-26#183). Optional: the API answered without one - # as of 2026-09-27, but its terms require a registered key. + # 511NY developer key (server-26#183/#198). Issued with the NYSDOT Developer + # Access Agreement grant; the keyless legacy API is gone since the 511x cutover. ny511_api_key: Optional[str] = None + # Base URL for the 511NY feed. Defaults to the legacy API, which is dead since + # the 511x cutover (server-26#198); set it to whatever the DAA grant documents. + ny511_api_base: Optional[str] = None + # Camera providers (app/internal/cameras). Keys only; every provider is still + # OFF until an admin enables it, and licence-gated ones need an acknowledgement. + ontario511_api_key: Optional[str] = None # Gemini (intelligence extraction, embeddings, incident summaries) gemini_api_key: Optional[str] = None diff --git a/drb-c2-core/app/internal/cameras/__init__.py b/drb-c2-core/app/internal/cameras/__init__.py new file mode 100644 index 0000000..71f1c4b --- /dev/null +++ b/drb-c2-core/app/internal/cameras/__init__.py @@ -0,0 +1,6 @@ +"""Pluggable public-camera providers (live ops wall). + +Every source in ``design-canvas/camera-sources.md`` is one adapter here, each +carrying its licence status from that catalog. Providers default OFF; the +licence gate and the admin toggle live in ``registry``. +""" diff --git a/drb-c2-core/app/internal/cameras/base.py b/drb-c2-core/app/internal/cameras/base.py new file mode 100644 index 0000000..5051d2b --- /dev/null +++ b/drb-c2-core/app/internal/cameras/base.py @@ -0,0 +1,251 @@ +"""Provider interface, licence model and the per-provider catalog cache. + +A provider is metadata (who runs the cameras, what their terms say, what +region they cover) plus one async ``fetch_catalog`` that returns normalised +camera records. Catalogs are public and identical for every org, so -- like +ny511.py -- they are cached in memory, fetched lazily on request, and never +written to Firestore. A failed refresh keeps serving the last good catalog and +reports the error and its age (the AI-silent-failures lesson: an empty layer +must never be the only symptom of a dead feed). + +Camera detection (God's Eye View idea #5) is NOT built: it is blocked on +server-26#199 (NYSDOT's written answer on imagery analytics) and #200 (CISO +ruling on frame retention). ``CameraSignalSink`` is the seam it would plug into. +""" +import asyncio +import re +import time +from dataclasses import dataclass +from enum import Enum +from typing import Any, Dict, List, Optional, Protocol, Tuple + +from app.internal.logger import logger + +RETRY_AFTER_FAILURE_S = 60 +DEFAULT_CATALOG_TTL_S = 15 * 60 # camera lists change rarely (GEV uses the same 15 min) +FETCH_TIMEOUT_S = 20.0 +USER_AGENT = "DRB-c2-core/1.0 (+public camera catalog; contact via 511/agency programs)" + +Camera = Dict[str, Any] +BBox = Tuple[float, float, float, float] # south, west, north, east + + +class Licence(str, Enum): + """Licence status, as catalogued in design-canvas/camera-sources.md.""" + COMMERCIAL_OK = "commercial_ok" + ATTRIBUTION_REQUIRED = "attribution_required" # commercial OK, credit must render with imagery + AGREEMENT_REQUIRED = "agreement_required" # signed agreement / written permission DRB does not hold + NON_COMMERCIAL = "non_commercial" + UNVERIFIED = "unverified" # no licence read first-hand + PAID = "paid" # needs a commercial contract + PROHIBITED = "prohibited" # terms forbid this use outright -- cannot be enabled + + +# Statuses that may only be enabled after an admin explicitly acknowledges the +# warning (recorded with who/when in the settings doc and audit_log). +GATED = frozenset({Licence.AGREEMENT_REQUIRED, Licence.NON_COMMERCIAL, Licence.UNVERIFIED, Licence.PAID}) + + +@dataclass(frozen=True) +class LicenceInfo: + status: Licence + label: str # one line, shown prominently in Admin + terms: str # what the source's own terms say (quoted where read) + terms_url: Optional[str] = None + attribution: Optional[str] = None # must render wherever this provider's imagery appears + attribution_url: Optional[str] = None + disclaimer: Optional[str] = None # pass-through disclaimer (e.g. NYSDOT DAA) + issue: Optional[str] = None # tracking issue, e.g. "server-26#198" + + @property + def gated(self) -> bool: + return self.status in GATED + + def warning(self) -> Optional[str]: + """The blocking warning an admin must acknowledge, or None when ungated.""" + if self.status == Licence.PROHIBITED: + return f"{self.label}. This source's terms forbid this use; it cannot be enabled." + if not self.gated: + return None + return (f"{self.label}. {self.terms} Enabling it means DRB displays this source's imagery " + f"to every signed-in user. Only confirm if DRB actually holds the right to do so.") + + def as_dict(self) -> dict: + return { + "status": self.status.value, "label": self.label, "terms": self.terms, + "terms_url": self.terms_url, "attribution": self.attribution, + "attribution_url": self.attribution_url, "disclaimer": self.disclaimer, + "issue": self.issue, "gated": self.gated, "warning": self.warning(), + } + + +@dataclass(frozen=True) +class ProviderMeta: + id: str + name: str + region: str + coverage: BBox # rough bbox; a query outside it never triggers a fetch + licence: LicenceInfo + formats: str # "stills", "stills + HLS", ... + implemented: bool + notes: str # what is verified vs unverified about this adapter + homepage: Optional[str] = None + key_setting: Optional[str] = None # settings attribute holding a required API key + ttl_s: int = DEFAULT_CATALOG_TTL_S + + +class CameraSignalSink(Protocol): + """Future seam (NOT implemented): aggregate per-class counts from a camera + still attached to an incident as corroborating evidence. Blocked on + server-26#199 and #200. No frames, plates or faces -- class counts only.""" + + async def submit(self, camera_id: str, incident_id: str, counts: Dict[str, int], observed_at: float) -> None: ... + + +class CameraProvider: + """Base adapter. Subclasses set ``meta`` and implement ``fetch_catalog``.""" + meta: ProviderMeta + + async def fetch_catalog(self) -> List[Camera]: + raise NotImplementedError(f"{self.meta.id}: adapter is a stub ({self.meta.notes})") + + +class StubProvider(CameraProvider): + """A catalogued source with no working adapter -- listed so Admin shows its + licence status, but it can never be enabled.""" + + def __init__(self, meta: ProviderMeta): + assert not meta.implemented + self.meta = meta + + +class Catalog: + """In-memory cache of one provider's normalised camera list.""" + + def __init__(self, provider: CameraProvider): + self.provider = provider + self.items: List[Camera] = [] + self.fetched_at: Optional[float] = None # epoch s of last SUCCESSFUL fetch + self.last_attempt_at: Optional[float] = None + self.error: Optional[str] = None + self._next_attempt = 0.0 + self._lock = asyncio.Lock() + + async def get(self) -> "Catalog": + if time.time() < self._next_attempt: + return self + async with self._lock: + if time.time() < self._next_attempt: + return self + ttl = self.provider.meta.ttl_s + self.last_attempt_at = time.time() + try: + items = await self.provider.fetch_catalog() + self.items = items + self.fetched_at = time.time() + self.error = None + self._next_attempt = time.time() + ttl + except Exception as e: + self._next_attempt = time.time() + min(ttl, RETRY_AFTER_FAILURE_S) + self.error = f"{type(e).__name__}: {e}"[:300] + logger.warning(f"cameras/{self.provider.meta.id}: refresh failed, serving {len(self.items)} cached: {self.error}") + return self + + def status(self) -> dict: + return {"fetched_at": self.fetched_at, "last_attempt_at": self.last_attempt_at, + "error": self.error, "count": len(self.items)} + + +# ── helpers shared by adapters ─────────────────────────────────────────────── + +def make_camera(provider_id: str, raw_id: Any, lat: float, lon: float, *, name: str = "", roadway: str = "", + direction: str = "", image_url: Optional[str] = None, video_url: Optional[str] = None, + heading_deg: Optional[float] = None) -> Camera: + """One normalised record. ``id`` is provider-prefixed so two catalogs can't collide.""" + if heading_deg is None: + heading_deg = heading_from_text(direction) + return { + "id": f"{provider_id}:{raw_id}", + "provider": provider_id, + "lat": lat, "lon": lon, + "name": name, "roadway": roadway, "direction": direction, + # Approximate at best: most sources publish the road's direction of + # travel, not the camera's true heading. The UI labels viewsheds so. + "heading_deg": heading_deg, + "image_url": image_url, "video_url": video_url, + } + + +_COMPASS = {"N": 0, "NE": 45, "E": 90, "SE": 135, "S": 180, "SW": 225, "W": 270, "NW": 315} +_WORDS = {"north": "N", "south": "S", "east": "E", "west": "W", + "northeast": "NE", "northwest": "NW", "southeast": "SE", "southwest": "SW"} + + +def heading_from_text(text: Optional[str]) -> Optional[float]: + """'Northbound' / 'NB' / 'West' / 'SE' -> degrees; None when it can't tell. + 'Both directions', 'Unknown', 'Eastbound/Westbound' etc. are None on purpose.""" + if not text: + return None + t = text.strip().lower() + t = re.sub(r"bound$", "", t).strip() + if t in _WORDS: + return float(_COMPASS[_WORDS[t]]) + t2 = re.sub(r"b$", "", t).upper() if re.fullmatch(r"(n|s|e|w|ne|nw|se|sw)b?", t) else t.upper() + return float(_COMPASS[t2]) if t2 in _COMPASS else None + + +def finite(v: Any) -> Optional[float]: + try: + f = float(v) + except (TypeError, ValueError): + return None + return f if f == f and f not in (float("inf"), float("-inf")) else None + + +def plausible(lat: Optional[float], lon: Optional[float], coverage: BBox) -> bool: + if lat is None or lon is None or (lat == 0 and lon == 0): + return False + s, w, n, e = coverage + return s <= lat <= n and w <= lon <= e + + +def https_on(url: Any, *hosts: str) -> Optional[str]: + """Return ``url`` only when it is https on one of ``hosts`` (or a subdomain). + Pins imagery to the agency's own hosts so no catalog field can point the + browser somewhere else.""" + if not isinstance(url, str) or not url.startswith("https://"): + return None + host = url[len("https://"):].split("/", 1)[0].split(":", 1)[0].lower() + return url if any(host == h or host.endswith("." + h) for h in hosts) else None + + +def bbox_intersects(a: BBox, b: BBox) -> bool: + return not (a[2] < b[0] or b[2] < a[0] or a[3] < b[1] or b[3] < a[1]) + + +def in_bbox(items: List[Camera], bbox: BBox) -> List[Camera]: + s, w, n, e = bbox + return [i for i in items if s <= i["lat"] <= n and w <= i["lon"] <= e] + + +async def get_json(url: str, params: Optional[dict] = None, headers: Optional[dict] = None) -> Any: + """GET a JSON catalog. Redirects are refused: a catalog that moves is a + change to review, not something to follow to an unknown host.""" + import httpx # local import keeps module import cheap for the test stubs + h = {"User-Agent": USER_AGENT, "Accept": "application/json", **(headers or {})} + async with httpx.AsyncClient(timeout=FETCH_TIMEOUT_S, follow_redirects=False) as client: + r = await client.get(url, params=params, headers=h) + r.raise_for_status() + ctype = r.headers.get("content-type", "") + if "json" not in ctype: + # 511NY's dead legacy API answers 200 text/html (server-26#198). + raise ValueError(f"expected JSON, got {ctype or 'no content-type'} (HTTP {r.status_code})") + return r.json() + + +def require_parsed(raw_count: int, out: List[Camera], what: str) -> List[Camera]: + """Every record failing to normalise means the schema changed under us -- + that is an error, not an empty catalog.""" + if raw_count and not out: + raise ValueError(f"0 of {raw_count} {what} records parsed -- upstream schema change?") + return out diff --git a/drb-c2-core/app/internal/cameras/general.py b/drb-c2-core/app/internal/cameras/general.py new file mode 100644 index 0000000..3e31f58 --- /dev/null +++ b/drb-c2-core/app/internal/cameras/general.py @@ -0,0 +1,368 @@ +"""General / other-region camera sources -- design-canvas/camera-sources.md §C, +including every God's Eye View (GEV) source. + +Endpoints and record shapes for the implemented adapters were checked +first-hand against each live catalog on 2026-10-04 (Ontario 511 excepted: it now +answers "Invalid Key" without a developer key, so its shape is the legacy CARS +v2 shape, UNVERIFIED). Imagery URLs are pinned to each agency's own hosts. None +of these cover DRB's region; they exist so a non-NY customer has a path, and +every one of them is OFF until an admin enables it. + +GEV is MIT-licensed code, but nothing here is copied from it, and none of GEV's +bundled data (config/*.json curated catalogs) is used -- the MIT grant does not +cover data. +""" +import asyncio +from typing import List + +from app.config import settings +from app.internal.cameras.base import ( + Camera, CameraProvider, Licence, LicenceInfo, ProviderMeta, StubProvider, finite, get_json, https_on, + make_camera, plausible, require_parsed, +) + + +def _unverified(label: str, terms: str, url: str = None) -> LicenceInfo: + return LicenceInfo(status=Licence.UNVERIFIED, label=label, terms=terms, terms_url=url) + + +# ── Caltrans CWWP2 ─────────────────────────────────────────────────────────── +class CaltransProvider(CameraProvider): + DISTRICTS = (4, 7, 11, 3) # SF Bay, LA, San Diego, Sacramento + meta = ProviderMeta( + id="caltrans", name="Caltrans CWWP2", region="California (districts 4, 7, 11, 3)", + coverage=(32.4, -124.6, 42.1, -114.0), + licence=_unverified( + "Licence UNVERIFIED -- likely commercial OK; Conditions of Use not yet read", + "\"Commercial Wholesale Web Portal\", \"no charge\"; Conditions of Use apply and have not been read.", + "https://cwwp2.dot.ca.gov/documentation/cctv/cctv.htm"), + formats="stills (~5 min) + HLS", implemented=True, + notes="Schema verified 2026-10-04 (D4). HLS host CORS unverified -- the player falls back to the still.", + homepage="https://cwwp2.dot.ca.gov", + ) + + async def _district(self, d: int) -> list: + payload = await get_json(f"https://cwwp2.dot.ca.gov/data/d{d}/cctv/cctvStatusD{d:02d}.json") + return payload.get("data") or [] if isinstance(payload, dict) else [] + + async def fetch_catalog(self) -> List[Camera]: + results = await asyncio.gather(*(self._district(d) for d in self.DISTRICTS), return_exceptions=True) + ok = [r for r in results if not isinstance(r, BaseException)] + if not ok: + raise RuntimeError(f"every Caltrans district failed: {results[0]}") + rows = [row for district in ok for row in district] + out: List[Camera] = [] + for row in rows: + c = (row or {}).get("cctv") or {} + if str(c.get("inService", "")).lower() != "true": + continue + loc = c.get("location") or {} + lat, lon = finite(loc.get("latitude")), finite(loc.get("longitude")) + if not plausible(lat, lon, self.meta.coverage): + continue + img = https_on(((c.get("imageData") or {}).get("static") or {}).get("currentImageURL"), "dot.ca.gov") + if not img: + continue + vid = https_on((c.get("imageData") or {}).get("streamingVideoURL"), "dot.ca.gov") + name = str(loc.get("locationName") or "") + code = name.split("--", 1)[0].strip() or str(c.get("index")) + out.append(make_camera("caltrans", f"d{loc.get('district')}-{code}", lat, lon, + name=name.split("--", 1)[-1].strip() or name, roadway=str(loc.get("route") or ""), + direction=str(loc.get("direction") or ""), image_url=img, video_url=vid)) + return require_parsed(len(rows), out, "Caltrans") if rows else out + + +# ── DelDOT ─────────────────────────────────────────────────────────────────── +class DelDotProvider(CameraProvider): + meta = ProviderMeta( + id="deldot", name="DelDOT TMC", region="Delaware", + coverage=(38.4, -75.8, 39.9, -75.0), + licence=_unverified("Licence UNVERIFIED -- \"public data\", no explicit licence found", + "No licence or terms of reuse located for tmc.deldot.gov's camera catalog."), + formats="live HLS only (no stills)", implemented=True, + notes="Schema verified 2026-10-04 (361 cams); https HLS host sends ACAO: * (verified). Video-only, so a " + "camera shows nothing when its stream is down. Played directly from the browser -- no DRB egress.", + homepage="https://deldot.gov/map/", + ) + + async def fetch_catalog(self) -> List[Camera]: + payload = await get_json("https://tmc.deldot.gov/json/videocamera.json") + rows = payload.get("videoCameras") or [] if isinstance(payload, dict) else [] + out: List[Camera] = [] + for r in rows: + if r.get("enabled") is not True or r.get("status") != "Active" or not r.get("id"): + continue + lat, lon = finite(r.get("lat")), finite(r.get("lon")) + vid = https_on((r.get("urls") or {}).get("m3u8s"), "deldot.gov") + if not plausible(lat, lon, self.meta.coverage) or not vid: + continue + out.append(make_camera("deldot", r["id"], lat, lon, name=str(r.get("title") or r["id"]), + roadway=str(r.get("county") or ""), video_url=vid)) + return require_parsed(len(rows), out, "DelDOT") + + +# ── Ontario 511 ────────────────────────────────────────────────────────────── +class Ontario511Provider(CameraProvider): + meta = ProviderMeta( + id="ontario511", name="Ontario 511", region="Ontario, Canada", + coverage=(41.6, -95.2, 57.0, -74.3), + licence=_unverified("Licence UNVERIFIED -- Ontario open-data terms page not read", + "GEV reports the Open Government Licence - Ontario; not confirmed first-hand.", + "https://511on.ca/developers/doc"), + formats="stills", implemented=True, + notes="Needs ONTARIO511_API_KEY (verified 2026-10-04: keyless now answers \"Invalid Key\"); 10 calls/60 s. " + "Record shape is the CARS v2 shape (Id/Latitude/Longitude/Views[]), UNVERIFIED since the key change.", + homepage="https://511on.ca", key_setting="ontario511_api_key", + ) + + async def fetch_catalog(self) -> List[Camera]: + if not settings.ontario511_api_key: + raise RuntimeError("ONTARIO511_API_KEY not set") + rows = await get_json("https://511on.ca/api/v2/get/cameras", + params={"format": "json", "lang": "en", "key": settings.ontario511_api_key}) + rows = rows if isinstance(rows, list) else [] + out: List[Camera] = [] + for r in rows: + lat, lon = finite(r.get("Latitude")), finite(r.get("Longitude")) + if not r.get("Id") or not plausible(lat, lon, self.meta.coverage): + continue + views = [v for v in (r.get("Views") or []) if str(v.get("Status", "")).lower() == "enabled"] + img = next((https_on(v.get("Url"), "511on.ca") for v in views if https_on(v.get("Url"), "511on.ca")), None) + if not img: + continue + out.append(make_camera("ontario511", r["Id"], lat, lon, name=str(r.get("Location") or ""), + roadway=str(r.get("Roadway") or ""), direction=str(r.get("Direction") or ""), + image_url=img)) + return require_parsed(len(rows), out, "Ontario 511") + + +# ── DriveBC ────────────────────────────────────────────────────────────────── +class DriveBcProvider(CameraProvider): + meta = ProviderMeta( + id="drivebc", name="DriveBC", region="British Columbia, Canada", + coverage=(48.2, -139.1, 60.1, -114.0), + licence=LicenceInfo( + status=Licence.ATTRIBUTION_REQUIRED, label="Commercial OK with attribution (OGL - British Columbia)", + terms="Open Government Licence - British Columbia: reuse permitted with attribution.", + terms_url="https://www2.gov.bc.ca/gov/content/data/policy-standards/open-data/open-government-licence-bc", + attribution="DriveBC -- contains information licensed under the Open Government Licence - British Columbia", + attribution_url="https://www.drivebc.ca"), + formats="stills", implemented=True, notes="Schema verified 2026-10-04 (1,066 cams).", + homepage="https://www.drivebc.ca", + ) + + async def fetch_catalog(self) -> List[Camera]: + rows = await get_json("https://www.drivebc.ca/api/webcams/") + rows = rows if isinstance(rows, list) else [] + out: List[Camera] = [] + for r in rows: + if r.get("is_on") is not True or r.get("should_appear") is not True: + continue + cid = r.get("id") + coords = (r.get("location") or {}).get("coordinates") or [] + if not isinstance(cid, int) or len(coords) < 2: + continue + lon, lat = finite(coords[0]), finite(coords[1]) + if not plausible(lat, lon, self.meta.coverage): + continue + out.append(make_camera("drivebc", cid, lat, lon, name=str(r.get("name") or ""), + roadway=str(r.get("highway_description") or ""), + direction=str(r.get("orientation") or ""), + image_url=f"https://www.drivebc.ca/images/{cid}.jpg")) + return require_parsed(len(rows), out, "DriveBC") + + +# ── Open Calgary ───────────────────────────────────────────────────────────── +class CalgaryProvider(CameraProvider): + meta = ProviderMeta( + id="calgary", name="Open Calgary traffic cameras", region="Calgary, Alberta", + coverage=(50.8, -114.5, 51.3, -113.8), + licence=LicenceInfo( + status=Licence.ATTRIBUTION_REQUIRED, label="Commercial OK with attribution (OGL - City of Calgary)", + terms="Open Government Licence - City of Calgary: reuse permitted with attribution.", + terms_url="https://data.calgary.ca/stories/s/Open-Calgary-Terms-of-Use/u45n-7awa", + attribution="Contains information licensed under the Open Government Licence - City of Calgary", + attribution_url="https://data.calgary.ca"), + formats="stills", implemented=True, + notes="Schema verified 2026-10-04 (216 cams). Catalog lists http:// stills; upgraded to https on the same host.", + homepage="https://data.calgary.ca", + ) + + async def fetch_catalog(self) -> List[Camera]: + rows = await get_json("https://data.calgary.ca/resource/k7p9-kppz.json", params={"$limit": "1000"}) + rows = rows if isinstance(rows, list) else [] + out: List[Camera] = [] + for r in rows: + url = str((r.get("camera_url") or {}).get("url") or "") + if url.startswith("http://"): + url = "https://" + url[len("http://"):] + img = https_on(url, "trafficcam.calgary.ca") + coords = (r.get("point") or {}).get("coordinates") or [] + if not img or len(coords) < 2: + continue + lon, lat = finite(coords[0]), finite(coords[1]) + if not plausible(lat, lon, self.meta.coverage): + continue + cid = img.rsplit("/", 1)[-1].rsplit(".", 1)[0] + out.append(make_camera("calgary", cid, lat, lon, name=str(r.get("camera_location") or cid), + roadway=str(r.get("quadrant") or ""), image_url=img)) + return require_parsed(len(rows), out, "Calgary") + + +# ── TfL JamCams ────────────────────────────────────────────────────────────── +class TflProvider(CameraProvider): + IMAGE_PREFIX = "https://s3-eu-west-1.amazonaws.com/jamcams.tfl.gov.uk/" + meta = ProviderMeta( + id="tfl", name="TfL JamCams", region="London, UK", + coverage=(51.2, -0.7, 51.8, 0.4), + licence=LicenceInfo( + status=Licence.ATTRIBUTION_REQUIRED, + label="Commercial OK with attribution (\"Powered by TfL Open Data\") -- TfL terms page not re-read", + terms="TfL transport data terms: free incl. commercial, with attribution (per GEV; the terms page " + "returned 403 during the survey -- re-read before selling).", + terms_url="https://tfl.gov.uk/info-for/open-data-users/our-open-data", + attribution="Powered by TfL Open Data", attribution_url="https://tfl.gov.uk"), + formats="stills (+ short MP4, unused)", implemented=True, notes="Schema verified 2026-10-04 (890 cams).", + homepage="https://api.tfl.gov.uk", + ) + + async def fetch_catalog(self) -> List[Camera]: + rows = await get_json("https://api.tfl.gov.uk/Place/Type/JamCam") + rows = rows if isinstance(rows, list) else [] + out: List[Camera] = [] + for r in rows: + props = {p.get("key"): p.get("value") for p in (r.get("additionalProperties") or []) if isinstance(p, dict)} + if str(props.get("available", "")).lower() != "true": + continue + img = str(props.get("imageUrl") or "") + lat, lon = finite(r.get("lat")), finite(r.get("lon")) + raw_id = str(r.get("id") or "").replace("JamCams_", "") + if not img.startswith(self.IMAGE_PREFIX) or not raw_id or not plausible(lat, lon, self.meta.coverage): + continue + out.append(make_camera("tfl", raw_id, lat, lon, name=str(r.get("commonName") or raw_id), + direction=str(props.get("view") or ""), image_url=img)) + return require_parsed(len(rows), out, "TfL") + + +# ── Live Traffic NSW ───────────────────────────────────────────────────────── +class NswProvider(CameraProvider): + meta = ProviderMeta( + id="nsw", name="Live Traffic NSW", region="New South Wales, Australia", + coverage=(-37.6, 140.9, -28.0, 153.7), + licence=LicenceInfo( + status=Licence.ATTRIBUTION_REQUIRED, label="Commercial OK with attribution (CC BY 4.0)", + terms="Catalog's own rights block: copyright Transport for NSW, licence per the Open Data Hub dataset " + "(CC BY 4.0).", + terms_url="https://opendata.transport.nsw.gov.au/dataset/live-traffic-cameras", + attribution="Transport for NSW, CC BY 4.0", attribution_url="https://www.livetraffic.com"), + formats="stills", implemented=True, notes="Schema verified 2026-10-04 (241 cams).", + homepage="https://www.livetraffic.com", + ) + + async def fetch_catalog(self) -> List[Camera]: + payload = await get_json("https://data.livetraffic.com/cameras/traffic-cam.json") + feats = payload.get("features") or [] if isinstance(payload, dict) else [] + out: List[Camera] = [] + for f in feats: + p = f.get("properties") or {} + coords = (f.get("geometry") or {}).get("coordinates") or [] + img = https_on(p.get("href"), "transport.nsw.gov.au") + if not img or len(coords) < 2 or not f.get("id"): + continue + lon, lat = finite(coords[0]), finite(coords[1]) + if not plausible(lat, lon, self.meta.coverage): + continue + out.append(make_camera("nsw", f["id"], lat, lon, name=str(p.get("title") or ""), + roadway=str(p.get("region") or ""), direction=str(p.get("direction") or ""), + image_url=img)) + return require_parsed(len(feats), out, "NSW") + + +# ── Fintraffic / Digitraffic weathercams ───────────────────────────────────── +class FintrafficProvider(CameraProvider): + meta = ProviderMeta( + id="fintraffic", name="Fintraffic / Digitraffic weathercams", region="Finland", + coverage=(59.5, 19.0, 70.2, 31.7), + licence=LicenceInfo( + status=Licence.ATTRIBUTION_REQUIRED, label="Commercial OK with attribution (CC BY 4.0)", + terms="\"Source: Fintraffic / digitraffic.fi, license CC 4.0 BY\".", + terms_url="https://www.digitraffic.fi/en/terms-of-service/", + attribution="Source: Fintraffic / digitraffic.fi, license CC 4.0 BY", + attribution_url="https://www.digitraffic.fi"), + formats="stills (~10 min)", implemented=True, + notes="Schema verified 2026-10-04 (810 stations, several presets each). Digitraffic asks clients to " + "identify themselves (Digitraffic-User header).", + homepage="https://www.digitraffic.fi", ttl_s=30 * 60, + ) + + async def fetch_catalog(self) -> List[Camera]: + payload = await get_json("https://tie.digitraffic.fi/api/weathercam/v1/stations", + headers={"Digitraffic-User": "DRB/c2-core"}) + feats = payload.get("features") or [] if isinstance(payload, dict) else [] + out: List[Camera] = [] + for f in feats: + p = f.get("properties") or {} + coords = (f.get("geometry") or {}).get("coordinates") or [] + sid = str(p.get("id") or "") + if not sid or str(p.get("collectionStatus", "")).upper() != "GATHERING" or len(coords) < 2: + continue + lon, lat = finite(coords[0]), finite(coords[1]) + if not plausible(lat, lon, self.meta.coverage): + continue + for preset in p.get("presets") or []: + pid = str((preset or {}).get("id") or "") + if preset.get("inCollection") is not True or not pid.startswith(sid) or not pid[1:].isdigit(): + continue + out.append(make_camera("fintraffic", pid, lat, lon, name=f"{p.get('name') or sid} ({pid[-2:]})", + image_url=f"https://weathercam.digitraffic.fi/{pid}.jpg")) + return require_parsed(len(feats), out, "Fintraffic") + + +# ── Stubs: catalogued, no adapter ──────────────────────────────────────────── +def _stub(id, name, region, coverage, licence, formats, notes, homepage=None) -> StubProvider: + return StubProvider(ProviderMeta(id=id, name=name, region=region, coverage=coverage, licence=licence, + formats=formats, implemented=False, notes=notes, homepage=homepage)) + + +STUBS = [ + _stub("austin", "City of Austin open data", "Austin, Texas", (30.0, -98.1, 30.6, -97.5), + _unverified("Licence UNVERIFIED -- city open data, no explicit licence", "No explicit licence per GEV."), + "stills (5 min)", "Stub: Socrata column mapping not built.", "https://data.austintexas.gov"), + _stub("txdot", "TxDOT ITS", "Texas (per district)", (25.8, -106.7, 36.6, -93.5), + _unverified("Licence UNVERIFIED -- \"public data\"", "No explicit licence per GEV."), + "base64 JPEG inside JSON", "Stub: frames are base64 inside JSON, so the browser cannot show them " + "without a DRB frame proxy (video/image egress on DRB's bill -- not built).", "https://its.txdot.gov"), + _stub("md_chart", "Maryland CHART", "Maryland", (37.9, -79.5, 39.8, -75.0), + _unverified("Licence UNVERIFIED -- MD open data portal terms not read", "Not read."), + "live video URLs", "Stub: catalog schema not verified.", "https://chart.maryland.gov"), + _stub("wsdot", "WSDOT Traveler Information API", "Washington State", (45.5, -124.9, 49.1, -116.9), + _unverified("Licence UNVERIFIED -- WSDOT disclaimer only", "Free access code required; terms not read."), + "stills", "Stub: needs a WSDOT access code; schema not verified.", "https://wsdot.wa.gov/traffic/api/"), + _stub("tallinn", "Tallinn city cameras", "Tallinn, Estonia", (59.35, 24.55, 59.5, 24.95), + _unverified("Licence UNVERIFIED", "No licence found."), + "stills", "Stub: GEV uses a curated catalog file; GEV's data is not covered by its MIT licence.", + "https://ristmikud.tallinn.ee"), + _stub("tarktee", "Tarktee (Estonian Transport Administration)", "Estonia", (57.5, 21.7, 59.8, 28.3), + _unverified("Licence UNVERIFIED", "No licence found."), + "DATEX2 stills", "Stub: DATEX2 XML parser not built.", "https://tarktee.transpordiamet.ee"), + _stub("warendorf", "Warendorf municipal webcam", "Warendorf, Germany", (51.9, 7.9, 52.0, 8.1), + _unverified("Licence UNVERIFIED", "No licence found."), + "stills", "Stub: single curated webcam; no catalog API.", "https://www.warendorf.de"), + _stub("trafficland", "TrafficLand", "US (18k cams, 50+ DOTs)", (18.0, -170.0, 72.0, -60.0), + LicenceInfo(status=Licence.PAID, label="Paid commercial licence -- no contract", + terms="Licensed API; contractual redistribution with 50+ DOTs. Price UNVERIFIED.", + terms_url="http://www.trafficland.com/api.html"), + "live video", "Stub: the clean route if DRB ever needs video beyond 511NY; needs a contract."), + _stub("windy", "Windy Webcams API", "Global (scenic/weather)", (-90.0, -180.0, 90.0, 180.0), + LicenceInfo(status=Licence.PAID, label="Paid professional tier for commercial use -- no contract", + terms="Free tier image URLs expire after 10 min; commercial terms UNVERIFIED.", + terms_url="https://api.windy.com/webcams/docs"), + "tokenised stills", "Stub: scenic/weather cams, not traffic."), + _stub("earthcam", "EarthCam", "Global", (-90.0, -180.0, 90.0, 180.0), + LicenceInfo(status=Licence.PAID, label="Paid commercial licence -- no contract", + terms="Commercial licensing exists; terms UNVERIFIED."), + "live video", "Stub: low local value."), +] + +PROVIDERS = [CaltransProvider(), DelDotProvider(), Ontario511Provider(), DriveBcProvider(), CalgaryProvider(), + TflProvider(), NswProvider(), FintrafficProvider(), *STUBS] diff --git a/drb-c2-core/app/internal/cameras/regional.py b/drb-c2-core/app/internal/cameras/regional.py new file mode 100644 index 0000000..a128182 --- /dev/null +++ b/drb-c2-core/app/internal/cameras/regional.py @@ -0,0 +1,164 @@ +"""Tri-state (Westchester-first) camera sources -- design-canvas/camera-sources.md §A. + +Only 511NY has a working adapter. Every other regional source is listed so +Admin shows its licence status, and is a stub because DRB has no right to use +it yet (NYC DOT, Westchester DPW), it carries no cameras (CTroads), or no +developer program exists (511NJ). Do not "fix" a stub by scraping: CTroads, +511NJ's app and the new 511ny.org backend all ban robots/reverse engineering. +""" +from typing import List + +from app.internal import ny511 +from app.internal.cameras.base import ( + Camera, CameraProvider, Licence, LicenceInfo, ProviderMeta, StubProvider, make_camera, require_parsed, +) + +NY_BBOX = (40.4, -80.0, 45.1, -71.7) +TRISTATE_BBOX = (40.3, -75.6, 42.1, -71.7) + +NYSDOT_DISCLAIMER = ( + "NYSDOT compiles the data from the best available sources and makes a best effort to disseminate it. " + "Data availability and accuracy are subject to change, and may change as often as daily. Posted " + "restrictions in the field may differ from, and supersede, the information in this feed. NYSDOT and its " + "partner agencies make no warranty regarding the data, express or implied, and expressly disclaim all " + "warranties of merchantability and fitness for any particular purpose. NYSDOT makes no warranty that the " + "data will be provided without interruption or that it will be free of errors. Access is provided on an " + "\"as is\" and \"with all faults\" basis." +) + + +class Ny511Provider(CameraProvider): + meta = ProviderMeta( + id="ny511", + name="511NY (NYSDOT)", + region="New York State (aggregates NYSDOT, Thruway, NYC DOT, MTA B&T, PANYNJ, TRANSCOM partners)", + coverage=NY_BBOX, + licence=LicenceInfo( + status=Licence.AGREEMENT_REQUIRED, + label="Requires NYSDOT Developer Access Agreement -- access to the new 511x feed NOT granted yet", + terms=("Commercial redistribution is allowed under the DAA (\"You may redistribute, enhance, repackage, " + "and add value to the data\"; \"You may not alter the substance of a report\"; must display the " + "time the data was published or last updated). Access is granted per request via the 511NY " + "Data Feed Request Form; DRB's 2026-09-27 request went to the retired system. Overlays, " + "detection and frame archiving are not addressed (server-26#199)."), + terms_url="https://511ny.org/help/24", + # DAA permitted use: tag line + logo + link. The approved logo is not + # in the repo yet (owner action), so only the tag line + link render. + attribution="powered by 511NY", + attribution_url="https://511ny.org", + disclaimer=NYSDOT_DISCLAIMER, + issue="server-26#198", + ), + formats="stills (60 s) + HLS on NYSDOT hosts (legacy shape)", + implemented=True, + notes=("Legacy request/record shape behind settings.ny511_api_base + ny511_api_key, plus a tolerant " + "parser for the 511x XML camera inventory. UNVERIFIED until the DAA grant documents the new " + "endpoints: base URL, query params, XML element names, still/HLS URL shape and CORS. The " + "legacy endpoint has returned HTML since the 511x cutover, so expect a feed error until then."), + homepage="https://511ny.org", + key_setting="ny511_api_key", + ttl_s=ny511.CAMERAS_TTL_S, + ) + + async def fetch_catalog(self) -> List[Camera]: + rows = await ny511._fetch("getcameras", ny511.normalize_camera) + out = [ + make_camera("ny511", r["id"], r["lat"], r["lon"], name=r["name"], roadway=r["roadway"], + direction=r["direction"], image_url=r["image_url"], video_url=r["video_url"]) + for r in rows + ] + return require_parsed(len(rows), out, "511NY") + + +NYC_DOT = StubProvider(ProviderMeta( + id="nyc_dot", + name="NYC DOT TMC (webcams.nyctmc.org)", + region="New York City (~975 cams, ~80 in the Bronx)", + coverage=(40.45, -74.30, 40.95, -73.65), + licence=LicenceInfo( + status=Licence.AGREEMENT_REQUIRED, + label="Requires a signed NYC DOT data-sharing agreement -- NOT signed", + terms=("NYC DOT Data Feeds page: \"Developers must sign a data-sharing agreement\" (tmcdot@dot.nyc.gov). " + "The API answers keyless, but keyless use is outside those terms. Agreement terms unpublished."), + terms_url="https://www.nyc.gov/html/dot/html/about/datafeeds.shtml", + issue="server-26#201", + ), + formats="JPEG stills (~15 s), no CORS", + implemented=False, + notes="Stub on purpose: do not integrate until the agreement is signed (server-26#201). Many of these cams " + "are likely already re-published by 511NY.", + homepage="https://webcams.nyctmc.org", +)) + +WESTCHESTER_DPW = StubProvider(ProviderMeta( + id="westchester_dpw", + name="Westchester County DPW traffic cameras", + region="Westchester County (~21 locations, mostly Bronx River Pkwy)", + coverage=(40.88, -73.99, 41.37, -73.48), + licence=LicenceInfo( + status=Licence.AGREEMENT_REQUIRED, + label="No reuse licence -- written permission from County DPW required, NOT obtained", + terms="Only a disclaimer: data \"provided 'as is' without warranty\". No API, no reuse licence.", + terms_url="https://experience.arcgis.com/experience/a59314e926e94301b9083edd6bb52a11", + issue="server-26#202", + ), + formats="stills (5 s), viewer only", + implemented=False, + notes="Stub: there is no documented API or feed, and no permission. Ask DPW in writing (server-26#202).", + homepage="https://www.westchestergov.com/trafficcamera", +)) + +TRANSCOM = StubProvider(ProviderMeta( + id="transcom_xcm", + name="TRANSCOM XCM data exchange", + region="NY/NJ/CT tri-state", + coverage=TRISTATE_BBOX, + licence=LicenceInfo( + status=Licence.UNVERIFIED, + label="Events/travel times commercial OK; cameras UNVERIFIED", + terms="Free registration, open to \"commercial vendors ... for use in their applications\". Whether the " + "feed carries camera URLs is unverified.", + terms_url="https://data.xcmdata.org/DEWeb/Pages/aboutus", + ), + formats="XML events + link travel times (cameras unverified)", + implemented=False, + notes="Stub: register first, then confirm whether any camera inventory exists. Its events feed would be a " + "separate (event) provider, not a camera one.", + homepage="https://data.xcmdata.org", +)) + +CTROADS = StubProvider(ProviderMeta( + id="ctroads", + name="CTDOT / CTroads", + region="Connecticut", + coverage=(40.95, -73.75, 42.06, -71.78), + licence=LicenceInfo( + status=Licence.PROHIBITED, + label="No sanctioned camera path -- site terms forbid camera reuse", + terms="The keyed developer API covers Message Signs, Events and Advisories only -- no cameras. The site " + "T&C forbid use \"for any personal, public or commercial purpose\" and robots.", + terms_url="https://ctroads.org/termsandconditions", + ), + formats="events/signs only (no cameras)", + implemented=False, + notes="Cannot be enabled. CT cameras may surface via 511NY's \"Connecticut Statewide\" region once the DAA " + "grant arrives.", + homepage="https://ctroads.org/developers/doc", +)) + +NJ511 = StubProvider(ProviderMeta( + id="nj511", + name="511NJ / NJDOT / NJTA / PANYNJ", + region="New Jersey", + coverage=(38.9, -75.6, 41.4, -73.9), + licence=LicenceInfo( + status=Licence.UNVERIFIED, + label="No published developer program -- reuse UNVERIFIED", + terms="511NJ is a JS app with no published developer program; NJDOT only cites RSS/Twitter.", + ), + formats="unknown", + implemented=False, + notes="Stub: ask NJDOT 511 if northern NJ matters. Some cams appear in 511NY.", +)) + +PROVIDERS = [Ny511Provider(), NYC_DOT, WESTCHESTER_DPW, TRANSCOM, CTROADS, NJ511] diff --git a/drb-c2-core/app/internal/cameras/registry.py b/drb-c2-core/app/internal/cameras/registry.py new file mode 100644 index 0000000..f58e873 --- /dev/null +++ b/drb-c2-core/app/internal/cameras/registry.py @@ -0,0 +1,198 @@ +"""Camera provider registry: settings, licence gate, and bbox queries. + +Settings live in Firestore ``config/camera_providers``:: + + {"providers": {"": {"enabled": bool, "updated_at": iso, "updated_by": {...}, + "acknowledgement": {"by_uid", "by_email", "at", "licence_status", "warning"} | None}}} + +Rules (owner brief, 2026-10-04): + * every provider defaults OFF -- a missing doc/entry means off; + * a stub (no adapter) or a PROHIBITED source can never be enabled; + * a GATED licence (agreement required / non-commercial / unverified / paid) + can only be enabled with an explicit acknowledgement, recorded with who and + when in the settings doc AND audit_log. The audit write is a precondition + for a gated enable: no record, no enable; + * an acknowledgement is bound to the licence status it was given for. If a + provider's catalogued status changes in code, the old acknowledgement no + longer counts and the provider stops serving until re-acknowledged. +""" +import asyncio +import time +from datetime import datetime, timezone +from typing import Dict, List, Optional, Tuple + +from app.config import settings +from app.internal import firestore as fstore +from app.internal.cameras import general, regional +from app.internal.cameras.base import ( + BBox, Camera, CameraProvider, Catalog, Licence, bbox_intersects, in_bbox, +) +from app.internal.logger import logger + +COLLECTION, DOC_ID = "config", "camera_providers" +_TTL_S = 30.0 +MAX_ITEMS = 1500 # per response, same cap as /traffic/511 + +PROVIDERS: Dict[str, CameraProvider] = {p.meta.id: p for p in [*regional.PROVIDERS, *general.PROVIDERS]} +CATALOGS: Dict[str, Catalog] = {pid: Catalog(p) for pid, p in PROVIDERS.items() if p.meta.implemented} + +_cache: Dict[str, dict] = {} +_cache_ts = 0.0 + + +class ProviderError(Exception): + """A refused settings change. ``code`` is the HTTP status the router returns.""" + + def __init__(self, code: int, message: str, **extra): + super().__init__(message) + self.code, self.message, self.extra = code, message, extra + + +def invalidate_cache() -> None: + global _cache_ts + _cache_ts = 0.0 + + +async def get_settings() -> Dict[str, dict]: + """Per-provider settings. Unreadable Firestore means everything is OFF (fail closed).""" + global _cache, _cache_ts + if _cache_ts and time.monotonic() - _cache_ts < _TTL_S: + return _cache + try: + doc = await fstore.doc_get(COLLECTION, DOC_ID) or {} + provs = doc.get("providers") if isinstance(doc.get("providers"), dict) else {} + _cache = {k: v for k, v in provs.items() if isinstance(v, dict)} + except Exception as e: + logger.warning(f"camera providers: settings unreadable ({e}); treating every provider as OFF") + _cache = {} + _cache_ts = time.monotonic() + return _cache + + +def effective(provider: CameraProvider, entry: Optional[dict]) -> Tuple[bool, Optional[str]]: + """(serving?, reason-if-not) for a provider given its settings entry.""" + meta = provider.meta + if not meta.implemented: + return False, "adapter is a stub" + if meta.licence.status == Licence.PROHIBITED: + return False, "source terms forbid this use" + if not entry or not entry.get("enabled"): + return False, "disabled (default)" + if meta.licence.gated: + ack = entry.get("acknowledgement") or {} + if ack.get("licence_status") != meta.licence.status.value: + return False, "licence acknowledgement missing or out of date" + return True, None + + +async def is_serving(provider_id: str) -> bool: + p = PROVIDERS.get(provider_id) + if p is None: + return False + return effective(p, (await get_settings()).get(provider_id))[0] + + +async def set_enabled(provider_id: str, enabled: bool, actor: Tuple[str, str], + acknowledge_licence: bool = False) -> dict: + p = PROVIDERS.get(provider_id) + if p is None: + raise ProviderError(404, f"unknown camera provider '{provider_id}'") + meta, lic = p.meta, p.meta.licence + uid, email = actor + now = datetime.now(timezone.utc).isoformat() + entry: dict = {"enabled": bool(enabled), "updated_at": now, "updated_by": {"uid": uid, "email": email}} + + if enabled: + if not meta.implemented: + raise ProviderError(409, f"{meta.name}: no working adapter -- {meta.notes}") + if lic.status == Licence.PROHIBITED: + raise ProviderError(409, lic.warning() or "prohibited") + if lic.gated: + if not acknowledge_licence: + raise ProviderError(409, f"{meta.name}: licence acknowledgement required", + requires_acknowledgement=True, warning=lic.warning(), + licence=lic.as_dict()) + entry["acknowledgement"] = {"by_uid": uid, "by_email": email, "at": now, + "licence_status": lic.status.value, "warning": lic.warning()} + # Precondition, not a side effect: a gated enable with no audit + # record must not happen. A failure here aborts before the write. + try: + await _audit(actor, "camera_provider.enable_acknowledged", provider_id, + {"licence_status": lic.status.value, "licence_label": lic.label, + "warning": lic.warning(), "acknowledged_at": now}) + except Exception as e: + logger.error(f"camera providers: audit write failed, refusing gated enable of {provider_id} ({e})") + raise ProviderError(503, "audit log unavailable -- a licence-gated provider cannot be enabled " + "without an audit record; try again") + else: + entry["acknowledgement"] = None # re-enabling a gated source needs a fresh acknowledgement + + before = (await get_settings()).get(provider_id) or {} + await fstore.doc_set(COLLECTION, DOC_ID, {"providers": {provider_id: entry}}, merge=True) + invalidate_cache() + if not (enabled and lic.gated): + try: + await _audit(actor, "camera_provider.enable" if enabled else "camera_provider.disable", provider_id, + {"from": bool(before.get("enabled")), "to": bool(enabled), + "licence_status": lic.status.value}) + except Exception as e: # the toggle already happened; never 500 a completed write + logger.error(f"camera providers: audit write failed for {provider_id} ({e})") + logger.info(f"camera provider {provider_id} -> enabled={enabled} by {email or uid}") + return await describe(provider_id) + + +async def _audit(actor: Tuple[str, str], action: str, provider_id: str, details: dict) -> None: + from app.internal import audit + await audit.write_audit(actor_uid=actor[0], actor_email=actor[1], action=action, + details={"provider_id": provider_id, **details}) + + +async def describe(provider_id: str) -> dict: + p = PROVIDERS[provider_id] + entry = (await get_settings()).get(provider_id) or {} + serving, reason = effective(p, entry) + m = p.meta + cat = CATALOGS.get(provider_id) + key_missing = bool(m.key_setting) and not getattr(settings, m.key_setting, None) + return { + "id": m.id, "name": m.name, "region": m.region, "formats": m.formats, + "implemented": m.implemented, "notes": m.notes, "homepage": m.homepage, + "licence": m.licence.as_dict(), + "enabled": bool(entry.get("enabled")), + "serving": serving, "not_serving_reason": reason, + "acknowledgement": entry.get("acknowledgement"), + "updated_at": entry.get("updated_at"), "updated_by": entry.get("updated_by"), + "key_setting": m.key_setting, "key_missing": key_missing, + "health": cat.status() if cat else None, + } + + +async def list_providers() -> List[dict]: + await get_settings() + return [await describe(pid) for pid in PROVIDERS] + + +def credit(provider_id: str) -> dict: + lic = PROVIDERS[provider_id].meta.licence + return {"attribution": lic.attribution, "attribution_url": lic.attribution_url, "disclaimer": lic.disclaimer} + + +async def cameras_in_bbox(bbox: BBox) -> dict: + """Cameras from every SERVING provider whose coverage meets ``bbox``. + Providers that are off are never fetched.""" + conf = await get_settings() + live = [pid for pid, cat in CATALOGS.items() + if effective(PROVIDERS[pid], conf.get(pid))[0] and bbox_intersects(PROVIDERS[pid].meta.coverage, bbox)] + await asyncio.gather(*(CATALOGS[pid].get() for pid in live)) + cams: List[Camera] = [] + providers = [] + for pid in live: + cat = CATALOGS[pid] + hits = in_bbox(cat.items, bbox) + room = max(0, MAX_ITEMS - len(cams)) + cams.extend(hits[:room]) + providers.append({"id": pid, "name": PROVIDERS[pid].meta.name, **credit(pid), + "fetched_at": cat.fetched_at, "error": cat.error, + "total_in_bbox": len(hits), "truncated": len(hits) > room}) + return {"cameras": cams, "providers": providers, + "serving_count": sum(1 for pid in CATALOGS if effective(PROVIDERS[pid], conf.get(pid))[0])} diff --git a/drb-c2-core/app/internal/ny511.py b/drb-c2-core/app/internal/ny511.py index e8708e7..65e37ce 100644 --- a/drb-c2-core/app/internal/ny511.py +++ b/drb-c2-core/app/internal/ny511.py @@ -9,6 +9,25 @@ makes no 511 calls at all. A failed refresh keeps serving the last good data and reports the error and its age to the caller -- an empty layer must never be the only symptom of a dead feed (the AI-silent-failures lesson). + +**Feed status (server-26#198), 2026-10-04:** 511ny.org cut over to the CARS +"511x" platform around 2026-09-24..30 and the legacy ``/api/getcameras`` and +``/api/getevents`` endpoints now answer HTTP 200 with the SPA's HTML. The new +developer feed is granted only after the NYSDOT Developer Access Agreement +request is approved ("If approved for use, you will receive instructions for +accessing the data feeds & documentation" -- 511NY "For Developers" page), so +its URL shape is NOT public. This client therefore keeps the LEGACY request and +record shape behind ``settings.ny511_api_base`` (default: the legacy base) and +``settings.ny511_api_key``. UNVERIFIED until the grant arrives: + * the new base URL and whether it still takes ``?key=&format=json``; + * whether cameras still come as JSON (the public feed list says + "Camera inventory (XML)") and keep the legacy field names; + * the still/HLS URL shape and its CORS headers. +Never point this at the new site's own backend (api-511x-nysdot.carsprogram.org): +the 511NY Terms of Use ban robots and reverse engineering outside the DAA. + +Access is also gated by the camera-provider registry (``ny511`` provider, +default OFF, DAA acknowledgement required) -- see app/internal/cameras/. """ import asyncio import time @@ -20,11 +39,14 @@ import httpx from app.config import settings from app.internal.logger import logger -_BASE = "https://511ny.org/api" +LEGACY_BASE = "https://511ny.org/api" CAMERAS_TTL_S = 60 * 60 # camera list is near-static EVENTS_TTL_S = 2 * 60 # accidents/closures change minute to minute RETRY_AFTER_FAILURE_S = 60 -_DESCRIPTION_MAX = 500 +# The DAA forbids altering "the substance of a report", so descriptions are +# shown as published. This cap only guards against a pathological record; no +# real 511NY description comes near it, and a cut one is marked with an ellipsis. +_DESCRIPTION_MAX = 4000 class _Feed: @@ -61,9 +83,22 @@ async def _fetch(path: str, normalize) -> List[Dict[str, Any]]: if settings.ny511_api_key: params["key"] = settings.ny511_api_key async with httpx.AsyncClient(timeout=20.0) as client: - r = await client.get(f"{_BASE}/{path}", params=params) + base = (settings.ny511_api_base or LEGACY_BASE).rstrip("/") + r = await client.get(f"{base}/{path}", params=params) r.raise_for_status() - raw = r.json() + text = getattr(r, "text", None) + if isinstance(text, str) and text.lstrip()[:1] == "<": + # The 511x camera inventory is published as XML; HTML is the dead + # legacy endpoint serving the SPA shell. + raw = parse_inventory_xml(text) + else: + try: + raw = r.json() + except ValueError as e: + raise ValueError( + "511NY answered neither JSON nor XML (the legacy API has served the 511x " + "site's HTML since the cutover, server-26#198)" + ) from e if not isinstance(raw, list): raise ValueError(f"expected a JSON list, got {type(raw).__name__}") out = [n for n in (normalize(x) for x in raw) if n is not None] @@ -73,6 +108,69 @@ async def _fetch(path: str, normalize) -> List[Dict[str, Any]]: return out +# ── XML camera inventory (511x) -- UNVERIFIED shape ────────────────────────── +# The 511NY "For Developers" page lists the camera feed as "Camera inventory +# (XML)" but publishes no schema (it arrives with the DAA grant). This parser is +# deliberately tolerant: any element carrying latitude + longitude children (or +# attributes) is a record, and child names are matched case-insensitively onto +# the legacy JSON field names so normalize_camera() works unchanged. Replace the +# alias table with the real element names once the grant's documentation is in hand. +_XML_ALIASES = { + "ID": ("id", "cameraid", "camera_id", "deviceid", "device_id"), + "Latitude": ("latitude", "lat"), + "Longitude": ("longitude", "lon", "lng", "long"), + "Name": ("name", "cameraname", "location", "description", "title"), + "RoadwayName": ("roadwayname", "roadway", "road", "route"), + "DirectionOfTravel": ("directionoftravel", "direction"), + "Url": ("url", "imageurl", "image_url", "snapshoturl", "image", "stillurl"), + "VideoUrl": ("videourl", "video_url", "streamurl", "video", "hlsurl"), + "Disabled": ("disabled",), + "Blocked": ("blocked",), +} +_XML_LOOKUP = {alias: key for key, aliases in _XML_ALIASES.items() for alias in aliases} +_XML_MAX_BYTES = 32 * 1024 * 1024 + + +def _local(tag: str) -> str: + return tag.rsplit("}", 1)[-1].lower() + + +def _xml_bool(v: Any) -> Any: + if isinstance(v, str) and v.strip().lower() in ("true", "false", "1", "0"): + return v.strip().lower() in ("true", "1") + return v + + +def parse_inventory_xml(text: str) -> List[Dict[str, Any]]: + """Map an XML camera inventory onto legacy-shaped dicts. Raises on HTML or + on any DTD/entity declaration (stdlib expat would expand entities).""" + head = text.lstrip()[:512].lower() + if head.startswith(" _XML_MAX_BYTES: + raise ValueError("511NY XML inventory larger than the 32 MB safety cap") + if " Optional[tuple]: try: lat, lon = float(x["Latitude"]), float(x["Longitude"]) diff --git a/drb-c2-core/app/main.py b/drb-c2-core/app/main.py index f62b20d..cebf2a6 100644 --- a/drb-c2-core/app/main.py +++ b/drb-c2-core/app/main.py @@ -17,7 +17,7 @@ from app.internal.auth import ( require_node_service_or_firebase_token, ) from app.routers import nodes, systems, calls, upload, tokens, incidents, alerts, admin, trips, places, links, users -from app.routers import enrollment, media, org, waitlist, telemetry, replay, traffic +from app.routers import enrollment, media, org, waitlist, telemetry, replay, traffic, cameras from app.internal import dynsec from app.internal import firestore as fstore @@ -128,6 +128,8 @@ app.include_router(alerts.router, dependencies=[Depends(require_service_or_fi app.include_router(trips.router, dependencies=[Depends(require_service_or_firebase_token)]) app.include_router(places.router, dependencies=[Depends(require_service_or_firebase_token)]) app.include_router(traffic.router, dependencies=[Depends(require_service_or_firebase_token)]) +app.include_router(cameras.router, dependencies=[Depends(require_service_or_firebase_token)]) +app.include_router(cameras.admin_router) # auth: admin only, per endpoint (needs the principal for audit) app.include_router(upload.router) # auth is per-node, handled inline app.include_router(admin.router) # auth is per-endpoint (read: firebase, write: admin) app.include_router(replay.router) # auth: admin only (every route spends or reads a replay run) diff --git a/drb-c2-core/app/routers/cameras.py b/drb-c2-core/app/routers/cameras.py new file mode 100644 index 0000000..8596287 --- /dev/null +++ b/drb-c2-core/app/routers/cameras.py @@ -0,0 +1,51 @@ +"""Public-camera endpoints for the live ops wall. + +``GET /cameras`` -- any signed-in user (auth applied at include time in main.py): +cameras inside a bbox from every provider an admin has enabled, each with the +provider's required attribution, disclaimer and catalog freshness. + +``/admin/camera-providers`` -- platform admin only: list every catalogued +provider with its licence status and health, and toggle it. Gated licences +need ``acknowledge_licence: true``; that acknowledgement is stored with who and +when and written to audit_log (see app/internal/cameras/registry.py). +""" +from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi.responses import JSONResponse +from pydantic import BaseModel + +from app.internal.auth import describe_actor, require_admin_token +from app.internal.cameras import registry + +router = APIRouter(prefix="/cameras", tags=["cameras"]) +admin_router = APIRouter(prefix="/admin/camera-providers", tags=["admin"]) + + +@router.get("") +async def get_cameras( + south: float = Query(..., ge=-90, le=90), + west: float = Query(..., ge=-180, le=180), + north: float = Query(..., ge=-90, le=90), + east: float = Query(..., ge=-180, le=180), +): + if south > north or west > east: + raise HTTPException(400, "bbox must satisfy south<=north and west<=east") + return await registry.cameras_in_bbox((south, west, north, east)) + + +class ProviderToggle(BaseModel): + enabled: bool + acknowledge_licence: bool = False + + +@admin_router.get("") +async def list_camera_providers(_=Depends(require_admin_token)): + return {"providers": await registry.list_providers()} + + +@admin_router.put("/{provider_id}") +async def set_camera_provider(provider_id: str, body: ProviderToggle, principal: dict = Depends(require_admin_token)): + try: + return await registry.set_enabled(provider_id, body.enabled, describe_actor(principal), + acknowledge_licence=body.acknowledge_licence) + except registry.ProviderError as e: + return JSONResponse(status_code=e.code, content={"detail": e.message, **e.extra}) diff --git a/drb-c2-core/app/routers/traffic.py b/drb-c2-core/app/routers/traffic.py index da80ffa..e8eb352 100644 --- a/drb-c2-core/app/routers/traffic.py +++ b/drb-c2-core/app/routers/traffic.py @@ -4,6 +4,7 @@ from typing import Optional from fastapi import APIRouter, HTTPException, Query from app.internal import ny511 +from app.internal.cameras import registry router = APIRouter(prefix="/traffic", tags=["traffic"]) @@ -27,6 +28,13 @@ async def get_511( if south > north or west > east: raise HTTPException(400, "bbox must satisfy south<=north and west<=east") wanted = {s.strip() for s in (layers or "").split(",") if s.strip()} + # 511NY data (events included) is covered by the same NYSDOT DAA as its + # cameras, so the whole feed follows the ny511 camera provider's switch, + # which defaults OFF and needs a licence acknowledgement (server-26#198). + if not await registry.is_serving("ny511"): + off = {"fetched_at": None, "error": None, "total_in_bbox": 0, "truncated": False, "disabled": True} + return {**{n: [] for n in ("cameras", "events") if n in wanted}, + **{f"{n}_status": off for n in ("cameras", "events") if n in wanted}} feeds = {name: getattr(ny511, name) for name in ("cameras", "events") if name in wanted} await asyncio.gather(*(f.get() for f in feeds.values())) @@ -35,4 +43,5 @@ async def get_511( hits = ny511.in_bbox(feed.items, south, west, north, east) out[name] = hits[:MAX_ITEMS] out[f"{name}_status"] = {**_feed_status(feed), "total_in_bbox": len(hits), "truncated": len(hits) > MAX_ITEMS} + out["credit"] = registry.credit("ny511") # DAA: source credit + as-is disclaimer travel with the data return out diff --git a/drb-c2-core/tests/test_camera_providers.py b/drb-c2-core/tests/test_camera_providers.py new file mode 100644 index 0000000..9b3af0c --- /dev/null +++ b/drb-c2-core/tests/test_camera_providers.py @@ -0,0 +1,283 @@ +"""Live ops wall camera providers: registry, licence gate, admin routes, adapters. + +What is held here: + * every provider defaults OFF, and an off provider is never fetched; + * stubs and PROHIBITED sources can never be enabled; + * a gated licence needs ``acknowledge_licence``, which is stored with who/when + and audited -- and an audit failure refuses the enable (fail closed); + * an acknowledgement given for a different licence status no longer counts; + * the admin routes are admin-only; + * each implemented adapter parses its (verified) upstream shape, pins imagery + to the agency's hosts, and treats an all-unparseable payload as an error. +""" +import asyncio +from unittest.mock import AsyncMock, patch + +import pytest +from fastapi import HTTPException +from fastapi.testclient import TestClient + +from app.internal.auth import require_admin_token, require_service_or_firebase_token +from app.internal.cameras import base, general, regional, registry +from app.internal.cameras.base import Licence +from app.main import app + +client = TestClient(app) +ADMIN = {"uid": "admin-1", "email": "admin@example.com", "role": "admin"} + + +class FakeStore: + def __init__(self): + self.docs = {} + self.audit = [] + + async def doc_get(self, col, doc): + return self.docs.get((col, doc)) + + async def doc_set(self, col, doc, data, merge=True): + cur = self.docs.setdefault((col, doc), {}) + if col == "audit_log": + self.audit.append(data) + return + for k, v in data.items(): # one-level deep merge is all registry needs + if merge and isinstance(v, dict) and isinstance(cur.get(k), dict): + cur[k] = {**cur[k], **v} + else: + cur[k] = v + + +@pytest.fixture(autouse=True) +def store(): + s = FakeStore() + registry.invalidate_cache() + for cat in registry.CATALOGS.values(): + cat.items, cat.fetched_at, cat.error, cat._next_attempt = [], None, None, 0.0 + with patch.object(registry.fstore, "doc_get", s.doc_get), patch.object(registry.fstore, "doc_set", s.doc_set), \ + patch("app.internal.audit.fstore.doc_set", s.doc_set): + yield s + registry.invalidate_cache() + app.dependency_overrides.clear() + + +def run(coro): + return asyncio.run(coro) + + +# ── registry / gate ────────────────────────────────────────────────────────── + +def test_every_catalogued_source_is_registered_and_defaults_off(): + listed = run(registry.list_providers()) + ids = {p["id"] for p in listed} + assert {"ny511", "nyc_dot", "westchester_dpw", "caltrans", "deldot", "tfl", "ontario511", "drivebc", + "calgary", "nsw", "fintraffic", "txdot", "austin", "tallinn", "tarktee", "warendorf"} <= ids + assert all(p["enabled"] is False and p["serving"] is False for p in listed) + assert all(p["licence"]["label"] for p in listed) + + +def test_off_provider_is_never_fetched(): + fetch = AsyncMock(return_value=[]) + with patch.object(registry.PROVIDERS["tfl"], "fetch_catalog", fetch): + out = run(registry.cameras_in_bbox((51.4, -0.2, 51.6, 0.1))) + assert out["cameras"] == [] and out["providers"] == [] and fetch.await_count == 0 + + +def test_stub_and_prohibited_cannot_be_enabled(): + for pid in ("nyc_dot", "westchester_dpw", "ctroads", "trafficland"): + with pytest.raises(registry.ProviderError) as e: + run(registry.set_enabled(pid, True, ("u", "e"), acknowledge_licence=True)) + assert e.value.code == 409 + + +def test_attribution_licence_enables_without_acknowledgement_and_is_audited(store): + out = run(registry.set_enabled("nsw", True, ("u1", "a@x"))) + assert out["serving"] is True and out["acknowledgement"] is None + assert store.audit[-1]["action"] == "camera_provider.enable" + assert store.audit[-1]["actor_email"] == "a@x" + + +def test_gated_licence_requires_acknowledgement(store): + with pytest.raises(registry.ProviderError) as e: + run(registry.set_enabled("ny511", True, ("u1", "a@x"))) + assert e.value.code == 409 and e.value.extra["requires_acknowledgement"] is True + assert "Developer Access Agreement" in e.value.extra["warning"] + assert not store.audit and not run(registry.is_serving("ny511")) + + out = run(registry.set_enabled("ny511", True, ("u1", "a@x"), acknowledge_licence=True)) + assert out["serving"] is True + ack = out["acknowledgement"] + assert ack["by_uid"] == "u1" and ack["by_email"] == "a@x" and ack["at"] + assert ack["licence_status"] == Licence.AGREEMENT_REQUIRED.value + assert store.audit[-1]["action"] == "camera_provider.enable_acknowledged" + assert store.audit[-1]["details"]["provider_id"] == "ny511" + + +def test_gated_enable_refused_when_audit_cannot_be_written(store): + with patch.object(registry, "_audit", AsyncMock(side_effect=RuntimeError("firestore down"))): + with pytest.raises(registry.ProviderError) as e: + run(registry.set_enabled("deldot", True, ("u", "e"), acknowledge_licence=True)) + assert e.value.code == 503 + assert not run(registry.is_serving("deldot")) + + +def test_disable_clears_acknowledgement(store): + run(registry.set_enabled("deldot", True, ("u", "e"), acknowledge_licence=True)) + out = run(registry.set_enabled("deldot", False, ("u", "e"))) + assert out["serving"] is False and out["acknowledgement"] is None + + +def test_stale_acknowledgement_stops_serving(): + p = registry.PROVIDERS["deldot"] + entry = {"enabled": True, "acknowledgement": {"licence_status": "non_commercial"}} + assert registry.effective(p, entry) == (False, "licence acknowledgement missing or out of date") + entry["acknowledgement"]["licence_status"] = p.meta.licence.status.value + assert registry.effective(p, entry) == (True, None) + + +def test_unreadable_settings_fail_closed(): + with patch.object(registry.fstore, "doc_get", AsyncMock(side_effect=RuntimeError("boom"))): + registry.invalidate_cache() + assert run(registry.get_settings()) == {} + + +def test_query_returns_attribution_and_skips_providers_outside_bbox(store): + run(registry.set_enabled("nsw", True, ("u", "e"))) + run(registry.set_enabled("tfl", True, ("u", "e"))) + cam = base.make_camera("nsw", "x", -33.87, 151.2, name="Sydney", image_url="https://webcams.transport.nsw.gov.au/a.jpg") + nsw_fetch, tfl_fetch = AsyncMock(return_value=[cam]), AsyncMock(return_value=[]) + with patch.object(registry.PROVIDERS["nsw"], "fetch_catalog", nsw_fetch), \ + patch.object(registry.PROVIDERS["tfl"], "fetch_catalog", tfl_fetch): + out = run(registry.cameras_in_bbox((-34.0, 151.0, -33.7, 151.4))) + assert [c["id"] for c in out["cameras"]] == ["nsw:x"] + assert out["providers"][0]["attribution"] == "Transport for NSW, CC BY 4.0" + assert tfl_fetch.await_count == 0 # London never fetched for a Sydney bbox + + +# ── admin routes ───────────────────────────────────────────────────────────── + +def test_admin_routes_require_admin(): + def deny(): + raise HTTPException(403, "Admin access required") + app.dependency_overrides[require_admin_token] = deny + assert client.get("/admin/camera-providers").status_code == 403 + assert client.put("/admin/camera-providers/nsw", json={"enabled": True}).status_code == 403 + assert client.get("/admin/camera-providers", headers={}).status_code == 403 + + +def test_admin_route_toggle_and_gate(store): + app.dependency_overrides[require_admin_token] = lambda: ADMIN + r = client.put("/admin/camera-providers/caltrans", json={"enabled": True}) + assert r.status_code == 409 and r.json()["requires_acknowledgement"] is True + r = client.put("/admin/camera-providers/caltrans", json={"enabled": True, "acknowledge_licence": True}) + assert r.status_code == 200 and r.json()["acknowledgement"]["by_email"] == "admin@example.com" + assert client.put("/admin/camera-providers/nope", json={"enabled": True}).status_code == 404 + listed = client.get("/admin/camera-providers").json()["providers"] + assert next(p for p in listed if p["id"] == "caltrans")["serving"] is True + + +def test_cameras_route_rejects_inverted_bbox(): + app.dependency_overrides[require_service_or_firebase_token] = lambda: {"uid": "v"} + assert client.get("/cameras", params={"south": 2, "west": 0, "north": 1, "east": 1}).status_code == 400 + r = client.get("/cameras", params={"south": 40.9, "west": -74.0, "north": 41.4, "east": -73.4}) + assert r.status_code == 200 and r.json()["cameras"] == [] + + +# ── adapters ───────────────────────────────────────────────────────────────── + +def _with_payload(payload): + return patch.object(general, "get_json", AsyncMock(return_value=payload)) + + +def test_heading_from_text(): + assert base.heading_from_text("Northbound") == 0 + assert base.heading_from_text("WB") == 270 + assert base.heading_from_text("SE") == 135 + assert base.heading_from_text("Both directions") is None + assert base.heading_from_text("Unknown") is None + + +def test_https_on_pins_hosts(): + assert base.https_on("https://cwwp2.dot.ca.gov/x.jpg", "dot.ca.gov") + assert base.https_on("http://cwwp2.dot.ca.gov/x.jpg", "dot.ca.gov") is None + assert base.https_on("https://dot.ca.gov.evil.com/x.jpg", "dot.ca.gov") is None + + +def test_caltrans_adapter(): + row = {"cctv": {"inService": "true", "index": "1", "location": { + "district": "4", "locationName": "TV102 -- I-580 : West of SR-24", "latitude": "37.82539", + "longitude": "-122.27291", "direction": "West", "route": "I-580"}, + "imageData": {"streamingVideoURL": "https://wzmedia.dot.ca.gov/D4/x.stream/playlist.m3u8", + "static": {"currentImageURL": "https://cwwp2.dot.ca.gov/data/d4/cctv/image/a/a.jpg"}}}} + off = {"cctv": {**row["cctv"], "inService": "false"}} + with _with_payload({"data": [row, off]}): + cams = run(general.CaltransProvider().fetch_catalog()) + assert len(cams) == len(general.CaltransProvider.DISTRICTS) # same row per district fetch + c = cams[0] + assert c["id"] == "caltrans:d4-TV102" and c["heading_deg"] == 270 and c["video_url"].endswith(".m3u8") + + +def test_deldot_adapter_requires_https_hls(): + good = {"id": "KCAM001", "title": "DE 1", "lat": 38.99, "lon": -75.44, "enabled": True, "status": "Active", + "urls": {"m3u8s": "https://video.deldot.gov:443/live/KCAM001.stream/playlist.m3u8", + "m3u8": "http://video.deldot.gov:1935/live/KCAM001.stream/playlist.m3u8"}} + down = {**good, "id": "KCAM002", "status": "Unavailable"} + with _with_payload({"videoCameras": [good, down]}): + cams = run(general.DelDotProvider().fetch_catalog()) + assert [c["id"] for c in cams] == ["deldot:KCAM001"] and cams[0]["image_url"] is None + + +def test_drivebc_calgary_tfl_nsw_fintraffic_adapters(): + with _with_payload([{"id": 1176, "name": "Lougheed", "is_on": True, "should_appear": True, "orientation": "W", + "location": {"coordinates": [-122.5, 49.2]}, "highway_description": "Hwy 7"}]): + assert run(general.DriveBcProvider().fetch_catalog())[0]["image_url"] == "https://www.drivebc.ca/images/1176.jpg" + with _with_payload([{"camera_url": {"url": "http://trafficcam.calgary.ca/loc86.jpg"}, "camera_location": "Stoney", + "point": {"coordinates": [-113.97, 50.9]}, "quadrant": "SE"}]): + c = run(general.CalgaryProvider().fetch_catalog())[0] + assert c["image_url"] == "https://trafficcam.calgary.ca/loc86.jpg" and c["id"] == "calgary:loc86" + with _with_payload([{"id": "JamCams_00002.00865", "commonName": "A406", "lat": 51.6, "lon": -0.016, + "additionalProperties": [{"key": "available", "value": "true"}, + {"key": "imageUrl", "value": general.TflProvider.IMAGE_PREFIX + "a.jpg"}]}]): + assert run(general.TflProvider().fetch_catalog())[0]["id"] == "tfl:00002.00865" + with _with_payload({"features": [{"id": "abc", "geometry": {"coordinates": [151.1, -34.0]}, + "properties": {"title": "5 Ways", "direction": "W", + "href": "https://webcams.transport.nsw.gov.au/x.jpeg"}}]}): + assert run(general.NswProvider().fetch_catalog())[0]["heading_deg"] == 270 + with _with_payload({"features": [{"geometry": {"coordinates": [23.99, 60.05, 0]}, + "properties": {"id": "C01503", "name": "kt51", "collectionStatus": "GATHERING", + "presets": [{"id": "C0150301", "inCollection": True}, + {"id": "C0150302", "inCollection": False}]}}]}): + cams = run(general.FintrafficProvider().fetch_catalog()) + assert [c["image_url"] for c in cams] == ["https://weathercam.digitraffic.fi/C0150301.jpg"] + + +def test_schema_change_is_an_error_not_an_empty_catalog(): + with _with_payload([{"renamed": True}]): + with pytest.raises(ValueError, match="schema"): + run(general.DriveBcProvider().fetch_catalog()) + + +def test_ontario_needs_its_key(monkeypatch): + monkeypatch.setattr(general.settings, "ontario511_api_key", None, raising=False) + with pytest.raises(RuntimeError, match="ONTARIO511_API_KEY"): + run(general.Ontario511Provider().fetch_catalog()) + + +def test_ny511_provider_wraps_legacy_client_and_carries_daa_metadata(): + row = {"id": "NYSDOT-1", "lat": 41.03, "lon": -73.76, "name": "I-287", "roadway": "I-287", + "direction": "Northbound", "image_url": "https://x/1", "video_url": None} + with patch.object(regional.ny511, "_fetch", AsyncMock(return_value=[row])): + cams = run(regional.Ny511Provider().fetch_catalog()) + assert cams[0]["id"] == "ny511:NYSDOT-1" and cams[0]["heading_deg"] == 0 + lic = regional.Ny511Provider.meta.licence + assert lic.attribution == "powered by 511NY" and lic.attribution_url == "https://511ny.org" + assert "with all faults" in lic.disclaimer and lic.gated + + +def test_failed_refresh_keeps_last_good_catalog(): + cat = registry.CATALOGS["nsw"] + cam = base.make_camera("nsw", "x", -33.8, 151.2) + with patch.object(cat.provider, "fetch_catalog", AsyncMock(return_value=[cam])): + run(cat.get()) + cat._next_attempt = 0.0 + with patch.object(cat.provider, "fetch_catalog", AsyncMock(side_effect=RuntimeError("down"))): + run(cat.get()) + assert cat.items == [cam] and "down" in cat.error and cat.fetched_at diff --git a/drb-c2-core/tests/test_ny511.py b/drb-c2-core/tests/test_ny511.py index ec86fdd..dc1d578 100644 --- a/drb-c2-core/tests/test_ny511.py +++ b/drb-c2-core/tests/test_ny511.py @@ -43,11 +43,14 @@ def test_normalize_camera_skips_disabled_blocked_and_zero_coords(): assert ny511.normalize_camera({**CAM, "Latitude": 0, "Longitude": 0}) is None -def test_normalize_event_parses_day_first_dates_and_truncates_description(): +def test_normalize_event_parses_day_first_dates_and_keeps_description_whole(): e = ny511.normalize_event(EVENT) assert e["start_local"] == "2026-09-28T09:00:00" # DD/MM, not MM/DD assert e["planned_end_local"] is None - assert len(e["description"]) == ny511._DESCRIPTION_MAX + 1 + # DAA: "You may not alter the substance of a report" -- no 500-char cut any more. + assert e["description"] == EVENT["Description"] + long = ny511.normalize_event({**EVENT, "Description": "y" * (ny511._DESCRIPTION_MAX + 50)}) + assert len(long["description"]) == ny511._DESCRIPTION_MAX + 1 # pathological only def test_failed_refresh_keeps_last_good_data_and_reports_error(): @@ -80,14 +83,29 @@ def test_schema_change_is_an_error_not_an_empty_layer(): assert "schema" in str(e) +def _serving(value: bool): + from app.internal.cameras import registry + return patch.object(registry, "is_serving", AsyncMock(return_value=value)) + + +def test_endpoint_is_off_until_the_ny511_provider_is_enabled(): + with _serving(False): + r = client.get("/traffic/511", params={"south": 40.9, "west": -74.0, "north": 41.4, "east": -73.4}) + body = r.json() + assert r.status_code == 200 and body["cameras"] == [] and body["events"] == [] + assert body["events_status"]["disabled"] is True + + def test_endpoint_filters_to_bbox_and_reports_status(): far = {**CAM, "ID": "NYSDOT-2", "Latitude": 42.9, "Longitude": -78.8} # Buffalo for feed, rows, norm in ((ny511.cameras, [CAM, far], ny511.normalize_camera), (ny511.events, [EVENT], ny511.normalize_event)): feed.items = [norm(r) for r in rows] feed.fetched_at, feed._next_attempt = 1.0, float("inf") - r = client.get("/traffic/511", params={"south": 40.9, "west": -74.0, "north": 41.4, "east": -73.4}) + with _serving(True): + r = client.get("/traffic/511", params={"south": 40.9, "west": -74.0, "north": 41.4, "east": -73.4}) assert r.status_code == 200 body = r.json() + assert body["credit"]["attribution"] == "powered by 511NY" and "as is" in body["credit"]["disclaimer"] assert [c["id"] for c in body["cameras"]] == ["NYSDOT-1"] assert body["cameras_status"] == {"fetched_at": 1.0, "error": None, "total_in_bbox": 1, "truncated": False} assert len(body["events"]) == 1 @@ -96,3 +114,24 @@ def test_endpoint_filters_to_bbox_and_reports_status(): def test_endpoint_rejects_inverted_bbox(): r = client.get("/traffic/511", params={"south": 41.4, "west": -74.0, "north": 40.9, "east": -73.4}) assert r.status_code == 400 + + +def test_xml_inventory_maps_onto_legacy_fields(): + xml = """ + NYSDOT-941.05-73.78 + I-287 at Exit 4I-287Eastbound + https://example.invalid/still.jpgfalse + true + """ + rows = ny511.parse_inventory_xml(xml) + cams = [ny511.normalize_camera(r) for r in rows] + assert cams[0]["id"] == "NYSDOT-9" and cams[0]["image_url"] == "https://example.invalid/still.jpg" + assert cams[1] is None # disabled flag honoured from XML text + + +def test_xml_parser_refuses_html_and_entities(): + import pytest + with pytest.raises(ValueError, match="HTML"): + ny511.parse_inventory_xml("511NY") + with pytest.raises(ValueError, match="DTD"): + ny511.parse_inventory_xml(']>&a;') diff --git a/drb-frontend/app/admin/page.tsx b/drb-frontend/app/admin/page.tsx index 5b2cf44..682a927 100644 --- a/drb-frontend/app/admin/page.tsx +++ b/drb-frontend/app/admin/page.tsx @@ -3,6 +3,7 @@ import { useAuth } from "@/components/AuthProvider"; import { c2api } from "@/lib/c2api"; import { ReplayTab } from "@/components/admin/ReplayTab"; +import { CameraProvidersTab } from "@/components/admin/CameraProvidersTab"; import { useEffect, useState, useRef, useCallback } from "react"; import { useRouter } from "next/navigation"; import type { UserRecord, AuditEntry, UserRole, CallRecord } from "@/lib/types"; @@ -1267,10 +1268,11 @@ function SttEvalTab() { // Main admin page // --------------------------------------------------------------------------- -type AdminTab = "features" | "correlation" | "replay" | "users" | "audit" | "calls" | "eval"; +type AdminTab = "features" | "cameras" | "correlation" | "replay" | "users" | "audit" | "calls" | "eval"; const TAB_LABELS: { key: AdminTab; label: string }[] = [ { key: "features", label: "AI Features" }, + { key: "cameras", label: "Camera providers" }, { key: "correlation", label: "Correlation Debug" }, { key: "replay", label: "Replay" }, { key: "calls", label: "Calls" }, @@ -1297,7 +1299,7 @@ export default function AdminPage() { if (!isAdmin) return null; // Users/Audit tabs benefit from full width; everything else is narrow - const wide = tab === "users" || tab === "audit" || tab === "replay"; + const wide = tab === "users" || tab === "audit" || tab === "replay" || tab === "cameras"; return (
@@ -1318,6 +1320,7 @@ export default function AdminPage() {
{tab === "features" && } + {tab === "cameras" && } {tab === "correlation" && } {tab === "replay" && } {tab === "calls" && } diff --git a/drb-frontend/app/globals.css b/drb-frontend/app/globals.css index 083802f..215b20b 100644 --- a/drb-frontend/app/globals.css +++ b/drb-frontend/app/globals.css @@ -246,3 +246,18 @@ html:not(.dark) .marketing-hero-bg { .skeleton { animation: skeleton-pulse 1.6s ease-in-out infinite; } + +/* ── Live ops wall transcript ticker ───────────────────────────────────────── + * The belt holds the items twice, so translating by -50% loops seamlessly. + * Paused on hover/focus; reduced-motion users get a static, scrollable row. */ +@keyframes drb-ticker { + from { transform: translateX(0); } + to { transform: translateX(-50%); } +} +.drb-ticker-belt { animation: drb-ticker var(--drb-ticker-duration, 90s) linear infinite; } +.drb-ticker:hover .drb-ticker-belt, +.drb-ticker:focus-within .drb-ticker-belt { animation-play-state: paused; } +@media (prefers-reduced-motion: reduce) { + .drb-ticker-belt { animation: none; } + .drb-ticker { overflow-x: auto; } +} diff --git a/drb-frontend/components/CameraFeed.tsx b/drb-frontend/components/CameraFeed.tsx index 8a0d736..11f5ea4 100644 --- a/drb-frontend/components/CameraFeed.tsx +++ b/drb-frontend/components/CameraFeed.tsx @@ -1,41 +1,217 @@ "use client"; -import { useEffect, useRef, useState } from "react"; +import { useEffect, useRef, useState, type RefObject } from "react"; import type HlsType from "hls.js"; -import type { Ny511Camera } from "@/lib/types"; +import type { Camera, CameraCredit } from "@/lib/types"; -// 511NY stills are served with max-age=60, so refreshing faster buys nothing. +// ── Feed hygiene (design-canvas/gods-eye-view-notes.md §2, top-5 #2) ───────── +// A 24/7 wall must not burn bandwidth on feeds nobody can see: +// * hidden tab or off-screen tile → no fetches, live video torn down; +// * at most MAX_LIVE_HLS live decoders page-wide (GEV uses 2); extra tiles +// show their still with a "live cap" badge (the drill-down gets one extra); +// * a failed refresh keeps the last good frame with a "stale Nm" badge; +// * a 64×36 downsample signature detects a frozen camera (same frame for +// FROZEN_AFTER_MS) without repainting it; a stalled HLS stream falls back. +// Video is played straight from the agency's host (no DRB egress). + +// Most stills are cached ~60 s upstream, so refreshing faster buys nothing. const SNAPSHOT_REFRESH_MS = 60_000; -// A failed stream is not given up on for good: 511NY feeds drop out briefly, -// and a permanent fallback left pinned tiles showing a stale still image. const VIDEO_RETRY_MS = 30_000; -// In-place recoveries hls.js gets per session before we fall back to the snapshot. const MAX_HLS_RECOVERIES = 3; +const VIDEO_STALL_MS = 20_000; +const FROZEN_AFTER_MS = 20 * 60_000; // > the slowest provider cadence (Fintraffic ~10 min) +export const MAX_LIVE_HLS = 2; + +// ── Page-wide live-stream leases ───────────────────────────────────────────── +const holders = new Set(); +const leaseListeners = new Set<() => void>(); +let nextLeaseId = 1; + +function tryAcquire(id: number, high: boolean): boolean { + if (holders.has(id)) return true; + if (holders.size < MAX_LIVE_HLS + (high ? 1 : 0)) { + holders.add(id); + return true; + } + return false; +} + +function release(id: number) { + if (holders.delete(id)) leaseListeners.forEach((l) => l()); +} + +function useLiveLease(want: boolean, high: boolean): boolean { + const idRef = useRef(0); + if (!idRef.current) idRef.current = nextLeaseId++; + const [granted, setGranted] = useState(false); + useEffect(() => { + const id = idRef.current; + if (!want) { release(id); setGranted(false); return; } + const attempt = () => setGranted(tryAcquire(id, high)); + attempt(); + leaseListeners.add(attempt); + return () => { leaseListeners.delete(attempt); release(id); }; + }, [want, high]); + return granted; +} + +// ── Visibility: tab shown AND element on screen ────────────────────────────── +function useOnScreen(ref: RefObject): boolean { + const [tabVisible, setTabVisible] = useState(true); + const [inView, setInView] = useState(true); + useEffect(() => { + const onVis = () => setTabVisible(!document.hidden); + onVis(); + document.addEventListener("visibilitychange", onVis); + return () => document.removeEventListener("visibilitychange", onVis); + }, []); + useEffect(() => { + const el = ref.current; + if (!el || typeof IntersectionObserver === "undefined") return; + const io = new IntersectionObserver((entries) => setInView(entries.some((e) => e.isIntersecting)), { rootMargin: "100px" }); + io.observe(el); + return () => io.disconnect(); + }, [ref]); + return tabVisible && inView; +} + +// Origins whose stills refuse CORS: shown without crossOrigin, no signature. +const noCorsOrigins = new Set(); +function originOf(url: string): string { + try { return new URL(url).origin; } catch { return url; } +} +function withBust(url: string, t: number): string { + return `${url}${url.includes("?") ? "&" : "?"}t=${t}`; +} + +function signature(img: HTMLImageElement): number[] | null { + try { + const c = document.createElement("canvas"); + c.width = 64; + c.height = 36; + const ctx = c.getContext("2d", { willReadFrequently: true }); + if (!ctx) return null; + ctx.drawImage(img, 0, 0, 64, 36); + const d = ctx.getImageData(0, 0, 64, 36).data; + const out: number[] = []; + for (let i = 0; i < d.length; i += 4) out.push((d[i] + d[i + 1] + d[i + 2]) / 3); + return out; + } catch { + return null; // tainted canvas + } +} +function sameFrame(a: number[] | null, b: number[] | null): boolean { + if (!a || !b || a.length !== b.length) return false; + let diff = 0; + for (let i = 0; i < a.length; i++) diff += Math.abs(a[i] - b[i]); + return diff / a.length < 1.5; +} + +function ageLabel(ms: number): string { + const m = Math.floor(ms / 60_000); + return m < 1 ? "<1m" : m < 60 ? `${m}m` : `${Math.floor(m / 60)}h${m % 60 ? ` ${m % 60}m` : ""}`; +} +function clock(t: number): string { + return new Date(t).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit" }); +} + +interface Props { + camera: Camera; + className?: string; + /** "still" never starts video (popups, thumbnails). */ + mode?: "auto" | "still"; + /** The drill-down camera may take one stream past the page-wide cap. */ + priority?: "normal" | "high"; + /** Required attribution for this provider — rendered on the imagery itself. */ + credit?: CameraCredit | null; +} /** - * One 511NY camera: the live HLS stream when the camera has one (NYSDOT's - * skyvdn hosts send Access-Control-Allow-Origin: *), otherwise its snapshot on - * a timer. A stream that fails falls back to the snapshot rather than a black box, - * and video is retried every VIDEO_RETRY_MS while it stays down. + * One public camera: live HLS when the camera has a stream and a lease is + * free, otherwise its still on a timer — with honest state badges and the + * provider's attribution on the frame. A stream that fails or stalls falls + * back to the still and is retried every VIDEO_RETRY_MS. */ -export function CameraFeed({ camera, className }: { camera: Ny511Camera; className?: string }) { +export function CameraFeed({ camera, className, mode = "auto", priority = "normal", credit }: Props) { + const rootRef = useRef(null); const videoRef = useRef(null); + const onScreen = useOnScreen(rootRef); const [videoFailed, setVideoFailed] = useState(false); - const [tick, setTick] = useState(() => Date.now()); - const useVideo = !!camera.video_url && !videoFailed; + const wantsVideo = mode === "auto" && !!camera.video_url && !videoFailed && onScreen; + const leased = useLiveLease(wantsVideo, priority === "high"); + const useVideo = wantsVideo && leased; + const capped = wantsVideo && !leased; + + // Still state + const [shownSrc, setShownSrc] = useState(null); + const [corsOk, setCorsOk] = useState(true); + const [lastGoodAt, setLastGoodAt] = useState(null); + const [frameSince, setFrameSince] = useState(null); // when the shown frame first appeared + const [stale, setStale] = useState(false); + const [now, setNow] = useState(() => Date.now()); + const sigRef = useRef(null); + + // Reset when the camera changes + useEffect(() => { + setShownSrc(null); setLastGoodAt(null); setFrameSince(null); setStale(false); setVideoFailed(false); + sigRef.current = null; + }, [camera.id]); useEffect(() => { - if (useVideo) return; - const id = setInterval(() => setTick(Date.now()), SNAPSHOT_REFRESH_MS); + const id = setInterval(() => { if (!document.hidden) setNow(Date.now()); }, 15_000); return () => clearInterval(id); - }, [useVideo]); + }, []); + // Still polling — only while on screen and not playing video + useEffect(() => { + const url = camera.image_url; + if (useVideo || !onScreen || !url) return; + let cancelled = false; + let probe: HTMLImageElement | null = null; + const load = (allowCors: boolean) => { + const origin = originOf(url); + const cors = allowCors && !noCorsOrigins.has(origin); + const src = withBust(url, Date.now()); + probe = new Image(); + if (cors) probe.crossOrigin = "anonymous"; + probe.onload = () => { + if (cancelled || !probe) return; + const t = Date.now(); + const sig = cors ? signature(probe) : null; + const unchanged = sameFrame(sig, sigRef.current); + sigRef.current = sig; + setStale(false); + setLastGoodAt(t); + setCorsOk(cors); + if (!unchanged) { + setShownSrc(src); + setFrameSince(t); + } + }; + probe.onerror = () => { + if (cancelled) return; + if (cors) { noCorsOrigins.add(origin); load(false); return; } + setStale(true); // keep the last good frame on screen + }; + probe.src = src; + }; + load(true); + const id = setInterval(() => load(true), SNAPSHOT_REFRESH_MS); + return () => { + cancelled = true; + clearInterval(id); + if (probe) { probe.onload = null; probe.onerror = null; probe.src = ""; } // abort in-flight + }; + }, [camera.image_url, useVideo, onScreen]); + + // Retry a failed stream later useEffect(() => { if (!videoFailed || !camera.video_url) return; const id = setTimeout(() => setVideoFailed(false), VIDEO_RETRY_MS); return () => clearTimeout(id); }, [videoFailed, camera.video_url]); + // HLS player + stall watchdog useEffect(() => { const video = videoRef.current; const src = camera.video_url; @@ -65,20 +241,58 @@ export function CameraFeed({ camera, className }: { camera: Ny511Camera; classNa }) .catch(() => setVideoFailed(true)); } + let lastT = -1; + let lastMove = Date.now(); + const watchdog = setInterval(() => { + if (video.currentTime !== lastT) { lastT = video.currentTime; lastMove = Date.now(); return; } + if (Date.now() - lastMove > VIDEO_STALL_MS) setVideoFailed(true); // frozen stream → still + }, 5_000); return () => { cancelled = true; + clearInterval(watchdog); hls?.destroy(); video.removeAttribute("src"); video.load(); // stop the download, not just the playback }; }, [useVideo, camera.video_url]); - if (useVideo) { - return