diff --git a/drb-edge-node/app/internal/credentials.py b/drb-edge-node/app/internal/credentials.py index d56213e..515470f 100644 --- a/drb-edge-node/app/internal/credentials.py +++ b/drb-edge-node/app/internal/credentials.py @@ -3,8 +3,12 @@ Manages the persisted node API key, plus the local-auth signing material used by app/internal/auth.py. The API key is provisioned by the C2 server after an admin approves the node. -It arrives via MQTT and is saved to /configs/credentials.json so it survives -container restarts. +install.sh fetches it over HTTPS (GET /nodes/{id}/credentials + the pickup +secret it saved to /configs/pickup_secret) and writes /configs/credentials.json; +the running app can also fetch it itself (app/internal/key_pickup.py) and +re-read the file at any time (reload_api_key), so a key written after startup +is picked up without a container restart (node-26#14). The legacy MQTT +delivery (nodes/{id}/api_key) still saves here too. The scrypt salt and session-signing secret are generated locally on first boot (never provisioned externally) and persisted the same way, so dashboard @@ -12,12 +16,14 @@ sessions survive a container restart instead of forcing every operator to re-login whenever the node restarts. """ import json +import os import secrets from pathlib import Path from app.config import settings from app.internal.logger import logger _CREDS_FILE = Path(settings.config_path) / "credentials.json" +_PICKUP_FILE = Path(settings.config_path) / "pickup_secret" _api_key: str | None = None _auth_salt: bytes | None = None _session_secret: bytes | None = None @@ -74,6 +80,37 @@ def get_api_key() -> str | None: return _api_key +def reload_api_key() -> str | None: + """Re-read the api_key from disk (install.sh, or an operator, may have + written it after startup). Returns the key now in effect. Only the api_key + is taken from the file — the auth material already in memory stays.""" + global _api_key + try: + data = json.loads(_CREDS_FILE.read_text()) + except FileNotFoundError: + return _api_key + except Exception as e: + logger.warning(f"Could not re-read credentials file: {e}") + return _api_key + key = data.get("api_key") + if key and key != _api_key: + _api_key = key + logger.info("Node API key re-read from disk (changed since it was last loaded).") + return _api_key + + +def get_pickup_secret() -> str | None: + """The one-time pickup secret install.sh saved at enrollment, if any.""" + try: + secret = _PICKUP_FILE.read_text().strip() + except FileNotFoundError: + return None + except Exception as e: + logger.warning(f"Could not read pickup secret: {e}") + return None + return secret or None + + def save_api_key(key: str) -> None: global _api_key _api_key = key @@ -88,4 +125,10 @@ def _write() -> None: data["auth_salt"] = _auth_salt.hex() if _session_secret is not None: data["session_secret"] = _session_secret.hex() - _CREDS_FILE.write_text(json.dumps(data)) + # Atomic replace with 0600: install.sh on the host writes this same file, + # and a torn half-written JSON would lose the key AND the auth material. + tmp = _CREDS_FILE.with_name(_CREDS_FILE.name + ".tmp") + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as f: + f.write(json.dumps(data)) + os.replace(tmp, _CREDS_FILE) diff --git a/drb-edge-node/app/internal/key_pickup.py b/drb-edge-node/app/internal/key_pickup.py new file mode 100644 index 0000000..9c1bb3a --- /dev/null +++ b/drb-edge-node/app/internal/key_pickup.py @@ -0,0 +1,73 @@ +""" +HTTPS api_key pickup — the node's own half of C2's enrollment flow +(Server/drb-c2-core/app/routers/enrollment.py, GET /nodes/{id}/credentials). + +Why this exists (node-26#14): key delivery used to be MQTT-only +(nodes/{id}/api_key), and a dynsec broker refuses a node that has no key — or +a stale one — so a locked-out node could never receive the key that would let +it in. install.sh already did this pickup once at install time; doing it from +the running app too means a node approved after install, or whose key was +reissued, recovers on its own with no SSH and no container restart. + +Auth is the pickup secret install.sh saved to /configs/pickup_secret at +enrollment (the same file, bind-mounted). Only ever sent to settings.c2_url, +over whatever scheme the operator configured (https in production). +""" +from typing import Optional + +import httpx + +from app.config import settings +from app.internal import credentials +from app.internal.logger import logger + +_TIMEOUT = 15 + + +async def fetch_api_key() -> Optional[str]: + """Ask C2 for this node's api_key. Returns it, or None when there isn't one + to be had yet (not approved, no pickup secret, C2 unreachable, rejected). + Never raises — the caller retries on its own backoff.""" + if not settings.c2_url: + logger.warning("Key pickup skipped: C2_URL is not configured.") + return None + secret = credentials.get_pickup_secret() + if not secret: + logger.warning( + f"Key pickup skipped: no {settings.config_path}/pickup_secret. This node was not enrolled by " + "install.sh on this machine — re-run install.sh, or have an admin Reissue key and write it in." + ) + return None + + url = f"{settings.c2_url.rstrip('/')}/nodes/{settings.node_id}/credentials" + try: + async with httpx.AsyncClient(timeout=_TIMEOUT) as client: + r = await client.get(url, headers={"X-Pickup-Secret": secret}) + except Exception as e: + logger.warning(f"Key pickup: could not reach C2 ({e}).") + return None + + if r.status_code == 401: + logger.error( + "Key pickup: C2 rejected the saved pickup secret (401) — it was rotated by a re-enroll elsewhere. " + "Re-run install.sh with --token to recover." + ) + return None + if r.status_code == 404: + logger.error("Key pickup: C2 does not know this node (404) — it was deleted server-side. Re-run install.sh.") + return None + if r.status_code != 200: + logger.warning(f"Key pickup: unexpected HTTP {r.status_code} from C2.") + return None + + try: + body = r.json() + except ValueError: + logger.warning("Key pickup: C2 returned a non-JSON body.") + return None + key = body.get("api_key") if isinstance(body, dict) else None + if not key or not isinstance(key, str): + status = body.get("approval_status") if isinstance(body, dict) else None + logger.info(f"Key pickup: no api_key yet (approval_status={status!r}) — waiting for an admin to approve.") + return None + return key diff --git a/drb-edge-node/app/internal/mqtt_manager.py b/drb-edge-node/app/internal/mqtt_manager.py index f7e3030..39d6332 100644 --- a/drb-edge-node/app/internal/mqtt_manager.py +++ b/drb-edge-node/app/internal/mqtt_manager.py @@ -6,12 +6,30 @@ from typing import Optional, Callable, Awaitable, Dict, Any import paho.mqtt.client as mqtt from app.config import settings from app.internal.logger import logger -from app.internal import credentials +from app.internal import credentials, key_pickup CommandCallback = Callable[[Dict[str, Any]], Awaitable[None]] ConfigCallback = Callable[[Dict[str, Any]], Awaitable[None]] ApiKeyCallback = Callable[[Dict[str, Any]], Awaitable[None]] +# CONNACK codes meaning "your credentials are wrong", as paho reports them: +# MQTT 3.1.1 return codes 4/5, and the MQTT 5 reason codes paho maps them to +# (134 Bad user name or password, 135 Not authorized). +_AUTH_FAILURE_CODES = {4, 5, 134, 135} + +# Credential-recovery backoff (node-26#14). The pickup endpoint is cheap, but a +# node that is simply not approved yet must not hammer C2 for days. +RECOVERY_MIN_DELAY = 30 +RECOVERY_MAX_DELAY = 900 + + +def _is_auth_failure(reason_code) -> bool: + value = getattr(reason_code, "value", reason_code) + if isinstance(value, int) and value in _AUTH_FAILURE_CODES: + return True + text = str(reason_code).lower() + return "not authori" in text or "bad user name or password" in text + class MQTTManager: def __init__(self): @@ -19,6 +37,11 @@ class MQTTManager: self._loop: Optional[asyncio.AbstractEventLoop] = None self._connected = False self._connect_task: Optional[asyncio.Task] = None + # The api_key the CURRENT paho client was built with — not necessarily + # credentials.get_api_key(), which can move ahead of it (MQTT key push, + # install.sh writing the file). Recovery compares against this. + self._client_key: Optional[str] = None + self._recovery_task: Optional[asyncio.Task] = None self.on_command: Optional[CommandCallback] = None self.on_config_push: Optional[ConfigCallback] = None @@ -53,6 +76,7 @@ class MQTTManager: ) api_key = credentials.get_api_key() + self._client_key = api_key if api_key: # Post-cutover auth: broker's dynsec plugin authenticates this # exact (username, password) pair as this node's own client — see @@ -70,18 +94,14 @@ class MQTTManager: client.username_pw_set(settings.mqtt_user, settings.mqtt_pass) else: # No api_key on disk and no legacy shared login configured. A - # dynsec broker (allow_anonymous false) refuses this outright — - # expected, not a bug to route around here: this node hasn't been - # enrolled/approved yet, and the enrollment flow that would fix - # that client-side is a later, separate pass (out of scope here; - # see MQTT-PUBLIC-AUTH-PLAN.md). paho's reconnect_delay_set() - # below bounds the retry rate (2..60s exponential backoff), so - # this degrades to a slow, clearly-logged refusal loop via - # _on_connect's "MQTT connect refused" line — not a hot spin. + # dynsec broker (allow_anonymous false) refuses this outright. + # connect() starts _recover_credentials() in this case, which + # fetches the key over HTTPS (key_pickup.py) and rebuilds this + # client with it — node-26#14. paho's reconnect_delay_set() below + # bounds the refusal loop meanwhile (2..60s backoff). logger.warning( "No API key on disk and no legacy MQTT_USER configured — " - "connecting without credentials; the broker is expected to " - "refuse this until the node is enrolled/approved." + "connecting without credentials while the key is fetched from C2." ) if settings.mqtt_tls: @@ -120,6 +140,10 @@ class MQTTManager: asyncio.run_coroutine_threadsafe(self._flush_offline_buffer(), self._loop) else: logger.error(f"MQTT connect refused: {reason_code}") + if _is_auth_failure(reason_code) and self._loop is not None: + # paho thread -> event loop. A refused login never fixes itself + # by retrying the same password, so go get the right one. + self._loop.call_soon_threadsafe(self._start_recovery, f"broker refused login ({reason_code})") def _on_disconnect(self, client, userdata, disconnect_flags, reason_code, properties): self._connected = False @@ -144,6 +168,72 @@ class MQTTManager: self._loop = asyncio.get_event_loop() self._client = self._build_client() self._connect_task = asyncio.create_task(self._connect_with_retry()) + if not self._client_key and not settings.mqtt_user: + self._start_recovery("no API key at startup") + + # ------------------------------------------------------------------ + # Credential self-heal (node-26#14) + # + # A node with no key, or a stale one, used to sit in an MQTT refusal loop + # forever: key delivery was MQTT-only, and credentials were read once at + # startup. Now: re-read credentials.json (install.sh may have written it), + # else fetch from C2 over HTTPS with the pickup secret; if that yields a + # key different from the one the client is using, rebuild the client with + # it in place. No container restart. + # ------------------------------------------------------------------ + def _start_recovery(self, why: str) -> None: + if self._recovery_task and not self._recovery_task.done(): + return + logger.warning(f"MQTT credential recovery started: {why}") + self._recovery_task = asyncio.create_task(self._recover_credentials()) + + async def _recover_credentials(self) -> None: + delay = RECOVERY_MIN_DELAY + while True: + tried = self._client_key + key = credentials.reload_api_key() + if not key or key == tried: + fetched = await key_pickup.fetch_api_key() + if fetched and fetched != tried: + credentials.save_api_key(fetched) + key = fetched + if key and key != tried: + logger.info("MQTT credential recovery: new API key in hand — reconnecting with it.") + await self._rebuild_client() + return + if self._connected: + return + if key and key == tried: + logger.error( + "MQTT credential recovery: C2 returns the same key the broker refuses. The broker's login for " + "this node is out of sync with C2 (dynsec) — an admin re-approve re-syncs it without rotating." + ) + logger.info(f"MQTT credential recovery: retrying in {delay}s.") + await asyncio.sleep(delay) + delay = min(delay * 2, RECOVERY_MAX_DELAY) + + async def _rebuild_client(self) -> None: + """Swap the paho client for one built from the current credentials.""" + old = self._client + if self._connect_task and not self._connect_task.done(): + self._connect_task.cancel() + if old is not None: + # loop_stop() joins paho's thread, which can be mid-backoff — off the event loop. + await asyncio.to_thread(old.loop_stop) + try: + old.disconnect() + except Exception: + pass + self._connected = False + self._client = self._build_client() + self._connect_task = asyncio.create_task(self._connect_with_retry()) + + def update_api_key(self, key: str) -> None: + """A key arrived while connected (legacy MQTT push / reissue): use it on + the next reconnect without dropping the current, working session.""" + if self._client is not None and key != self._client_key: + self._client.username_pw_set(settings.node_id, key) + self._client_key = key async def _connect_with_retry(self): """Attempt the initial TCP connect, retrying with backoff until it succeeds.""" @@ -161,6 +251,8 @@ class MQTTManager: delay = min(delay * 2, 60) async def disconnect(self): + if self._recovery_task: + self._recovery_task.cancel() if self._connect_task: self._connect_task.cancel() if self._client: diff --git a/drb-edge-node/app/internal/update_request.py b/drb-edge-node/app/internal/update_request.py new file mode 100644 index 0000000..7376b22 --- /dev/null +++ b/drb-edge-node/app/internal/update_request.py @@ -0,0 +1,41 @@ +""" +Hand an MQTT `node_update` command to the host-level updater (node-26#15). + +The edge-node container has no business updating its own host: it writes a +sentinel file into /configs (bind-mounted from /configs), and the +host's systemd drb-update.path unit — installed by install.sh — triggers +drb-update.service, which runs Client/scripts/drb-update.sh. The host treats +the file as a bare "please update" flag and never reads its content as input, +so nothing C2 sends can pick the ref, image or command that runs. +""" +import json +import os +from datetime import datetime, timezone +from pathlib import Path + +from app.config import settings +from app.internal.logger import logger + +SENTINEL_NAME = "update-requested" + + +def sentinel_path() -> Path: + return Path(settings.config_path) / SENTINEL_NAME + + +def request_host_update() -> bool: + """Drop the sentinel. Returns False if it could not be written.""" + path = sentinel_path() + try: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(SENTINEL_NAME + ".tmp") + tmp.write_text(json.dumps({"requested_at": datetime.now(timezone.utc).isoformat()})) + os.replace(tmp, path) + except OSError as e: + logger.error(f"Node update requested but the sentinel could not be written ({e}).") + return False + logger.info( + f"Node update requested — handed to the host updater via {path}. If nothing happens within a minute, " + "this node predates the updater: re-run install.sh on it once." + ) + return True diff --git a/drb-edge-node/app/main.py b/drb-edge-node/app/main.py index eadb50e..e4457bb 100644 --- a/drb-edge-node/app/main.py +++ b/drb-edge-node/app/main.py @@ -170,17 +170,15 @@ async def on_command(payload: dict): await asyncio.sleep(2) await op25_client.start() elif action == "node_update": - # TODO: Full OTA update — register a host-level systemd service (e.g. drb-update.service) - # that stops all DRB containers, runs `docker compose pull`, then `docker compose up -d`. - # The C2 server triggers it by sending this MQTT command; the host service watches for the - # restart signal (e.g. via a Unix socket, a sentinel file, or a lightweight webhook). - # Not implemented yet — for now, just restart the container so any pre-pulled image - # is picked up (requires a prior `docker compose pull` on the host). - logger.info("Node update requested — restarting container to pick up latest image.") - await mqtt_manager.publish_status("offline") - await asyncio.sleep(1) - import os - os._exit(0) # Docker restart=unless-stopped will bring the container back up + # node-26#15: the container cannot (and must not) update itself. It + # drops a sentinel in /configs; the host's drb-update.path unit + # (installed by install.sh) sees it and runs drb-update.service, which + # pulls the node's own pinned repo ref + images and recreates the + # stack. The sentinel's content is ignored by the host — C2 cannot + # choose what gets installed. Offline/locked-out nodes get the same + # request over HTTPS via the updater's timer instead. + from app.internal.update_request import request_host_update + request_host_update() else: logger.warning(f"Unknown command: {action}") @@ -189,6 +187,7 @@ async def on_api_key(payload: dict): key = payload.get("api_key") if key: credentials.save_api_key(key) + mqtt_manager.update_api_key(key) logger.info("Node API key received and saved.") diff --git a/drb-edge-node/tests/test_credential_recovery.py b/drb-edge-node/tests/test_credential_recovery.py new file mode 100644 index 0000000..e717b39 --- /dev/null +++ b/drb-edge-node/tests/test_credential_recovery.py @@ -0,0 +1,205 @@ +""" +node-26#14 — a node with no key, or a stale key, recovers on its own: +re-read credentials.json, else fetch over HTTPS with the pickup secret, then +rebuild the MQTT client in place. node-26#15 — node_update hands off to the +host updater via a sentinel file. +""" +import asyncio +import json +from unittest.mock import AsyncMock + +import httpx +import pytest + +from app.config import settings +from app.internal import credentials, key_pickup, mqtt_manager as mm_mod, update_request +from app.internal.mqtt_manager import MQTTManager, _is_auth_failure + + +@pytest.fixture(autouse=True) +def isolated(tmp_path, monkeypatch): + monkeypatch.setattr(credentials, "_CREDS_FILE", tmp_path / "credentials.json") + monkeypatch.setattr(credentials, "_PICKUP_FILE", tmp_path / "pickup_secret") + monkeypatch.setattr(credentials, "_api_key", None) + monkeypatch.setattr(settings, "config_path", str(tmp_path)) + monkeypatch.setattr(settings, "c2_url", "https://c2.example") + monkeypatch.setattr(settings, "mqtt_user", None) + monkeypatch.setattr(mm_mod, "RECOVERY_MIN_DELAY", 0) + yield tmp_path + + +def _manager(client_key=None): + m = MQTTManager() + m._client_key = client_key + m._rebuild_client = AsyncMock() + return m + + +# --- credentials.py ----------------------------------------------------------- + +def test_reload_picks_up_a_key_written_after_startup(isolated): + credentials.load() + assert credentials.get_api_key() is None + data = json.loads((isolated / "credentials.json").read_text()) + data["api_key"] = "written-by-install-sh" + (isolated / "credentials.json").write_text(json.dumps(data)) + assert credentials.reload_api_key() == "written-by-install-sh" + + +def test_write_is_atomic_and_private(isolated): + credentials.save_api_key("k1") + f = isolated / "credentials.json" + assert json.loads(f.read_text())["api_key"] == "k1" + assert (f.stat().st_mode & 0o777) == 0o600 + assert not (isolated / "credentials.json.tmp").exists() + + +def test_pickup_secret_read_and_absent(isolated): + assert credentials.get_pickup_secret() is None + (isolated / "pickup_secret").write_text("abc123\n") + assert credentials.get_pickup_secret() == "abc123" + + +# --- auth-failure classification ------------------------------------------------- + +@pytest.mark.parametrize("code", [4, 5, 134, 135]) +def test_auth_failure_codes(code): + assert _is_auth_failure(code) + + +def test_non_auth_failures_do_not_trigger_recovery(): + assert not _is_auth_failure(3) # server unavailable + assert not _is_auth_failure(136) + + +def test_paho_reason_code_objects_classified(): + from paho.mqtt.reasoncodes import ReasonCode + from paho.mqtt.packettypes import PacketTypes + assert _is_auth_failure(ReasonCode(PacketTypes.CONNACK, "Not authorized")) + assert _is_auth_failure(ReasonCode(PacketTypes.CONNACK, "Bad user name or password")) + assert not _is_auth_failure(ReasonCode(PacketTypes.CONNACK, "Server unavailable")) + + +# --- recovery loop ---------------------------------------------------------------- + +async def test_recovery_uses_key_already_on_disk_without_calling_c2(isolated, monkeypatch): + credentials.save_api_key("disk-key") + monkeypatch.setattr(credentials, "_api_key", None) # process started before the write + fetch = AsyncMock() + monkeypatch.setattr(key_pickup, "fetch_api_key", fetch) + m = _manager(client_key=None) + await m._recover_credentials() + m._rebuild_client.assert_awaited_once() + fetch.assert_not_awaited() + + +async def test_recovery_fetches_over_https_and_persists(isolated, monkeypatch): + monkeypatch.setattr(key_pickup, "fetch_api_key", AsyncMock(return_value="fresh-key")) + m = _manager(client_key="stale-key") + monkeypatch.setattr(credentials, "_api_key", "stale-key") + await m._recover_credentials() + m._rebuild_client.assert_awaited_once() + assert credentials.get_api_key() == "fresh-key" + assert json.loads((isolated / "credentials.json").read_text())["api_key"] == "fresh-key" + + +async def test_recovery_backs_off_until_approved(isolated, monkeypatch): + fetch = AsyncMock(side_effect=[None, None, "approved-key"]) + monkeypatch.setattr(key_pickup, "fetch_api_key", fetch) + m = _manager() + await asyncio.wait_for(m._recover_credentials(), timeout=5) + assert fetch.await_count == 3 + m._rebuild_client.assert_awaited_once() + + +async def test_recovery_stops_if_connection_recovers_meanwhile(isolated, monkeypatch): + monkeypatch.setattr(key_pickup, "fetch_api_key", AsyncMock(return_value=None)) + m = _manager() + m._connected = True + await asyncio.wait_for(m._recover_credentials(), timeout=5) + m._rebuild_client.assert_not_awaited() + + +async def test_start_recovery_is_single_flight(isolated, monkeypatch): + gate = asyncio.Event() + + async def slow_fetch(): + await gate.wait() + return "k" + monkeypatch.setattr(key_pickup, "fetch_api_key", slow_fetch) + m = _manager() + m._start_recovery("a") + first = m._recovery_task + m._start_recovery("b") + assert m._recovery_task is first + gate.set() + await first + + +async def test_connect_without_key_starts_recovery(isolated, monkeypatch): + m = MQTTManager() + started = [] + monkeypatch.setattr(m, "_connect_with_retry", AsyncMock()) + monkeypatch.setattr(m, "_start_recovery", lambda why: started.append(why)) + await m.connect() + assert started == ["no API key at startup"] + + +def test_update_api_key_changes_creds_for_next_reconnect(isolated): + m = MQTTManager() + m._client = m._build_client() + m.update_api_key("rotated") + assert m._client._password == b"rotated" + assert m._client_key == "rotated" + + +# --- key_pickup.fetch_api_key ----------------------------------------------------- + +def _mock_http(monkeypatch, handler): + real = httpx.AsyncClient + + def factory(*a, **kw): + kw["transport"] = httpx.MockTransport(handler) + return real(*a, **kw) + monkeypatch.setattr(key_pickup.httpx, "AsyncClient", factory) + + +async def test_fetch_sends_pickup_secret_and_returns_key(isolated, monkeypatch): + (isolated / "pickup_secret").write_text("the-secret") + seen = {} + + def handler(request): + seen["url"] = str(request.url) + seen["secret"] = request.headers.get("X-Pickup-Secret") + return httpx.Response(200, json={"approval_status": "approved", "api_key": "K"}) + _mock_http(monkeypatch, handler) + assert await key_pickup.fetch_api_key() == "K" + assert seen == {"url": f"https://c2.example/nodes/{settings.node_id}/credentials", "secret": "the-secret"} + + +@pytest.mark.parametrize("resp", [ + httpx.Response(200, json={"approval_status": "pending", "api_key": None}), + httpx.Response(401, json={"detail": "Invalid pickup secret"}), + httpx.Response(404), + httpx.Response(500), + httpx.Response(200, text="not json"), +]) +async def test_fetch_returns_none_when_no_key(isolated, monkeypatch, resp): + (isolated / "pickup_secret").write_text("s") + _mock_http(monkeypatch, lambda request: resp) + assert await key_pickup.fetch_api_key() is None + + +async def test_fetch_without_pickup_secret_does_not_call_c2(isolated, monkeypatch): + def handler(request): + raise AssertionError("must not call C2 without a pickup secret") + _mock_http(monkeypatch, handler) + assert await key_pickup.fetch_api_key() is None + + +# --- node_update -> host sentinel --------------------------------------------------- + +def test_node_update_writes_sentinel(isolated): + assert update_request.request_host_update() is True + data = json.loads((isolated / "update-requested").read_text()) + assert "requested_at" in data diff --git a/drb-edge-node/tests/test_host_updater.py b/drb-edge-node/tests/test_host_updater.py new file mode 100644 index 0000000..c2006b2 --- /dev/null +++ b/drb-edge-node/tests/test_host_updater.py @@ -0,0 +1,179 @@ +""" +node-26#15 — Client/scripts/drb-update.sh, the host-level updater. + +Runs the real script with stub git/docker/curl on PATH, so what is checked is +the script's decision logic and its security guards: it only updates when +asked (MQTT sentinel or a new C2 request), never takes anything but "update +now" from C2, refuses a mismatched git origin or registry, and keeps node +credentials off curl's argv. +""" +import json +import os +import shutil +import stat +import subprocess +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "drb-update.sh" + +pytestmark = pytest.mark.skipif( + not (SCRIPT.exists() and shutil.which("bash") and shutil.which("jq") and shutil.which("flock")), + reason="needs the repo's scripts/ (absent inside the image) plus bash, jq and flock", +) + +REPO = "https://git.example.net/logan/node-26.git" +OLD, NEW = "a" * 40, "b" * 40 + +STUB_GIT = r'''#!/usr/bin/env bash +echo "git $*" >> "$STUB/calls" +while [ "$1" = -c ]; do shift 2; done +case "$1" in + rev-parse) if [ "$2" = HEAD ]; then cat "$STUB/head"; else echo "$STUB_NEW"; fi ;; + remote) echo "$STUB_ORIGIN" ;; + fetch) exit 0 ;; + show-ref) [ "$4" = "refs/tags/v1" ] ;; + checkout) echo "${@: -1}" > "$STUB/head" ;; + *) exit 0 ;; +esac +''' +STUB_DOCKER = '#!/usr/bin/env bash\necho "docker $*" >> "$STUB/calls"\n' +STUB_CURL = r'''#!/usr/bin/env bash +echo "curl $*" >> "$STUB/calls" +for a in "$@"; do case "$a" in @*) cat "${a#@}" >> "$STUB/headers" ;; esac; done +cat "$STUB/response" +''' + + +def _exe(path: Path, body: str): + path.write_text(body) + path.chmod(path.stat().st_mode | stat.S_IEXEC) + + +@pytest.fixture +def env(tmp_path): + stub = tmp_path / "stub" + bindir = stub / "bin" + bindir.mkdir(parents=True) + _exe(bindir / "git", STUB_GIT) + _exe(bindir / "docker", STUB_DOCKER) + _exe(bindir / "curl", STUB_CURL) + (stub / "head").write_text(OLD + "\n") + (stub / "calls").write_text("") + (stub / "response").write_text(json.dumps({"update_requested_at": None})) + + inst = tmp_path / "node" + (inst / ".git").mkdir(parents=True) + (inst / "configs").mkdir() + (inst / ".env").write_text( + "NODE_ID=node-7\nC2_URL=https://api.example.net\nIMAGE_REGISTRY=git.example.net\n" + ) + (inst / "configs" / "credentials.json").write_text(json.dumps({"api_key": "k" * 64})) + (inst / "configs" / "pickup_secret").write_text("p" * 48) + + conf = tmp_path / "updater.conf" + conf.write_text(f"INSTALL_DIR={inst}\nREPO_URL={REPO}\nREF=v1\nREGISTRY=git.example.net\nBUILD=0\n") + + e = dict(os.environ) + e.update({ + "PATH": f"{bindir}:{os.environ['PATH']}", + "STUB": str(stub), "STUB_NEW": NEW, "STUB_ORIGIN": REPO, + "DRB_UPDATER_CONF": str(conf), + "DRB_UPDATER_STATE": str(tmp_path / "state"), + "DRB_UPDATER_LOCK": str(tmp_path / "lock"), + }) + return {"env": e, "stub": stub, "inst": inst, "conf": conf, "state": tmp_path / "state"} + + +def _run(env): + return subprocess.run(["bash", str(SCRIPT)], env=env["env"], capture_output=True, text=True, timeout=30) + + +def _calls(env): + return (env["stub"] / "calls").read_text() + + +def test_no_request_means_checkin_only(env): + r = _run(env) + assert r.returncode == 0, r.stderr + calls = _calls(env) + assert "update-check" in calls + assert "docker" not in calls + assert "fetch" not in calls + + +def test_c2_request_updates_once_and_reports(env): + (env["stub"] / "response").write_text(json.dumps({"update_requested_at": "2026-10-04T12:00:00+00:00"})) + r = _run(env) + assert r.returncode == 0, r.stderr + r.stdout + calls = _calls(env) + assert "docker compose pull" in calls + assert "docker compose up -d --no-build --force-recreate --remove-orphans" in calls + assert (env["stub"] / "head").read_text().strip() == NEW + assert '"applied_request_at":"2026-10-04T12:00:00+00:00"' in calls + assert '"result":"ok"' in calls + # Same request again: nothing to do. + (env["stub"] / "calls").write_text("") + assert _run(env).returncode == 0 + assert "docker" not in _calls(env) + + +def test_mqtt_sentinel_triggers_update_and_is_consumed(env): + sentinel = env["inst"] / "configs" / "update-requested" + sentinel.write_text('{"ref": "evil", "cmd": "rm -rf /"}') + r = _run(env) + assert r.returncode == 0, r.stderr + assert not sentinel.exists() + calls = _calls(env) + assert "docker compose pull" in calls + assert "evil" not in calls and "rm -rf" not in calls + + +def test_c2_response_is_never_used_as_a_ref_or_command(env): + (env["stub"] / "response").write_text(json.dumps({"update_requested_at": "$(touch /tmp/pwned); main"})) + r = _run(env) + assert r.returncode == 0 + assert "docker" not in _calls(env) # malformed request is ignored, not executed + + +def test_refuses_when_git_origin_differs(env): + env["env"]["STUB_ORIGIN"] = "https://attacker.example/node-26.git" + (env["inst"] / "configs" / "update-requested").write_text("") + r = _run(env) + assert r.returncode != 0 + assert "refusing" in r.stderr + assert "docker" not in _calls(env) + + +def test_refuses_when_env_registry_differs(env): + (env["inst"] / ".env").write_text( + "NODE_ID=node-7\nC2_URL=https://api.example.net\nIMAGE_REGISTRY=evil.example\n" + ) + (env["inst"] / "configs" / "update-requested").write_text("") + r = _run(env) + assert r.returncode != 0 + assert "docker compose pull" not in _calls(env) + + +def test_non_https_repo_in_conf_is_rejected(env): + env["conf"].write_text(env["conf"].read_text().replace("https://", "http://")) + r = _run(env) + assert r.returncode != 0 + assert "git" not in _calls(env) + + +def test_credentials_go_in_a_header_file_never_argv(env): + _run(env) + calls = _calls(env) + assert "k" * 64 not in calls and "p" * 48 not in calls + headers = (env["stub"] / "headers").read_text() + assert f"Authorization: Bearer {'k' * 64}" in headers + assert f"X-Pickup-Secret: {'p' * 48}" in headers + + +def test_plain_http_c2_skips_checkin(env): + (env["inst"] / ".env").write_text("NODE_ID=node-7\nC2_URL=http://api.example.net\nIMAGE_REGISTRY=git.example.net\n") + r = _run(env) + assert r.returncode == 0 + assert "curl" not in _calls(env) diff --git a/install.sh b/install.sh index e6bd0c7..c82aafa 100644 --- a/install.sh +++ b/install.sh @@ -17,9 +17,13 @@ # 2. Install Docker + compose plugin + git + curl + jq # 3. Clone logan/node-26 at a PINNED ref into $INSTALL_DIR # 4. Write .env (non-interactive from flags/env, interactive fallback) -# 5. Enroll with C2 (POST /nodes/enroll) and poll for the api_key -# 6. docker compose pull && up -d (prebuilt; --build opts into the ~1h build) -# 7. Print the approval step +# 5. Enroll with C2 (POST /nodes/enroll), then BLOCK until an admin approves +# and the api_key is written (node-26#14). Ctrl-C safe: re-run resumes +# from configs/pickup_secret. A re-run also refreshes a stale key. +# 6. docker compose pull && up -d, ALWAYS recreating edge-node so it loads +# the key on disk (prebuilt; --build opts into the ~1h build) +# 7. Install the host updater (systemd timer + path unit, node-26#15) +# 8. Print a summary # # It does NOT set up WireGuard. WireGuard-per-node was evaluated and rejected # (Server/MQTT-PUBLIC-AUTH-PLAN.md, Server/infra/main.tf:67-73). A field node @@ -85,7 +89,14 @@ REGISTRY_PASS="${DRB_REGISTRY_PASS:-}" DO_BUILD=0 # 0 = pull prebuilt images (default), 1 = build on the Pi (~1h for op25) DO_START=1 ASSUME_YES=0 -ENROLL_WAIT="${DRB_ENROLL_WAIT:-0}" # seconds to block waiting for admin approval; 0 = don't block +# How long to block waiting for admin approval. Default: forever (node-26#14 — +# the old default of 0 started a keyless stack that never came online). +# N>0 = give up after N seconds and start anyway; 0 = don't wait at all. Either +# way the edge-node app keeps fetching the key itself and comes online on its +# own once approved. +ENROLL_WAIT="${DRB_ENROLL_WAIT:-forever}" +APP_URL="${DRB_APP_URL:-}" +INSTALL_UPDATER=1 C='\033[0;36m'; G='\033[0;32m'; Y='\033[1;33m'; R='\033[0;31m'; N='\033[0m' say() { printf "${C}==>${N} %s\n" "$*"; } @@ -111,7 +122,12 @@ Usage: install.sh [options] --dir PATH install location (default /opt/drb/node-26) --build build images locally instead of pulling (~1h for op25) --no-start configure and enroll, but do not start containers - --wait-approval SEC block up to SEC seconds polling for admin approval + --wait-approval SEC stop waiting for admin approval after SEC seconds + (default: wait forever; 0 = don't wait). The node + still fetches its key by itself once approved. + --app-url URL admin UI base for the approval message + (default: C2 URL without its "api." prefix) + --no-updater don't install the host updater (no remote updates) -y, --yes never prompt; fail instead of asking Every option also has an env var: DRB_NODE_ID, DRB_C2_URL, DRB_ENROLLMENT_TOKEN, @@ -138,12 +154,17 @@ while [ $# -gt 0 ]; do --build) DO_BUILD=1; shift ;; --no-start) DO_START=0; shift ;; --wait-approval) ENROLL_WAIT="$2"; shift 2 ;; + --app-url) APP_URL="$2"; shift 2 ;; + --no-updater) INSTALL_UPDATER=0; shift ;; -y|--yes) ASSUME_YES=1; shift ;; -h|--help) usage; exit 0 ;; *) die "unknown option: $1 (try --help)" ;; esac done +[ "$ENROLL_WAIT" = forever ] || [[ "$ENROLL_WAIT" =~ ^[0-9]+$ ]] \ + || die "--wait-approval takes a number of seconds (or omit it to wait until approved)" + # Prompts must come from the terminal, not from the `curl |` pipe on stdin. ask() { # ask VAR "prompt" "default" local __v="$1" __p="$2" __d="${3:-}" __r="" @@ -226,7 +247,9 @@ mkdir -p "$(dirname "$INSTALL_DIR")" if [ -d "$INSTALL_DIR/.git" ]; then ok "existing install at $INSTALL_DIR — updating in place (.env is preserved)" git -C "$INSTALL_DIR" remote set-url origin "$REPO_URL" - git -C "$INSTALL_DIR" fetch --tags --prune origin + # --force: a moved release tag (v1 re-cut) must replace the local copy, + # otherwise fetch refuses with "would clobber existing tag" and set -e dies. + git -C "$INSTALL_DIR" fetch --force --tags --prune origin else git clone --no-checkout "$REPO_URL" "$INSTALL_DIR" fi @@ -345,39 +368,68 @@ PICKUP_FILE="$INSTALL_DIR/configs/pickup_secret" # GET /nodes/{id}/credentials. Sets CRED_HTTP + CRED_BODY (no -f: we need the # body and status on a 4xx). One implementation so first-run and re-run agree. creds_pickup() { # creds_pickup PICKUP_SECRET - local _tmp; _tmp="$(mktemp)" + local _tmp _hdr; _tmp="$(mktemp)"; _hdr="$(mktemp)"; chmod 600 "$_hdr" + # Secret via a 0600 header file, never curl's argv (visible in `ps`). + printf 'X-Pickup-Secret: %s\n' "$1" > "$_hdr" CRED_HTTP="$(curl -sS -o "$_tmp" -w '%{http_code}' \ - "$C2_URL/nodes/$NODE_ID/credentials" -H "X-Pickup-Secret: $1" 2>/dev/null || echo 000)" + "$C2_URL/nodes/$NODE_ID/credentials" -H "@$_hdr" 2>/dev/null || echo 000)" CRED_BODY="$(cat "$_tmp" 2>/dev/null || true)" - rm -f "$_tmp" + rm -f "$_tmp" "$_hdr" } cred_field() { printf '%s' "${CRED_BODY:-}" | jq -r "$1 // empty" 2>/dev/null || true; } +KEY_WRITTEN=0 +# Merge into the existing file: the edge-node app keeps its dashboard auth +# salt + session secret in the same JSON, and clobbering them logs everyone +# out. Atomic (tmp + mv) so a Ctrl-C can never leave half a file. write_creds() { # write_creds API_KEY - umask 077; jq -n --arg k "$1" '{api_key:$k}' > "$CREDS"; umask 022 - ok "api_key received and written to configs/credentials.json" + local _tmp="$CREDS.tmp" + umask 077 + if [ -s "$CREDS" ] && jq -e . "$CREDS" >/dev/null 2>&1; then + jq --arg k "$1" '.api_key = $k' "$CREDS" > "$_tmp" + else + jq -n --arg k "$1" '{api_key:$k}' > "$_tmp" + fi + umask 022 + mv -f "$_tmp" "$CREDS" + KEY_WRITTEN=1 + ok "api_key written to configs/credentials.json" } +[ -n "$APP_URL" ] || { APP_URL="${C2_URL/:\/\/api./://}"; } +APP_URL="${APP_URL%/}" + say_pending() { # say_pending APPROVAL_STATUS — not an error: node is enrolled, key not minted yet warn "not approved yet — C2 reports approval_status=${1:-pending}, no api_key minted." - warn "an admin must, at /settings/nodes : Approve '$NODE_ID' (assigning a system is separate)." - warn "then re-run this installer — it reuses configs/pickup_secret — or fetch it directly:" - warn " curl -fsS $C2_URL/nodes/$NODE_ID/credentials -H \"X-Pickup-Secret: \$(cat $PICKUP_FILE)\" | jq -r .api_key" + warn "an admin must approve '$NODE_ID' at ${APP_URL}/settings/nodes. The node keeps fetching" + warn "its key by itself and comes online once approved; re-running this installer also works." } -# Poll creds_pickup for up to ENROLL_WAIT seconds while still pending. Result -# left in CRED_HTTP/CRED_BODY. --wait-approval is what would have avoided the -# original prod bug; the default (0) does not block, so the re-run path below -# must stand on its own. +# Block until the node is approved and C2 hands back an api_key (node-26#14). +# Result left in CRED_HTTP/CRED_BODY. Returns 0 with a key, 1 on timeout/0-wait, +# dies on a credential error (401/404) that waiting cannot fix. wait_for_key() { # wait_for_key PICKUP_SECRET - [ "${ENROLL_WAIT:-0}" -gt 0 ] || return 0 - local _end; _end=$(( $(date +%s) + ENROLL_WAIT )) - say "Waiting up to ${ENROLL_WAIT}s for an admin to approve '$NODE_ID'" - while [ "$(date +%s)" -lt "$_end" ]; do - sleep 10 + [ "$ENROLL_WAIT" = 0 ] && return 1 + local _end=0 _shown=0 _delay=5 + [ "$ENROLL_WAIT" = forever ] || _end=$(( $(date +%s) + ENROLL_WAIT )) + trap 'printf "\n"; warn "interrupted — nothing is lost: re-run the same command to resume waiting (configs/pickup_secret is saved)."; exit 130' INT TERM + while :; do + case "$CRED_HTTP" in + 200) [ -n "$(cred_field '.api_key')" ] && { trap - INT TERM; return 0; } ;; + 401) die "C2 rejected the saved pickup secret (401) while waiting — re-run with --token DRB-… to re-enroll." ;; + 404) die "C2 no longer knows node '$NODE_ID' (404) — it was deleted. Re-run with --token DRB-… to enroll again." ;; + *) : ;; # 000/5xx: transient, keep waiting + esac + if [ "$_shown" = 0 ]; then + printf "\n${Y}Please have an admin approve node '%s' at %s/settings/nodes to continue.${N}\n" \ + "$NODE_ID" "$APP_URL" + printf "Waiting for approval%s… (Ctrl-C is safe — re-run the same command to resume)\n\n" \ + "$([ "$_end" = 0 ] && echo "" || echo " for up to ${ENROLL_WAIT}s")" + _shown=1 + fi + if [ "$_end" != 0 ] && [ "$(date +%s)" -ge "$_end" ]; then trap - INT TERM; return 1; fi + sleep "$_delay"; [ "$_delay" -lt 30 ] && _delay=$(( _delay + 5 )) creds_pickup "$1" - [ "$CRED_HTTP" = 200 ] || return 0 - [ -z "$(cred_field '.api_key')" ] || return 0 done } @@ -390,12 +442,15 @@ do_fresh_enroll() { BODY="$(jq -nc --arg id "$NODE_ID" --arg n "${NODE_NAME:-$NODE_ID}" \ --argjson lat "${NODE_LAT:-0}" --argjson lon "${NODE_LON:-0}" \ '{node_id:$id,name:$n,lat:$lat,lon:$lon}')" - # Token goes in a header from a shell variable — never on a process command - # line, never echoed. + # Token goes in a 0600 header file — never on a process command line (curl's + # argv is visible in `ps`; CISO condition, server-26#61), never echoed. + local _ehdr; _ehdr="$(mktemp)"; chmod 600 "$_ehdr" + printf 'X-Enrollment-Token: %s\n' "$ENROLLMENT_TOKEN" > "$_ehdr" if ! RESP="$(curl -fsS -X POST "$C2_URL/nodes/enroll" \ -H "Content-Type: application/json" \ - -H "X-Enrollment-Token: $ENROLLMENT_TOKEN" \ + -H "@$_ehdr" \ --data "$BODY" 2>&1)"; then + rm -f "$_ehdr" case "$RESP" in *403*) die "enroll refused (403): node '$NODE_ID' is already approved on C2, and this machine has no configs/pickup_secret to collect its key with. A token @@ -409,6 +464,7 @@ do_fresh_enroll() { *) die "enroll failed: $RESP" ;; esac fi + rm -f "$_ehdr" PICKUP="$(printf '%s' "$RESP" | jq -r '.pickup_secret')" STATUS="$(printf '%s' "$RESP" | jq -r '.approval_status')" [ -n "$PICKUP" ] && [ "$PICKUP" != null ] || die "enroll returned no pickup_secret: $RESP" @@ -418,21 +474,32 @@ do_fresh_enroll() { creds_pickup "$PICKUP" [ "$CRED_HTTP" = 200 ] || die "post-enroll credential fetch failed (HTTP $CRED_HTTP): ${CRED_BODY:-}" - KEY="$(cred_field '.api_key')" - if [ -z "$KEY" ]; then - wait_for_key "$PICKUP" || true - KEY="$(cred_field '.api_key')" - fi - if [ -n "$KEY" ]; then - write_creds "$KEY" + if wait_for_key "$PICKUP"; then + write_creds "$(cred_field '.api_key')" else say_pending "$(cred_field '.approval_status')" fi } -if [ -s "$CREDS" ] && jq -e '.api_key // empty' "$CREDS" >/dev/null 2>&1; then +DISK_KEY="$(jq -r '.api_key // empty' "$CREDS" 2>/dev/null || true)" +if [ -n "$DISK_KEY" ] && [ -s "$PICKUP_FILE" ] && [ -n "${C2_URL:-}" ]; then + # Key on disk, but it may be STALE (an admin Reissue, or the old approve + # that rotated on every click). Ask C2 for the current one and replace it if + # different. Best-effort: on any error keep the key we have. + say "Enrollment — key on disk; checking it is still current" + creds_pickup "$(cat "$PICKUP_FILE")" + SERVER_KEY="$(cred_field '.api_key')" + if [ "$CRED_HTTP" = 200 ] && [ -n "$SERVER_KEY" ] && [ "$SERVER_KEY" != "$DISK_KEY" ]; then + warn "the key on disk is not the one C2 has on file (reissued?) — replacing it" + write_creds "$SERVER_KEY" + elif [ "$CRED_HTTP" = 200 ] && [ -n "$SERVER_KEY" ]; then + ok "api_key on disk matches C2" + else + warn "could not confirm the key with C2 (HTTP $CRED_HTTP) — keeping the key on disk" + fi +elif [ -n "$DISK_KEY" ]; then say "Enrollment" - ok "api_key already on disk — skipping enrollment" + ok "api_key already on disk (no pickup secret to re-check it with) — skipping enrollment" elif [ -z "${C2_URL:-}" ]; then say "Enrollment" warn "no C2_URL — skipping enrollment" @@ -446,16 +513,11 @@ elif [ -s "$PICKUP_FILE" ]; then creds_pickup "$PICKUP" case "$CRED_HTTP" in 200) - API_KEY="$(cred_field '.api_key')" - if [ -z "$API_KEY" ]; then - wait_for_key "$PICKUP" || true - API_KEY="$(cred_field '.api_key')" - fi - if [ -n "$API_KEY" ]; then - write_creds "$API_KEY" + if wait_for_key "$PICKUP"; then + write_creds "$(cred_field '.api_key')" else - # Still pending. Enrolled and idempotent — next run collects the key. - # Clean exit, fall through to start the stack. NOT a failure. + # Only with --wait-approval N/0. Enrolled and idempotent; the app + # fetches the key itself once approved. NOT a failure. say_pending "$(cred_field '.approval_status')" fi ;; @@ -510,12 +572,30 @@ if [ "$DO_START" = 1 ]; then fi docker compose up --no-build -d fi - ok "stack started" + # ALWAYS recreate edge-node (node-26#14): `up -d` leaves an already-running + # container alone, and a process started before the key was written would + # otherwise keep running keyless. Cheap, and makes every re-run a fix. + docker compose up -d --no-build --force-recreate --no-deps edge-node + ok "stack started (edge-node recreated so it loads configs/credentials.json)" else say "Skipping start (--no-start). Run: cd $INSTALL_DIR && make up-prebuilt" fi -# ── 7. What the operator does next ────────────────────────────────────────── +# ── 7. Host updater (node-26#15) ─────────────────────────────────────────── +UPDATER_STATE="not installed (--no-updater)" +if [ "$INSTALL_UPDATER" = 1 ]; then + say "Installing the host updater" + UPD_ARGS=(--dir "$INSTALL_DIR" --repo-url "$REPO_URL" --ref "$REF" --registry "$REGISTRY") + [ "$DO_BUILD" = 1 ] && UPD_ARGS+=(--build) + if bash "$INSTALL_DIR/scripts/install-updater.sh" "${UPD_ARGS[@]}"; then + UPDATER_STATE="enabled — checks C2 every ~15 min; 'Update Node Software' in the admin UI" + else + warn "host updater NOT installed — this node cannot be updated remotely" + UPDATER_STATE="FAILED to install (see above)" + fi +fi + +# ── 8. Summary ────────────────────────────────────────────────────────────── IP="$(hostname -I 2>/dev/null | awk '{print $1}')" cat <}/ (user: ${DASHBOARD_USER}) logs cd $INSTALL_DIR && docker compose logs -f edge-node + updater ${UPDATER_STATE} + api key $([ -n "$(jq -r '.api_key // empty' "$CREDS" 2>/dev/null)" ] && echo "present" || echo "NOT YET — node fetches it once approved") -NEXT — an admin must approve this node before it can do anything: - - 1. Open /settings/nodes - 2. Approve "$NODE_ID" - 3. Assign it a radio system +NEXT — assign this node a radio system at ${APP_URL}/nodes/${NODE_ID} EOF if [ "${GENERATED_DASH:-0}" = 1 ]; then printf "${Y}Generated dashboard password (shown once): %s${N}\n\n" "$DASHBOARD_PASS" fi -if [ ! -s "$CREDS" ]; then +if [ -z "$(jq -r '.api_key // empty' "$CREDS" 2>/dev/null)" ]; then if [ -s "$PICKUP_FILE" ]; then - printf "${Y}This node has no api_key yet. After an admin approves it, re-run the same\ninstall command — it reuses configs/pickup_secret and will collect the key\n(no --token needed for the re-run).${N}\n\n" + printf "${Y}This node has no api_key yet. Once an admin approves it at %s/settings/nodes\nthe node fetches the key and comes online by itself (no re-run needed).${N}\n\n" "$APP_URL" else printf "${Y}This node has no api_key and no saved pickup secret, so a plain re-run cannot\nfix it. Re-run with --token DRB-… to enroll; or, if the node is already\napproved, have an admin Reissue key and write it into\n%s by hand.${N}\n\n" "$CREDS" fi diff --git a/scripts/drb-update.sh b/scripts/drb-update.sh new file mode 100644 index 0000000..02bb58b --- /dev/null +++ b/scripts/drb-update.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# +# DRB node host updater (node-26#15). Runs as root from systemd: +# drb-update.timer -> every ~15 min: HTTPS check-in with C2; update if asked +# drb-update.path -> fires when the edge-node container drops +# /configs/update-requested (MQTT node_update) +# Both start drb-update.service, which runs this script. install-updater.sh +# writes the units and /etc/drb/updater.conf; install.sh calls it. +# +# SECURITY MODEL — this is a remote-update path onto devices in the field: +# * What gets installed comes ONLY from /etc/drb/updater.conf (root-owned, +# written by install.sh): the repo URL, the ref, the image registry. C2's +# answer is a single timestamp meaning "update now"; it is never used as a +# ref, URL, path or command. The MQTT sentinel's content is never read. +# * git origin must equal the configured REPO_URL (https only) and the .env +# IMAGE_REGISTRY must equal the configured REGISTRY, or nothing runs. +# * Node credentials go to curl via a 0600 header file, never argv. +# * C2 must be https. +# What this does NOT defend against: a compromised git host or registry — +# refs and images are not signed (see the issue filed with node-26#15). +# +# The whole body is inside main(), called on the last line: bash parses the +# function completely before running it, so `git checkout` replacing this +# very file mid-run cannot change what is executing. +set -euo pipefail + +main() { + local CONF="${DRB_UPDATER_CONF:-/etc/drb/updater.conf}" + local STATE_DIR="${DRB_UPDATER_STATE:-/var/lib/drb-update}" + local LOCK="${DRB_UPDATER_LOCK:-/run/drb-update.lock}" + + log() { printf 'drb-update: %s\n' "$*"; } + err() { printf 'drb-update: ERROR: %s\n' "$*" >&2; } + conf() { grep -E "^$1=" "$CONF" 2>/dev/null | head -1 | cut -d= -f2- || true; } + + [ -r "$CONF" ] || { err "$CONF missing — re-run install.sh"; return 1; } + local INSTALL_DIR REPO_URL REF REGISTRY BUILD + INSTALL_DIR="$(conf INSTALL_DIR)"; REPO_URL="$(conf REPO_URL)"; REF="$(conf REF)" + REGISTRY="$(conf REGISTRY)"; BUILD="$(conf BUILD)" + + [[ "$INSTALL_DIR" =~ ^/[A-Za-z0-9._/-]+$ ]] && [ -d "$INSTALL_DIR/.git" ] \ + || { err "bad INSTALL_DIR '$INSTALL_DIR' in $CONF"; return 1; } + [[ "$REPO_URL" =~ ^https://[A-Za-z0-9._/:-]+$ ]] || { err "REPO_URL must be https: '$REPO_URL'"; return 1; } + [[ "$REF" =~ ^[A-Za-z0-9._][A-Za-z0-9._/-]*$ ]] || { err "bad REF '$REF'"; return 1; } + [[ "$REGISTRY" =~ ^[A-Za-z0-9.-]+(:[0-9]+)?$ ]] || { err "bad REGISTRY '$REGISTRY'"; return 1; } + + mkdir -p "$STATE_DIR"; chmod 700 "$STATE_DIR" + exec 9>"$LOCK" + flock -n 9 || { log "another update is already running"; return 0; } + + cd "$INSTALL_DIR" + envget() { grep -E "^$1=" .env 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"' || true; } + local NODE_ID C2_URL + NODE_ID="$(envget NODE_ID)"; C2_URL="$(envget C2_URL)"; C2_URL="${C2_URL%/}" + [[ "$NODE_ID" =~ ^[A-Za-z0-9_-]+$ ]] || { err "no valid NODE_ID in $INSTALL_DIR/.env"; return 1; } + + local reason="" + local SENTINEL="$INSTALL_DIR/configs/update-requested" + if [ -e "$SENTINEL" ] || [ -L "$SENTINEL" ]; then + rm -f "$SENTINEL" # content deliberately ignored + reason="mqtt node_update" + fi + + # ── HTTPS check-in ────────────────────────────────────────────────────── + local tmpd; tmpd="$(mktemp -d)"; chmod 700 "$tmpd" + # shellcheck disable=SC2064 + trap "rm -rf '$tmpd'" RETURN + + local sha; sha="$(git rev-parse HEAD 2>/dev/null || echo '')" + local requested="" + + checkin() { # checkin APPLIED_AT RESULT -> stdout: response body + [[ "$C2_URL" == https://* ]] || return 1 + local key="" pickup="" + key="$(jq -r '.api_key // empty' configs/credentials.json 2>/dev/null || true)" + pickup="$(cat configs/pickup_secret 2>/dev/null || true)" + : > "$tmpd/h"; chmod 600 "$tmpd/h" + if [[ "$key" =~ ^[A-Za-z0-9]+$ ]]; then + printf 'Authorization: Bearer %s\n' "$key" >> "$tmpd/h" + fi + if [[ "$pickup" =~ ^[A-Za-z0-9]+$ ]]; then + printf 'X-Pickup-Secret: %s\n' "$pickup" >> "$tmpd/h" + fi + [ -s "$tmpd/h" ] || return 1 + local body + body="$(jq -nc --arg sha "$sha" --arg a "$1" --arg r "$2" \ + '{sha:$sha} + (if $a != "" then {applied_request_at:$a} else {} end) + + (if $r != "" then {result:$r} else {} end)')" + curl -fsS --proto '=https' --max-time 20 -X POST \ + "$C2_URL/nodes/$NODE_ID/update-check" \ + -H "Content-Type: application/json" -H "@$tmpd/h" --data "$body" + } + + local resp + if [[ "$C2_URL" != https://* ]]; then + log "C2_URL is not https — skipping the HTTPS check-in" + elif resp="$(checkin "" "" 2>"$tmpd/err")"; then + requested="$(printf '%s' "$resp" | jq -r '.update_requested_at // empty' 2>/dev/null || true)" + if [ -n "$requested" ] && ! [[ "$requested" =~ ^[0-9T:.+Z-]{10,40}$ ]]; then + log "ignoring malformed update_requested_at from C2" + requested="" + fi + else + log "HTTPS check-in failed: $(tr -d '\n' < "$tmpd/err" | head -c 200)" + fi + + local last=""; last="$(cat "$STATE_DIR/last_request" 2>/dev/null || true)" + local new_request="" + if [ -n "$requested" ] && [ "$requested" != "$last" ]; then + new_request="$requested" + reason="${reason:-C2 request $requested}" + fi + [ -n "$reason" ] || { log "no update requested"; return 0; } + + # ── Update ───────────────────────────────────────────────────────────── + log "updating ($reason) — repo $REPO_URL @ $REF" + local result=ok + if do_update; then + log "update complete: $(git rev-parse --short HEAD)" + else + result=failed + log "update FAILED — will retry on the next timer run" + fi + if [ -n "$new_request" ]; then + [ "$result" = ok ] && printf '%s' "$new_request" > "$STATE_DIR/last_request" + sha="$(git rev-parse HEAD 2>/dev/null || echo '')" + checkin "$new_request" "$result" >/dev/null 2>&1 || log "could not report the result to C2" + fi + [ "$result" = ok ] +} + +do_update() { + local origin; origin="$(git remote get-url origin)" + [ "$origin" = "$REPO_URL" ] || { err "git origin '$origin' != configured '$REPO_URL' — refusing"; return 1; } + + # --force: a moved release tag (v1) must replace the local copy. + git -c protocol.allow=never -c protocol.https.allow=always fetch --force --tags --prune origin || return 1 + + local target + if git show-ref --verify -q "refs/remotes/origin/$REF"; then + target="refs/remotes/origin/$REF" + elif git show-ref --verify -q "refs/tags/$REF"; then + target="refs/tags/$REF" + else + err "ref '$REF' not found on origin"; return 1 + fi + local old new + old="$(git rev-parse HEAD)"; new="$(git rev-parse "$target^{commit}")" + if [ "$old" != "$new" ]; then + git -c advice.detachedHead=false checkout --force --detach "$new" || return 1 + log "checked out $REF: ${old:0:12} -> ${new:0:12}" + # Units/conf may have changed with the new scripts. + bash scripts/install-updater.sh --refresh || log "unit refresh failed (continuing with the old units)" + fi + + local env_registry + env_registry="$(grep -E '^IMAGE_REGISTRY=' .env | head -1 | cut -d= -f2- | tr -d '"' || true)" + [ "${env_registry:-git.vpn.cusano.net}" = "$REGISTRY" ] \ + || { err ".env IMAGE_REGISTRY '$env_registry' != configured '$REGISTRY' — refusing"; return 1; } + + if [ "$BUILD" = 1 ]; then + docker compose build || return 1 + else + docker compose pull || return 1 + fi + # --force-recreate: app code is bind-mounted from the checkout (install.sh + # D3), so a code-only change also needs every container restarted. + docker compose up -d --no-build --force-recreate --remove-orphans || return 1 + docker image prune -f >/dev/null 2>&1 || true +} + +main "$@" diff --git a/scripts/install-updater.sh b/scripts/install-updater.sh new file mode 100644 index 0000000..34e8cf7 --- /dev/null +++ b/scripts/install-updater.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# +# Install (or refresh) the DRB host updater units (node-26#15). Run as root. +# +# install-updater.sh --dir /opt/drb/node-26 --repo-url URL --ref v1 \ +# --registry git.vpn.cusano.net [--build] +# install-updater.sh --refresh # re-render units from /etc/drb/updater.conf +# +# Writes /etc/drb/updater.conf (root:root 0644 — the ONLY source of what the +# updater may install), drb-update.{service,timer,path}, and enables the timer +# and the path watcher. install.sh calls this; drb-update.sh calls --refresh +# after a checkout so unit changes ship with the repo. +set -euo pipefail + +CONF=/etc/drb/updater.conf +UNIT_DIR="${DRB_UNIT_DIR:-/etc/systemd/system}" +conf() { grep -E "^$1=" "$CONF" 2>/dev/null | head -1 | cut -d= -f2- || true; } +die() { printf 'install-updater: %s\n' "$*" >&2; exit 1; } + +INSTALL_DIR="" REPO_URL="" REF="" REGISTRY="" BUILD=0 REFRESH=0 +while [ $# -gt 0 ]; do + case "$1" in + --dir) INSTALL_DIR="$2"; shift 2 ;; + --repo-url) REPO_URL="$2"; shift 2 ;; + --ref) REF="$2"; shift 2 ;; + --registry) REGISTRY="$2"; shift 2 ;; + --build) BUILD=1; shift ;; + --refresh) REFRESH=1; shift ;; + *) die "unknown option: $1" ;; + esac +done + +[ "$(id -u)" -eq 0 ] || die "must run as root" +if [ "$REFRESH" = 1 ]; then + [ -r "$CONF" ] || die "$CONF missing — run install.sh" + INSTALL_DIR="$(conf INSTALL_DIR)"; REPO_URL="$(conf REPO_URL)"; REF="$(conf REF)" + REGISTRY="$(conf REGISTRY)"; BUILD="$(conf BUILD)" +fi + +[[ "$INSTALL_DIR" =~ ^/[A-Za-z0-9._/-]+$ ]] || die "bad --dir '$INSTALL_DIR' (absolute, no spaces)" +[[ "$REPO_URL" =~ ^https://[A-Za-z0-9._/:-]+$ ]] || die "--repo-url must be https: '$REPO_URL'" +[[ "$REF" =~ ^[A-Za-z0-9._][A-Za-z0-9._/-]*$ ]] || die "bad --ref '$REF'" +[[ "$REGISTRY" =~ ^[A-Za-z0-9.-]+(:[0-9]+)?$ ]] || die "bad --registry '$REGISTRY'" +[ "$BUILD" = 1 ] || BUILD=0 + +if ! command -v systemctl >/dev/null 2>&1 || [ ! -d /run/systemd/system ]; then + die "systemd not running — the host updater needs systemd (remote updates disabled on this node)" +fi + +install -d -m 755 /etc/drb +umask 022 +cat > "$CONF.tmp" < "$UNIT_DIR/drb-update.service" < "$UNIT_DIR/drb-update.timer" <<'EOF' +[Unit] +Description=DRB node updater — periodic HTTPS check-in with C2 + +[Timer] +OnBootSec=3min +OnUnitActiveSec=15min +RandomizedDelaySec=2min +Persistent=true + +[Install] +WantedBy=timers.target +EOF + +cat > "$UNIT_DIR/drb-update.path" </dev/null 2>&1 \ + || die "could not enable drb-update.timer/path" +printf 'install-updater: host updater enabled (timer every ~15 min + MQTT sentinel), tracking %s @ %s\n' \ + "$REPO_URL" "$REF"