install.sh (#14): after enrolling it now BLOCKS until an admin approves, printing "Please have an admin approve node '<id>' at <app>/settings/nodes to continue", polling GET /nodes/{id}/credentials with the pickup secret (no default timeout; Ctrl-C safe; re-run resumes from configs/pickup_secret). A re-run with a key on disk re-checks it against C2 and replaces a stale (reissued) key. credentials.json is merged atomically so the dashboard auth salt/secret survive. edge-node is ALWAYS force-recreated at the end, so a key written after the container started is loaded. --wait-approval N caps the wait; 0 = don't wait. git fetch uses --force so a re-cut v1 tag is accepted. edge-node (#14): with no key at startup, or on a broker auth refusal (CONNACK 4/5/134/135), mqtt_manager re-reads credentials.json, else fetches the key over HTTPS (key_pickup.py, pickup secret from /configs), persists it and rebuilds the paho client in place. Single-flight, 30s..15min backoff. A key pushed over MQTT while connected is applied for the next reconnect. OTA (#15): node_update no longer os._exit()s. It drops configs/update-requested; install.sh installs scripts/install-updater.sh's drb-update.{service,timer,path}. The path unit fires on the sentinel, the timer (~15 min) does an HTTPS check-in (POST /nodes/{id}/update-check, server-26 PR) so offline/locked-out nodes still update. drb-update.sh only installs the repo/ref/registry in root-owned /etc/drb/updater.conf; C2's answer is just a timestamp; origin and registry mismatches abort; creds go to curl via a 0600 header file. Tests: test_credential_recovery.py, test_host_updater.py (runs the real updater script with stub git/docker/curl). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Logan Cusano
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
6bf2c9dd93
commit
c2c349043b
@@ -3,8 +3,12 @@ Manages the persisted node API key, plus the local-auth signing material used
|
||||
by app/internal/auth.py.
|
||||
|
||||
The API key is provisioned by the C2 server after an admin approves the node.
|
||||
It arrives via MQTT and is saved to /configs/credentials.json so it survives
|
||||
container restarts.
|
||||
install.sh fetches it over HTTPS (GET /nodes/{id}/credentials + the pickup
|
||||
secret it saved to /configs/pickup_secret) and writes /configs/credentials.json;
|
||||
the running app can also fetch it itself (app/internal/key_pickup.py) and
|
||||
re-read the file at any time (reload_api_key), so a key written after startup
|
||||
is picked up without a container restart (node-26#14). The legacy MQTT
|
||||
delivery (nodes/{id}/api_key) still saves here too.
|
||||
|
||||
The scrypt salt and session-signing secret are generated locally on first boot
|
||||
(never provisioned externally) and persisted the same way, so dashboard
|
||||
@@ -12,12 +16,14 @@ sessions survive a container restart instead of forcing every operator to
|
||||
re-login whenever the node restarts.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
from app.config import settings
|
||||
from app.internal.logger import logger
|
||||
|
||||
_CREDS_FILE = Path(settings.config_path) / "credentials.json"
|
||||
_PICKUP_FILE = Path(settings.config_path) / "pickup_secret"
|
||||
_api_key: str | None = None
|
||||
_auth_salt: bytes | None = None
|
||||
_session_secret: bytes | None = None
|
||||
@@ -74,6 +80,37 @@ def get_api_key() -> str | None:
|
||||
return _api_key
|
||||
|
||||
|
||||
def reload_api_key() -> str | None:
|
||||
"""Re-read the api_key from disk (install.sh, or an operator, may have
|
||||
written it after startup). Returns the key now in effect. Only the api_key
|
||||
is taken from the file — the auth material already in memory stays."""
|
||||
global _api_key
|
||||
try:
|
||||
data = json.loads(_CREDS_FILE.read_text())
|
||||
except FileNotFoundError:
|
||||
return _api_key
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not re-read credentials file: {e}")
|
||||
return _api_key
|
||||
key = data.get("api_key")
|
||||
if key and key != _api_key:
|
||||
_api_key = key
|
||||
logger.info("Node API key re-read from disk (changed since it was last loaded).")
|
||||
return _api_key
|
||||
|
||||
|
||||
def get_pickup_secret() -> str | None:
|
||||
"""The one-time pickup secret install.sh saved at enrollment, if any."""
|
||||
try:
|
||||
secret = _PICKUP_FILE.read_text().strip()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not read pickup secret: {e}")
|
||||
return None
|
||||
return secret or None
|
||||
|
||||
|
||||
def save_api_key(key: str) -> None:
|
||||
global _api_key
|
||||
_api_key = key
|
||||
@@ -88,4 +125,10 @@ def _write() -> None:
|
||||
data["auth_salt"] = _auth_salt.hex()
|
||||
if _session_secret is not None:
|
||||
data["session_secret"] = _session_secret.hex()
|
||||
_CREDS_FILE.write_text(json.dumps(data))
|
||||
# Atomic replace with 0600: install.sh on the host writes this same file,
|
||||
# and a torn half-written JSON would lose the key AND the auth material.
|
||||
tmp = _CREDS_FILE.with_name(_CREDS_FILE.name + ".tmp")
|
||||
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, "w") as f:
|
||||
f.write(json.dumps(data))
|
||||
os.replace(tmp, _CREDS_FILE)
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
"""
|
||||
HTTPS api_key pickup — the node's own half of C2's enrollment flow
|
||||
(Server/drb-c2-core/app/routers/enrollment.py, GET /nodes/{id}/credentials).
|
||||
|
||||
Why this exists (node-26#14): key delivery used to be MQTT-only
|
||||
(nodes/{id}/api_key), and a dynsec broker refuses a node that has no key — or
|
||||
a stale one — so a locked-out node could never receive the key that would let
|
||||
it in. install.sh already did this pickup once at install time; doing it from
|
||||
the running app too means a node approved after install, or whose key was
|
||||
reissued, recovers on its own with no SSH and no container restart.
|
||||
|
||||
Auth is the pickup secret install.sh saved to /configs/pickup_secret at
|
||||
enrollment (the same file, bind-mounted). Only ever sent to settings.c2_url,
|
||||
over whatever scheme the operator configured (https in production).
|
||||
"""
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import settings
|
||||
from app.internal import credentials
|
||||
from app.internal.logger import logger
|
||||
|
||||
_TIMEOUT = 15
|
||||
|
||||
|
||||
async def fetch_api_key() -> Optional[str]:
|
||||
"""Ask C2 for this node's api_key. Returns it, or None when there isn't one
|
||||
to be had yet (not approved, no pickup secret, C2 unreachable, rejected).
|
||||
Never raises — the caller retries on its own backoff."""
|
||||
if not settings.c2_url:
|
||||
logger.warning("Key pickup skipped: C2_URL is not configured.")
|
||||
return None
|
||||
secret = credentials.get_pickup_secret()
|
||||
if not secret:
|
||||
logger.warning(
|
||||
f"Key pickup skipped: no {settings.config_path}/pickup_secret. This node was not enrolled by "
|
||||
"install.sh on this machine — re-run install.sh, or have an admin Reissue key and write it in."
|
||||
)
|
||||
return None
|
||||
|
||||
url = f"{settings.c2_url.rstrip('/')}/nodes/{settings.node_id}/credentials"
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
r = await client.get(url, headers={"X-Pickup-Secret": secret})
|
||||
except Exception as e:
|
||||
logger.warning(f"Key pickup: could not reach C2 ({e}).")
|
||||
return None
|
||||
|
||||
if r.status_code == 401:
|
||||
logger.error(
|
||||
"Key pickup: C2 rejected the saved pickup secret (401) — it was rotated by a re-enroll elsewhere. "
|
||||
"Re-run install.sh with --token to recover."
|
||||
)
|
||||
return None
|
||||
if r.status_code == 404:
|
||||
logger.error("Key pickup: C2 does not know this node (404) — it was deleted server-side. Re-run install.sh.")
|
||||
return None
|
||||
if r.status_code != 200:
|
||||
logger.warning(f"Key pickup: unexpected HTTP {r.status_code} from C2.")
|
||||
return None
|
||||
|
||||
try:
|
||||
body = r.json()
|
||||
except ValueError:
|
||||
logger.warning("Key pickup: C2 returned a non-JSON body.")
|
||||
return None
|
||||
key = body.get("api_key") if isinstance(body, dict) else None
|
||||
if not key or not isinstance(key, str):
|
||||
status = body.get("approval_status") if isinstance(body, dict) else None
|
||||
logger.info(f"Key pickup: no api_key yet (approval_status={status!r}) — waiting for an admin to approve.")
|
||||
return None
|
||||
return key
|
||||
@@ -6,12 +6,30 @@ from typing import Optional, Callable, Awaitable, Dict, Any
|
||||
import paho.mqtt.client as mqtt
|
||||
from app.config import settings
|
||||
from app.internal.logger import logger
|
||||
from app.internal import credentials
|
||||
from app.internal import credentials, key_pickup
|
||||
|
||||
CommandCallback = Callable[[Dict[str, Any]], Awaitable[None]]
|
||||
ConfigCallback = Callable[[Dict[str, Any]], Awaitable[None]]
|
||||
ApiKeyCallback = Callable[[Dict[str, Any]], Awaitable[None]]
|
||||
|
||||
# CONNACK codes meaning "your credentials are wrong", as paho reports them:
|
||||
# MQTT 3.1.1 return codes 4/5, and the MQTT 5 reason codes paho maps them to
|
||||
# (134 Bad user name or password, 135 Not authorized).
|
||||
_AUTH_FAILURE_CODES = {4, 5, 134, 135}
|
||||
|
||||
# Credential-recovery backoff (node-26#14). The pickup endpoint is cheap, but a
|
||||
# node that is simply not approved yet must not hammer C2 for days.
|
||||
RECOVERY_MIN_DELAY = 30
|
||||
RECOVERY_MAX_DELAY = 900
|
||||
|
||||
|
||||
def _is_auth_failure(reason_code) -> bool:
|
||||
value = getattr(reason_code, "value", reason_code)
|
||||
if isinstance(value, int) and value in _AUTH_FAILURE_CODES:
|
||||
return True
|
||||
text = str(reason_code).lower()
|
||||
return "not authori" in text or "bad user name or password" in text
|
||||
|
||||
|
||||
class MQTTManager:
|
||||
def __init__(self):
|
||||
@@ -19,6 +37,11 @@ class MQTTManager:
|
||||
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
self._connected = False
|
||||
self._connect_task: Optional[asyncio.Task] = None
|
||||
# The api_key the CURRENT paho client was built with — not necessarily
|
||||
# credentials.get_api_key(), which can move ahead of it (MQTT key push,
|
||||
# install.sh writing the file). Recovery compares against this.
|
||||
self._client_key: Optional[str] = None
|
||||
self._recovery_task: Optional[asyncio.Task] = None
|
||||
|
||||
self.on_command: Optional[CommandCallback] = None
|
||||
self.on_config_push: Optional[ConfigCallback] = None
|
||||
@@ -53,6 +76,7 @@ class MQTTManager:
|
||||
)
|
||||
|
||||
api_key = credentials.get_api_key()
|
||||
self._client_key = api_key
|
||||
if api_key:
|
||||
# Post-cutover auth: broker's dynsec plugin authenticates this
|
||||
# exact (username, password) pair as this node's own client — see
|
||||
@@ -70,18 +94,14 @@ class MQTTManager:
|
||||
client.username_pw_set(settings.mqtt_user, settings.mqtt_pass)
|
||||
else:
|
||||
# No api_key on disk and no legacy shared login configured. A
|
||||
# dynsec broker (allow_anonymous false) refuses this outright —
|
||||
# expected, not a bug to route around here: this node hasn't been
|
||||
# enrolled/approved yet, and the enrollment flow that would fix
|
||||
# that client-side is a later, separate pass (out of scope here;
|
||||
# see MQTT-PUBLIC-AUTH-PLAN.md). paho's reconnect_delay_set()
|
||||
# below bounds the retry rate (2..60s exponential backoff), so
|
||||
# this degrades to a slow, clearly-logged refusal loop via
|
||||
# _on_connect's "MQTT connect refused" line — not a hot spin.
|
||||
# dynsec broker (allow_anonymous false) refuses this outright.
|
||||
# connect() starts _recover_credentials() in this case, which
|
||||
# fetches the key over HTTPS (key_pickup.py) and rebuilds this
|
||||
# client with it — node-26#14. paho's reconnect_delay_set() below
|
||||
# bounds the refusal loop meanwhile (2..60s backoff).
|
||||
logger.warning(
|
||||
"No API key on disk and no legacy MQTT_USER configured — "
|
||||
"connecting without credentials; the broker is expected to "
|
||||
"refuse this until the node is enrolled/approved."
|
||||
"connecting without credentials while the key is fetched from C2."
|
||||
)
|
||||
|
||||
if settings.mqtt_tls:
|
||||
@@ -120,6 +140,10 @@ class MQTTManager:
|
||||
asyncio.run_coroutine_threadsafe(self._flush_offline_buffer(), self._loop)
|
||||
else:
|
||||
logger.error(f"MQTT connect refused: {reason_code}")
|
||||
if _is_auth_failure(reason_code) and self._loop is not None:
|
||||
# paho thread -> event loop. A refused login never fixes itself
|
||||
# by retrying the same password, so go get the right one.
|
||||
self._loop.call_soon_threadsafe(self._start_recovery, f"broker refused login ({reason_code})")
|
||||
|
||||
def _on_disconnect(self, client, userdata, disconnect_flags, reason_code, properties):
|
||||
self._connected = False
|
||||
@@ -144,6 +168,72 @@ class MQTTManager:
|
||||
self._loop = asyncio.get_event_loop()
|
||||
self._client = self._build_client()
|
||||
self._connect_task = asyncio.create_task(self._connect_with_retry())
|
||||
if not self._client_key and not settings.mqtt_user:
|
||||
self._start_recovery("no API key at startup")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Credential self-heal (node-26#14)
|
||||
#
|
||||
# A node with no key, or a stale one, used to sit in an MQTT refusal loop
|
||||
# forever: key delivery was MQTT-only, and credentials were read once at
|
||||
# startup. Now: re-read credentials.json (install.sh may have written it),
|
||||
# else fetch from C2 over HTTPS with the pickup secret; if that yields a
|
||||
# key different from the one the client is using, rebuild the client with
|
||||
# it in place. No container restart.
|
||||
# ------------------------------------------------------------------
|
||||
def _start_recovery(self, why: str) -> None:
|
||||
if self._recovery_task and not self._recovery_task.done():
|
||||
return
|
||||
logger.warning(f"MQTT credential recovery started: {why}")
|
||||
self._recovery_task = asyncio.create_task(self._recover_credentials())
|
||||
|
||||
async def _recover_credentials(self) -> None:
|
||||
delay = RECOVERY_MIN_DELAY
|
||||
while True:
|
||||
tried = self._client_key
|
||||
key = credentials.reload_api_key()
|
||||
if not key or key == tried:
|
||||
fetched = await key_pickup.fetch_api_key()
|
||||
if fetched and fetched != tried:
|
||||
credentials.save_api_key(fetched)
|
||||
key = fetched
|
||||
if key and key != tried:
|
||||
logger.info("MQTT credential recovery: new API key in hand — reconnecting with it.")
|
||||
await self._rebuild_client()
|
||||
return
|
||||
if self._connected:
|
||||
return
|
||||
if key and key == tried:
|
||||
logger.error(
|
||||
"MQTT credential recovery: C2 returns the same key the broker refuses. The broker's login for "
|
||||
"this node is out of sync with C2 (dynsec) — an admin re-approve re-syncs it without rotating."
|
||||
)
|
||||
logger.info(f"MQTT credential recovery: retrying in {delay}s.")
|
||||
await asyncio.sleep(delay)
|
||||
delay = min(delay * 2, RECOVERY_MAX_DELAY)
|
||||
|
||||
async def _rebuild_client(self) -> None:
|
||||
"""Swap the paho client for one built from the current credentials."""
|
||||
old = self._client
|
||||
if self._connect_task and not self._connect_task.done():
|
||||
self._connect_task.cancel()
|
||||
if old is not None:
|
||||
# loop_stop() joins paho's thread, which can be mid-backoff — off the event loop.
|
||||
await asyncio.to_thread(old.loop_stop)
|
||||
try:
|
||||
old.disconnect()
|
||||
except Exception:
|
||||
pass
|
||||
self._connected = False
|
||||
self._client = self._build_client()
|
||||
self._connect_task = asyncio.create_task(self._connect_with_retry())
|
||||
|
||||
def update_api_key(self, key: str) -> None:
|
||||
"""A key arrived while connected (legacy MQTT push / reissue): use it on
|
||||
the next reconnect without dropping the current, working session."""
|
||||
if self._client is not None and key != self._client_key:
|
||||
self._client.username_pw_set(settings.node_id, key)
|
||||
self._client_key = key
|
||||
|
||||
async def _connect_with_retry(self):
|
||||
"""Attempt the initial TCP connect, retrying with backoff until it succeeds."""
|
||||
@@ -161,6 +251,8 @@ class MQTTManager:
|
||||
delay = min(delay * 2, 60)
|
||||
|
||||
async def disconnect(self):
|
||||
if self._recovery_task:
|
||||
self._recovery_task.cancel()
|
||||
if self._connect_task:
|
||||
self._connect_task.cancel()
|
||||
if self._client:
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""
|
||||
Hand an MQTT `node_update` command to the host-level updater (node-26#15).
|
||||
|
||||
The edge-node container has no business updating its own host: it writes a
|
||||
sentinel file into /configs (bind-mounted from <install dir>/configs), and the
|
||||
host's systemd drb-update.path unit — installed by install.sh — triggers
|
||||
drb-update.service, which runs Client/scripts/drb-update.sh. The host treats
|
||||
the file as a bare "please update" flag and never reads its content as input,
|
||||
so nothing C2 sends can pick the ref, image or command that runs.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from app.config import settings
|
||||
from app.internal.logger import logger
|
||||
|
||||
SENTINEL_NAME = "update-requested"
|
||||
|
||||
|
||||
def sentinel_path() -> Path:
|
||||
return Path(settings.config_path) / SENTINEL_NAME
|
||||
|
||||
|
||||
def request_host_update() -> bool:
|
||||
"""Drop the sentinel. Returns False if it could not be written."""
|
||||
path = sentinel_path()
|
||||
try:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_name(SENTINEL_NAME + ".tmp")
|
||||
tmp.write_text(json.dumps({"requested_at": datetime.now(timezone.utc).isoformat()}))
|
||||
os.replace(tmp, path)
|
||||
except OSError as e:
|
||||
logger.error(f"Node update requested but the sentinel could not be written ({e}).")
|
||||
return False
|
||||
logger.info(
|
||||
f"Node update requested — handed to the host updater via {path}. If nothing happens within a minute, "
|
||||
"this node predates the updater: re-run install.sh on it once."
|
||||
)
|
||||
return True
|
||||
+10
-11
@@ -170,17 +170,15 @@ async def on_command(payload: dict):
|
||||
await asyncio.sleep(2)
|
||||
await op25_client.start()
|
||||
elif action == "node_update":
|
||||
# TODO: Full OTA update — register a host-level systemd service (e.g. drb-update.service)
|
||||
# that stops all DRB containers, runs `docker compose pull`, then `docker compose up -d`.
|
||||
# The C2 server triggers it by sending this MQTT command; the host service watches for the
|
||||
# restart signal (e.g. via a Unix socket, a sentinel file, or a lightweight webhook).
|
||||
# Not implemented yet — for now, just restart the container so any pre-pulled image
|
||||
# is picked up (requires a prior `docker compose pull` on the host).
|
||||
logger.info("Node update requested — restarting container to pick up latest image.")
|
||||
await mqtt_manager.publish_status("offline")
|
||||
await asyncio.sleep(1)
|
||||
import os
|
||||
os._exit(0) # Docker restart=unless-stopped will bring the container back up
|
||||
# node-26#15: the container cannot (and must not) update itself. It
|
||||
# drops a sentinel in /configs; the host's drb-update.path unit
|
||||
# (installed by install.sh) sees it and runs drb-update.service, which
|
||||
# pulls the node's own pinned repo ref + images and recreates the
|
||||
# stack. The sentinel's content is ignored by the host — C2 cannot
|
||||
# choose what gets installed. Offline/locked-out nodes get the same
|
||||
# request over HTTPS via the updater's timer instead.
|
||||
from app.internal.update_request import request_host_update
|
||||
request_host_update()
|
||||
else:
|
||||
logger.warning(f"Unknown command: {action}")
|
||||
|
||||
@@ -189,6 +187,7 @@ async def on_api_key(payload: dict):
|
||||
key = payload.get("api_key")
|
||||
if key:
|
||||
credentials.save_api_key(key)
|
||||
mqtt_manager.update_api_key(key)
|
||||
logger.info("Node API key received and saved.")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
"""
|
||||
node-26#14 — a node with no key, or a stale key, recovers on its own:
|
||||
re-read credentials.json, else fetch over HTTPS with the pickup secret, then
|
||||
rebuild the MQTT client in place. node-26#15 — node_update hands off to the
|
||||
host updater via a sentinel file.
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.config import settings
|
||||
from app.internal import credentials, key_pickup, mqtt_manager as mm_mod, update_request
|
||||
from app.internal.mqtt_manager import MQTTManager, _is_auth_failure
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolated(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(credentials, "_CREDS_FILE", tmp_path / "credentials.json")
|
||||
monkeypatch.setattr(credentials, "_PICKUP_FILE", tmp_path / "pickup_secret")
|
||||
monkeypatch.setattr(credentials, "_api_key", None)
|
||||
monkeypatch.setattr(settings, "config_path", str(tmp_path))
|
||||
monkeypatch.setattr(settings, "c2_url", "https://c2.example")
|
||||
monkeypatch.setattr(settings, "mqtt_user", None)
|
||||
monkeypatch.setattr(mm_mod, "RECOVERY_MIN_DELAY", 0)
|
||||
yield tmp_path
|
||||
|
||||
|
||||
def _manager(client_key=None):
|
||||
m = MQTTManager()
|
||||
m._client_key = client_key
|
||||
m._rebuild_client = AsyncMock()
|
||||
return m
|
||||
|
||||
|
||||
# --- credentials.py -----------------------------------------------------------
|
||||
|
||||
def test_reload_picks_up_a_key_written_after_startup(isolated):
|
||||
credentials.load()
|
||||
assert credentials.get_api_key() is None
|
||||
data = json.loads((isolated / "credentials.json").read_text())
|
||||
data["api_key"] = "written-by-install-sh"
|
||||
(isolated / "credentials.json").write_text(json.dumps(data))
|
||||
assert credentials.reload_api_key() == "written-by-install-sh"
|
||||
|
||||
|
||||
def test_write_is_atomic_and_private(isolated):
|
||||
credentials.save_api_key("k1")
|
||||
f = isolated / "credentials.json"
|
||||
assert json.loads(f.read_text())["api_key"] == "k1"
|
||||
assert (f.stat().st_mode & 0o777) == 0o600
|
||||
assert not (isolated / "credentials.json.tmp").exists()
|
||||
|
||||
|
||||
def test_pickup_secret_read_and_absent(isolated):
|
||||
assert credentials.get_pickup_secret() is None
|
||||
(isolated / "pickup_secret").write_text("abc123\n")
|
||||
assert credentials.get_pickup_secret() == "abc123"
|
||||
|
||||
|
||||
# --- auth-failure classification -------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("code", [4, 5, 134, 135])
|
||||
def test_auth_failure_codes(code):
|
||||
assert _is_auth_failure(code)
|
||||
|
||||
|
||||
def test_non_auth_failures_do_not_trigger_recovery():
|
||||
assert not _is_auth_failure(3) # server unavailable
|
||||
assert not _is_auth_failure(136)
|
||||
|
||||
|
||||
def test_paho_reason_code_objects_classified():
|
||||
from paho.mqtt.reasoncodes import ReasonCode
|
||||
from paho.mqtt.packettypes import PacketTypes
|
||||
assert _is_auth_failure(ReasonCode(PacketTypes.CONNACK, "Not authorized"))
|
||||
assert _is_auth_failure(ReasonCode(PacketTypes.CONNACK, "Bad user name or password"))
|
||||
assert not _is_auth_failure(ReasonCode(PacketTypes.CONNACK, "Server unavailable"))
|
||||
|
||||
|
||||
# --- recovery loop ----------------------------------------------------------------
|
||||
|
||||
async def test_recovery_uses_key_already_on_disk_without_calling_c2(isolated, monkeypatch):
|
||||
credentials.save_api_key("disk-key")
|
||||
monkeypatch.setattr(credentials, "_api_key", None) # process started before the write
|
||||
fetch = AsyncMock()
|
||||
monkeypatch.setattr(key_pickup, "fetch_api_key", fetch)
|
||||
m = _manager(client_key=None)
|
||||
await m._recover_credentials()
|
||||
m._rebuild_client.assert_awaited_once()
|
||||
fetch.assert_not_awaited()
|
||||
|
||||
|
||||
async def test_recovery_fetches_over_https_and_persists(isolated, monkeypatch):
|
||||
monkeypatch.setattr(key_pickup, "fetch_api_key", AsyncMock(return_value="fresh-key"))
|
||||
m = _manager(client_key="stale-key")
|
||||
monkeypatch.setattr(credentials, "_api_key", "stale-key")
|
||||
await m._recover_credentials()
|
||||
m._rebuild_client.assert_awaited_once()
|
||||
assert credentials.get_api_key() == "fresh-key"
|
||||
assert json.loads((isolated / "credentials.json").read_text())["api_key"] == "fresh-key"
|
||||
|
||||
|
||||
async def test_recovery_backs_off_until_approved(isolated, monkeypatch):
|
||||
fetch = AsyncMock(side_effect=[None, None, "approved-key"])
|
||||
monkeypatch.setattr(key_pickup, "fetch_api_key", fetch)
|
||||
m = _manager()
|
||||
await asyncio.wait_for(m._recover_credentials(), timeout=5)
|
||||
assert fetch.await_count == 3
|
||||
m._rebuild_client.assert_awaited_once()
|
||||
|
||||
|
||||
async def test_recovery_stops_if_connection_recovers_meanwhile(isolated, monkeypatch):
|
||||
monkeypatch.setattr(key_pickup, "fetch_api_key", AsyncMock(return_value=None))
|
||||
m = _manager()
|
||||
m._connected = True
|
||||
await asyncio.wait_for(m._recover_credentials(), timeout=5)
|
||||
m._rebuild_client.assert_not_awaited()
|
||||
|
||||
|
||||
async def test_start_recovery_is_single_flight(isolated, monkeypatch):
|
||||
gate = asyncio.Event()
|
||||
|
||||
async def slow_fetch():
|
||||
await gate.wait()
|
||||
return "k"
|
||||
monkeypatch.setattr(key_pickup, "fetch_api_key", slow_fetch)
|
||||
m = _manager()
|
||||
m._start_recovery("a")
|
||||
first = m._recovery_task
|
||||
m._start_recovery("b")
|
||||
assert m._recovery_task is first
|
||||
gate.set()
|
||||
await first
|
||||
|
||||
|
||||
async def test_connect_without_key_starts_recovery(isolated, monkeypatch):
|
||||
m = MQTTManager()
|
||||
started = []
|
||||
monkeypatch.setattr(m, "_connect_with_retry", AsyncMock())
|
||||
monkeypatch.setattr(m, "_start_recovery", lambda why: started.append(why))
|
||||
await m.connect()
|
||||
assert started == ["no API key at startup"]
|
||||
|
||||
|
||||
def test_update_api_key_changes_creds_for_next_reconnect(isolated):
|
||||
m = MQTTManager()
|
||||
m._client = m._build_client()
|
||||
m.update_api_key("rotated")
|
||||
assert m._client._password == b"rotated"
|
||||
assert m._client_key == "rotated"
|
||||
|
||||
|
||||
# --- key_pickup.fetch_api_key -----------------------------------------------------
|
||||
|
||||
def _mock_http(monkeypatch, handler):
|
||||
real = httpx.AsyncClient
|
||||
|
||||
def factory(*a, **kw):
|
||||
kw["transport"] = httpx.MockTransport(handler)
|
||||
return real(*a, **kw)
|
||||
monkeypatch.setattr(key_pickup.httpx, "AsyncClient", factory)
|
||||
|
||||
|
||||
async def test_fetch_sends_pickup_secret_and_returns_key(isolated, monkeypatch):
|
||||
(isolated / "pickup_secret").write_text("the-secret")
|
||||
seen = {}
|
||||
|
||||
def handler(request):
|
||||
seen["url"] = str(request.url)
|
||||
seen["secret"] = request.headers.get("X-Pickup-Secret")
|
||||
return httpx.Response(200, json={"approval_status": "approved", "api_key": "K"})
|
||||
_mock_http(monkeypatch, handler)
|
||||
assert await key_pickup.fetch_api_key() == "K"
|
||||
assert seen == {"url": f"https://c2.example/nodes/{settings.node_id}/credentials", "secret": "the-secret"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("resp", [
|
||||
httpx.Response(200, json={"approval_status": "pending", "api_key": None}),
|
||||
httpx.Response(401, json={"detail": "Invalid pickup secret"}),
|
||||
httpx.Response(404),
|
||||
httpx.Response(500),
|
||||
httpx.Response(200, text="not json"),
|
||||
])
|
||||
async def test_fetch_returns_none_when_no_key(isolated, monkeypatch, resp):
|
||||
(isolated / "pickup_secret").write_text("s")
|
||||
_mock_http(monkeypatch, lambda request: resp)
|
||||
assert await key_pickup.fetch_api_key() is None
|
||||
|
||||
|
||||
async def test_fetch_without_pickup_secret_does_not_call_c2(isolated, monkeypatch):
|
||||
def handler(request):
|
||||
raise AssertionError("must not call C2 without a pickup secret")
|
||||
_mock_http(monkeypatch, handler)
|
||||
assert await key_pickup.fetch_api_key() is None
|
||||
|
||||
|
||||
# --- node_update -> host sentinel ---------------------------------------------------
|
||||
|
||||
def test_node_update_writes_sentinel(isolated):
|
||||
assert update_request.request_host_update() is True
|
||||
data = json.loads((isolated / "update-requested").read_text())
|
||||
assert "requested_at" in data
|
||||
@@ -0,0 +1,179 @@
|
||||
"""
|
||||
node-26#15 — Client/scripts/drb-update.sh, the host-level updater.
|
||||
|
||||
Runs the real script with stub git/docker/curl on PATH, so what is checked is
|
||||
the script's decision logic and its security guards: it only updates when
|
||||
asked (MQTT sentinel or a new C2 request), never takes anything but "update
|
||||
now" from C2, refuses a mismatched git origin or registry, and keeps node
|
||||
credentials off curl's argv.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "drb-update.sh"
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (SCRIPT.exists() and shutil.which("bash") and shutil.which("jq") and shutil.which("flock")),
|
||||
reason="needs the repo's scripts/ (absent inside the image) plus bash, jq and flock",
|
||||
)
|
||||
|
||||
REPO = "https://git.example.net/logan/node-26.git"
|
||||
OLD, NEW = "a" * 40, "b" * 40
|
||||
|
||||
STUB_GIT = r'''#!/usr/bin/env bash
|
||||
echo "git $*" >> "$STUB/calls"
|
||||
while [ "$1" = -c ]; do shift 2; done
|
||||
case "$1" in
|
||||
rev-parse) if [ "$2" = HEAD ]; then cat "$STUB/head"; else echo "$STUB_NEW"; fi ;;
|
||||
remote) echo "$STUB_ORIGIN" ;;
|
||||
fetch) exit 0 ;;
|
||||
show-ref) [ "$4" = "refs/tags/v1" ] ;;
|
||||
checkout) echo "${@: -1}" > "$STUB/head" ;;
|
||||
*) exit 0 ;;
|
||||
esac
|
||||
'''
|
||||
STUB_DOCKER = '#!/usr/bin/env bash\necho "docker $*" >> "$STUB/calls"\n'
|
||||
STUB_CURL = r'''#!/usr/bin/env bash
|
||||
echo "curl $*" >> "$STUB/calls"
|
||||
for a in "$@"; do case "$a" in @*) cat "${a#@}" >> "$STUB/headers" ;; esac; done
|
||||
cat "$STUB/response"
|
||||
'''
|
||||
|
||||
|
||||
def _exe(path: Path, body: str):
|
||||
path.write_text(body)
|
||||
path.chmod(path.stat().st_mode | stat.S_IEXEC)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def env(tmp_path):
|
||||
stub = tmp_path / "stub"
|
||||
bindir = stub / "bin"
|
||||
bindir.mkdir(parents=True)
|
||||
_exe(bindir / "git", STUB_GIT)
|
||||
_exe(bindir / "docker", STUB_DOCKER)
|
||||
_exe(bindir / "curl", STUB_CURL)
|
||||
(stub / "head").write_text(OLD + "\n")
|
||||
(stub / "calls").write_text("")
|
||||
(stub / "response").write_text(json.dumps({"update_requested_at": None}))
|
||||
|
||||
inst = tmp_path / "node"
|
||||
(inst / ".git").mkdir(parents=True)
|
||||
(inst / "configs").mkdir()
|
||||
(inst / ".env").write_text(
|
||||
"NODE_ID=node-7\nC2_URL=https://api.example.net\nIMAGE_REGISTRY=git.example.net\n"
|
||||
)
|
||||
(inst / "configs" / "credentials.json").write_text(json.dumps({"api_key": "k" * 64}))
|
||||
(inst / "configs" / "pickup_secret").write_text("p" * 48)
|
||||
|
||||
conf = tmp_path / "updater.conf"
|
||||
conf.write_text(f"INSTALL_DIR={inst}\nREPO_URL={REPO}\nREF=v1\nREGISTRY=git.example.net\nBUILD=0\n")
|
||||
|
||||
e = dict(os.environ)
|
||||
e.update({
|
||||
"PATH": f"{bindir}:{os.environ['PATH']}",
|
||||
"STUB": str(stub), "STUB_NEW": NEW, "STUB_ORIGIN": REPO,
|
||||
"DRB_UPDATER_CONF": str(conf),
|
||||
"DRB_UPDATER_STATE": str(tmp_path / "state"),
|
||||
"DRB_UPDATER_LOCK": str(tmp_path / "lock"),
|
||||
})
|
||||
return {"env": e, "stub": stub, "inst": inst, "conf": conf, "state": tmp_path / "state"}
|
||||
|
||||
|
||||
def _run(env):
|
||||
return subprocess.run(["bash", str(SCRIPT)], env=env["env"], capture_output=True, text=True, timeout=30)
|
||||
|
||||
|
||||
def _calls(env):
|
||||
return (env["stub"] / "calls").read_text()
|
||||
|
||||
|
||||
def test_no_request_means_checkin_only(env):
|
||||
r = _run(env)
|
||||
assert r.returncode == 0, r.stderr
|
||||
calls = _calls(env)
|
||||
assert "update-check" in calls
|
||||
assert "docker" not in calls
|
||||
assert "fetch" not in calls
|
||||
|
||||
|
||||
def test_c2_request_updates_once_and_reports(env):
|
||||
(env["stub"] / "response").write_text(json.dumps({"update_requested_at": "2026-10-04T12:00:00+00:00"}))
|
||||
r = _run(env)
|
||||
assert r.returncode == 0, r.stderr + r.stdout
|
||||
calls = _calls(env)
|
||||
assert "docker compose pull" in calls
|
||||
assert "docker compose up -d --no-build --force-recreate --remove-orphans" in calls
|
||||
assert (env["stub"] / "head").read_text().strip() == NEW
|
||||
assert '"applied_request_at":"2026-10-04T12:00:00+00:00"' in calls
|
||||
assert '"result":"ok"' in calls
|
||||
# Same request again: nothing to do.
|
||||
(env["stub"] / "calls").write_text("")
|
||||
assert _run(env).returncode == 0
|
||||
assert "docker" not in _calls(env)
|
||||
|
||||
|
||||
def test_mqtt_sentinel_triggers_update_and_is_consumed(env):
|
||||
sentinel = env["inst"] / "configs" / "update-requested"
|
||||
sentinel.write_text('{"ref": "evil", "cmd": "rm -rf /"}')
|
||||
r = _run(env)
|
||||
assert r.returncode == 0, r.stderr
|
||||
assert not sentinel.exists()
|
||||
calls = _calls(env)
|
||||
assert "docker compose pull" in calls
|
||||
assert "evil" not in calls and "rm -rf" not in calls
|
||||
|
||||
|
||||
def test_c2_response_is_never_used_as_a_ref_or_command(env):
|
||||
(env["stub"] / "response").write_text(json.dumps({"update_requested_at": "$(touch /tmp/pwned); main"}))
|
||||
r = _run(env)
|
||||
assert r.returncode == 0
|
||||
assert "docker" not in _calls(env) # malformed request is ignored, not executed
|
||||
|
||||
|
||||
def test_refuses_when_git_origin_differs(env):
|
||||
env["env"]["STUB_ORIGIN"] = "https://attacker.example/node-26.git"
|
||||
(env["inst"] / "configs" / "update-requested").write_text("")
|
||||
r = _run(env)
|
||||
assert r.returncode != 0
|
||||
assert "refusing" in r.stderr
|
||||
assert "docker" not in _calls(env)
|
||||
|
||||
|
||||
def test_refuses_when_env_registry_differs(env):
|
||||
(env["inst"] / ".env").write_text(
|
||||
"NODE_ID=node-7\nC2_URL=https://api.example.net\nIMAGE_REGISTRY=evil.example\n"
|
||||
)
|
||||
(env["inst"] / "configs" / "update-requested").write_text("")
|
||||
r = _run(env)
|
||||
assert r.returncode != 0
|
||||
assert "docker compose pull" not in _calls(env)
|
||||
|
||||
|
||||
def test_non_https_repo_in_conf_is_rejected(env):
|
||||
env["conf"].write_text(env["conf"].read_text().replace("https://", "http://"))
|
||||
r = _run(env)
|
||||
assert r.returncode != 0
|
||||
assert "git" not in _calls(env)
|
||||
|
||||
|
||||
def test_credentials_go_in_a_header_file_never_argv(env):
|
||||
_run(env)
|
||||
calls = _calls(env)
|
||||
assert "k" * 64 not in calls and "p" * 48 not in calls
|
||||
headers = (env["stub"] / "headers").read_text()
|
||||
assert f"Authorization: Bearer {'k' * 64}" in headers
|
||||
assert f"X-Pickup-Secret: {'p' * 48}" in headers
|
||||
|
||||
|
||||
def test_plain_http_c2_skips_checkin(env):
|
||||
(env["inst"] / ".env").write_text("NODE_ID=node-7\nC2_URL=http://api.example.net\nIMAGE_REGISTRY=git.example.net\n")
|
||||
r = _run(env)
|
||||
assert r.returncode == 0
|
||||
assert "curl" not in _calls(env)
|
||||
Reference in New Issue
Block a user