install.sh (#14): after enrolling it now BLOCKS until an admin approves, printing "Please have an admin approve node '<id>' at <app>/settings/nodes to continue", polling GET /nodes/{id}/credentials with the pickup secret (no default timeout; Ctrl-C safe; re-run resumes from configs/pickup_secret). A re-run with a key on disk re-checks it against C2 and replaces a stale (reissued) key. credentials.json is merged atomically so the dashboard auth salt/secret survive. edge-node is ALWAYS force-recreated at the end, so a key written after the container started is loaded. --wait-approval N caps the wait; 0 = don't wait. git fetch uses --force so a re-cut v1 tag is accepted. edge-node (#14): with no key at startup, or on a broker auth refusal (CONNACK 4/5/134/135), mqtt_manager re-reads credentials.json, else fetches the key over HTTPS (key_pickup.py, pickup secret from /configs), persists it and rebuilds the paho client in place. Single-flight, 30s..15min backoff. A key pushed over MQTT while connected is applied for the next reconnect. OTA (#15): node_update no longer os._exit()s. It drops configs/update-requested; install.sh installs scripts/install-updater.sh's drb-update.{service,timer,path}. The path unit fires on the sentinel, the timer (~15 min) does an HTTPS check-in (POST /nodes/{id}/update-check, server-26 PR) so offline/locked-out nodes still update. drb-update.sh only installs the repo/ref/registry in root-owned /etc/drb/updater.conf; C2's answer is just a timestamp; origin and registry mismatches abort; creds go to curl via a 0600 header file. Tests: test_credential_recovery.py, test_host_updater.py (runs the real updater script with stub git/docker/curl). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Logan Cusano
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
6bf2c9dd93
commit
c2c349043b
@@ -0,0 +1,205 @@
|
||||
"""
|
||||
node-26#14 — a node with no key, or a stale key, recovers on its own:
|
||||
re-read credentials.json, else fetch over HTTPS with the pickup secret, then
|
||||
rebuild the MQTT client in place. node-26#15 — node_update hands off to the
|
||||
host updater via a sentinel file.
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.config import settings
|
||||
from app.internal import credentials, key_pickup, mqtt_manager as mm_mod, update_request
|
||||
from app.internal.mqtt_manager import MQTTManager, _is_auth_failure
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolated(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(credentials, "_CREDS_FILE", tmp_path / "credentials.json")
|
||||
monkeypatch.setattr(credentials, "_PICKUP_FILE", tmp_path / "pickup_secret")
|
||||
monkeypatch.setattr(credentials, "_api_key", None)
|
||||
monkeypatch.setattr(settings, "config_path", str(tmp_path))
|
||||
monkeypatch.setattr(settings, "c2_url", "https://c2.example")
|
||||
monkeypatch.setattr(settings, "mqtt_user", None)
|
||||
monkeypatch.setattr(mm_mod, "RECOVERY_MIN_DELAY", 0)
|
||||
yield tmp_path
|
||||
|
||||
|
||||
def _manager(client_key=None):
|
||||
m = MQTTManager()
|
||||
m._client_key = client_key
|
||||
m._rebuild_client = AsyncMock()
|
||||
return m
|
||||
|
||||
|
||||
# --- credentials.py -----------------------------------------------------------
|
||||
|
||||
def test_reload_picks_up_a_key_written_after_startup(isolated):
|
||||
credentials.load()
|
||||
assert credentials.get_api_key() is None
|
||||
data = json.loads((isolated / "credentials.json").read_text())
|
||||
data["api_key"] = "written-by-install-sh"
|
||||
(isolated / "credentials.json").write_text(json.dumps(data))
|
||||
assert credentials.reload_api_key() == "written-by-install-sh"
|
||||
|
||||
|
||||
def test_write_is_atomic_and_private(isolated):
|
||||
credentials.save_api_key("k1")
|
||||
f = isolated / "credentials.json"
|
||||
assert json.loads(f.read_text())["api_key"] == "k1"
|
||||
assert (f.stat().st_mode & 0o777) == 0o600
|
||||
assert not (isolated / "credentials.json.tmp").exists()
|
||||
|
||||
|
||||
def test_pickup_secret_read_and_absent(isolated):
|
||||
assert credentials.get_pickup_secret() is None
|
||||
(isolated / "pickup_secret").write_text("abc123\n")
|
||||
assert credentials.get_pickup_secret() == "abc123"
|
||||
|
||||
|
||||
# --- auth-failure classification -------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("code", [4, 5, 134, 135])
|
||||
def test_auth_failure_codes(code):
|
||||
assert _is_auth_failure(code)
|
||||
|
||||
|
||||
def test_non_auth_failures_do_not_trigger_recovery():
|
||||
assert not _is_auth_failure(3) # server unavailable
|
||||
assert not _is_auth_failure(136)
|
||||
|
||||
|
||||
def test_paho_reason_code_objects_classified():
|
||||
from paho.mqtt.reasoncodes import ReasonCode
|
||||
from paho.mqtt.packettypes import PacketTypes
|
||||
assert _is_auth_failure(ReasonCode(PacketTypes.CONNACK, "Not authorized"))
|
||||
assert _is_auth_failure(ReasonCode(PacketTypes.CONNACK, "Bad user name or password"))
|
||||
assert not _is_auth_failure(ReasonCode(PacketTypes.CONNACK, "Server unavailable"))
|
||||
|
||||
|
||||
# --- recovery loop ----------------------------------------------------------------
|
||||
|
||||
async def test_recovery_uses_key_already_on_disk_without_calling_c2(isolated, monkeypatch):
|
||||
credentials.save_api_key("disk-key")
|
||||
monkeypatch.setattr(credentials, "_api_key", None) # process started before the write
|
||||
fetch = AsyncMock()
|
||||
monkeypatch.setattr(key_pickup, "fetch_api_key", fetch)
|
||||
m = _manager(client_key=None)
|
||||
await m._recover_credentials()
|
||||
m._rebuild_client.assert_awaited_once()
|
||||
fetch.assert_not_awaited()
|
||||
|
||||
|
||||
async def test_recovery_fetches_over_https_and_persists(isolated, monkeypatch):
|
||||
monkeypatch.setattr(key_pickup, "fetch_api_key", AsyncMock(return_value="fresh-key"))
|
||||
m = _manager(client_key="stale-key")
|
||||
monkeypatch.setattr(credentials, "_api_key", "stale-key")
|
||||
await m._recover_credentials()
|
||||
m._rebuild_client.assert_awaited_once()
|
||||
assert credentials.get_api_key() == "fresh-key"
|
||||
assert json.loads((isolated / "credentials.json").read_text())["api_key"] == "fresh-key"
|
||||
|
||||
|
||||
async def test_recovery_backs_off_until_approved(isolated, monkeypatch):
|
||||
fetch = AsyncMock(side_effect=[None, None, "approved-key"])
|
||||
monkeypatch.setattr(key_pickup, "fetch_api_key", fetch)
|
||||
m = _manager()
|
||||
await asyncio.wait_for(m._recover_credentials(), timeout=5)
|
||||
assert fetch.await_count == 3
|
||||
m._rebuild_client.assert_awaited_once()
|
||||
|
||||
|
||||
async def test_recovery_stops_if_connection_recovers_meanwhile(isolated, monkeypatch):
|
||||
monkeypatch.setattr(key_pickup, "fetch_api_key", AsyncMock(return_value=None))
|
||||
m = _manager()
|
||||
m._connected = True
|
||||
await asyncio.wait_for(m._recover_credentials(), timeout=5)
|
||||
m._rebuild_client.assert_not_awaited()
|
||||
|
||||
|
||||
async def test_start_recovery_is_single_flight(isolated, monkeypatch):
|
||||
gate = asyncio.Event()
|
||||
|
||||
async def slow_fetch():
|
||||
await gate.wait()
|
||||
return "k"
|
||||
monkeypatch.setattr(key_pickup, "fetch_api_key", slow_fetch)
|
||||
m = _manager()
|
||||
m._start_recovery("a")
|
||||
first = m._recovery_task
|
||||
m._start_recovery("b")
|
||||
assert m._recovery_task is first
|
||||
gate.set()
|
||||
await first
|
||||
|
||||
|
||||
async def test_connect_without_key_starts_recovery(isolated, monkeypatch):
|
||||
m = MQTTManager()
|
||||
started = []
|
||||
monkeypatch.setattr(m, "_connect_with_retry", AsyncMock())
|
||||
monkeypatch.setattr(m, "_start_recovery", lambda why: started.append(why))
|
||||
await m.connect()
|
||||
assert started == ["no API key at startup"]
|
||||
|
||||
|
||||
def test_update_api_key_changes_creds_for_next_reconnect(isolated):
|
||||
m = MQTTManager()
|
||||
m._client = m._build_client()
|
||||
m.update_api_key("rotated")
|
||||
assert m._client._password == b"rotated"
|
||||
assert m._client_key == "rotated"
|
||||
|
||||
|
||||
# --- key_pickup.fetch_api_key -----------------------------------------------------
|
||||
|
||||
def _mock_http(monkeypatch, handler):
|
||||
real = httpx.AsyncClient
|
||||
|
||||
def factory(*a, **kw):
|
||||
kw["transport"] = httpx.MockTransport(handler)
|
||||
return real(*a, **kw)
|
||||
monkeypatch.setattr(key_pickup.httpx, "AsyncClient", factory)
|
||||
|
||||
|
||||
async def test_fetch_sends_pickup_secret_and_returns_key(isolated, monkeypatch):
|
||||
(isolated / "pickup_secret").write_text("the-secret")
|
||||
seen = {}
|
||||
|
||||
def handler(request):
|
||||
seen["url"] = str(request.url)
|
||||
seen["secret"] = request.headers.get("X-Pickup-Secret")
|
||||
return httpx.Response(200, json={"approval_status": "approved", "api_key": "K"})
|
||||
_mock_http(monkeypatch, handler)
|
||||
assert await key_pickup.fetch_api_key() == "K"
|
||||
assert seen == {"url": f"https://c2.example/nodes/{settings.node_id}/credentials", "secret": "the-secret"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("resp", [
|
||||
httpx.Response(200, json={"approval_status": "pending", "api_key": None}),
|
||||
httpx.Response(401, json={"detail": "Invalid pickup secret"}),
|
||||
httpx.Response(404),
|
||||
httpx.Response(500),
|
||||
httpx.Response(200, text="not json"),
|
||||
])
|
||||
async def test_fetch_returns_none_when_no_key(isolated, monkeypatch, resp):
|
||||
(isolated / "pickup_secret").write_text("s")
|
||||
_mock_http(monkeypatch, lambda request: resp)
|
||||
assert await key_pickup.fetch_api_key() is None
|
||||
|
||||
|
||||
async def test_fetch_without_pickup_secret_does_not_call_c2(isolated, monkeypatch):
|
||||
def handler(request):
|
||||
raise AssertionError("must not call C2 without a pickup secret")
|
||||
_mock_http(monkeypatch, handler)
|
||||
assert await key_pickup.fetch_api_key() is None
|
||||
|
||||
|
||||
# --- node_update -> host sentinel ---------------------------------------------------
|
||||
|
||||
def test_node_update_writes_sentinel(isolated):
|
||||
assert update_request.request_host_update() is True
|
||||
data = json.loads((isolated / "update-requested").read_text())
|
||||
assert "requested_at" in data
|
||||
@@ -0,0 +1,179 @@
|
||||
"""
|
||||
node-26#15 — Client/scripts/drb-update.sh, the host-level updater.
|
||||
|
||||
Runs the real script with stub git/docker/curl on PATH, so what is checked is
|
||||
the script's decision logic and its security guards: it only updates when
|
||||
asked (MQTT sentinel or a new C2 request), never takes anything but "update
|
||||
now" from C2, refuses a mismatched git origin or registry, and keeps node
|
||||
credentials off curl's argv.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "drb-update.sh"
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (SCRIPT.exists() and shutil.which("bash") and shutil.which("jq") and shutil.which("flock")),
|
||||
reason="needs the repo's scripts/ (absent inside the image) plus bash, jq and flock",
|
||||
)
|
||||
|
||||
REPO = "https://git.example.net/logan/node-26.git"
|
||||
OLD, NEW = "a" * 40, "b" * 40
|
||||
|
||||
STUB_GIT = r'''#!/usr/bin/env bash
|
||||
echo "git $*" >> "$STUB/calls"
|
||||
while [ "$1" = -c ]; do shift 2; done
|
||||
case "$1" in
|
||||
rev-parse) if [ "$2" = HEAD ]; then cat "$STUB/head"; else echo "$STUB_NEW"; fi ;;
|
||||
remote) echo "$STUB_ORIGIN" ;;
|
||||
fetch) exit 0 ;;
|
||||
show-ref) [ "$4" = "refs/tags/v1" ] ;;
|
||||
checkout) echo "${@: -1}" > "$STUB/head" ;;
|
||||
*) exit 0 ;;
|
||||
esac
|
||||
'''
|
||||
STUB_DOCKER = '#!/usr/bin/env bash\necho "docker $*" >> "$STUB/calls"\n'
|
||||
STUB_CURL = r'''#!/usr/bin/env bash
|
||||
echo "curl $*" >> "$STUB/calls"
|
||||
for a in "$@"; do case "$a" in @*) cat "${a#@}" >> "$STUB/headers" ;; esac; done
|
||||
cat "$STUB/response"
|
||||
'''
|
||||
|
||||
|
||||
def _exe(path: Path, body: str):
|
||||
path.write_text(body)
|
||||
path.chmod(path.stat().st_mode | stat.S_IEXEC)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def env(tmp_path):
|
||||
stub = tmp_path / "stub"
|
||||
bindir = stub / "bin"
|
||||
bindir.mkdir(parents=True)
|
||||
_exe(bindir / "git", STUB_GIT)
|
||||
_exe(bindir / "docker", STUB_DOCKER)
|
||||
_exe(bindir / "curl", STUB_CURL)
|
||||
(stub / "head").write_text(OLD + "\n")
|
||||
(stub / "calls").write_text("")
|
||||
(stub / "response").write_text(json.dumps({"update_requested_at": None}))
|
||||
|
||||
inst = tmp_path / "node"
|
||||
(inst / ".git").mkdir(parents=True)
|
||||
(inst / "configs").mkdir()
|
||||
(inst / ".env").write_text(
|
||||
"NODE_ID=node-7\nC2_URL=https://api.example.net\nIMAGE_REGISTRY=git.example.net\n"
|
||||
)
|
||||
(inst / "configs" / "credentials.json").write_text(json.dumps({"api_key": "k" * 64}))
|
||||
(inst / "configs" / "pickup_secret").write_text("p" * 48)
|
||||
|
||||
conf = tmp_path / "updater.conf"
|
||||
conf.write_text(f"INSTALL_DIR={inst}\nREPO_URL={REPO}\nREF=v1\nREGISTRY=git.example.net\nBUILD=0\n")
|
||||
|
||||
e = dict(os.environ)
|
||||
e.update({
|
||||
"PATH": f"{bindir}:{os.environ['PATH']}",
|
||||
"STUB": str(stub), "STUB_NEW": NEW, "STUB_ORIGIN": REPO,
|
||||
"DRB_UPDATER_CONF": str(conf),
|
||||
"DRB_UPDATER_STATE": str(tmp_path / "state"),
|
||||
"DRB_UPDATER_LOCK": str(tmp_path / "lock"),
|
||||
})
|
||||
return {"env": e, "stub": stub, "inst": inst, "conf": conf, "state": tmp_path / "state"}
|
||||
|
||||
|
||||
def _run(env):
|
||||
return subprocess.run(["bash", str(SCRIPT)], env=env["env"], capture_output=True, text=True, timeout=30)
|
||||
|
||||
|
||||
def _calls(env):
|
||||
return (env["stub"] / "calls").read_text()
|
||||
|
||||
|
||||
def test_no_request_means_checkin_only(env):
|
||||
r = _run(env)
|
||||
assert r.returncode == 0, r.stderr
|
||||
calls = _calls(env)
|
||||
assert "update-check" in calls
|
||||
assert "docker" not in calls
|
||||
assert "fetch" not in calls
|
||||
|
||||
|
||||
def test_c2_request_updates_once_and_reports(env):
|
||||
(env["stub"] / "response").write_text(json.dumps({"update_requested_at": "2026-10-04T12:00:00+00:00"}))
|
||||
r = _run(env)
|
||||
assert r.returncode == 0, r.stderr + r.stdout
|
||||
calls = _calls(env)
|
||||
assert "docker compose pull" in calls
|
||||
assert "docker compose up -d --no-build --force-recreate --remove-orphans" in calls
|
||||
assert (env["stub"] / "head").read_text().strip() == NEW
|
||||
assert '"applied_request_at":"2026-10-04T12:00:00+00:00"' in calls
|
||||
assert '"result":"ok"' in calls
|
||||
# Same request again: nothing to do.
|
||||
(env["stub"] / "calls").write_text("")
|
||||
assert _run(env).returncode == 0
|
||||
assert "docker" not in _calls(env)
|
||||
|
||||
|
||||
def test_mqtt_sentinel_triggers_update_and_is_consumed(env):
|
||||
sentinel = env["inst"] / "configs" / "update-requested"
|
||||
sentinel.write_text('{"ref": "evil", "cmd": "rm -rf /"}')
|
||||
r = _run(env)
|
||||
assert r.returncode == 0, r.stderr
|
||||
assert not sentinel.exists()
|
||||
calls = _calls(env)
|
||||
assert "docker compose pull" in calls
|
||||
assert "evil" not in calls and "rm -rf" not in calls
|
||||
|
||||
|
||||
def test_c2_response_is_never_used_as_a_ref_or_command(env):
|
||||
(env["stub"] / "response").write_text(json.dumps({"update_requested_at": "$(touch /tmp/pwned); main"}))
|
||||
r = _run(env)
|
||||
assert r.returncode == 0
|
||||
assert "docker" not in _calls(env) # malformed request is ignored, not executed
|
||||
|
||||
|
||||
def test_refuses_when_git_origin_differs(env):
|
||||
env["env"]["STUB_ORIGIN"] = "https://attacker.example/node-26.git"
|
||||
(env["inst"] / "configs" / "update-requested").write_text("")
|
||||
r = _run(env)
|
||||
assert r.returncode != 0
|
||||
assert "refusing" in r.stderr
|
||||
assert "docker" not in _calls(env)
|
||||
|
||||
|
||||
def test_refuses_when_env_registry_differs(env):
|
||||
(env["inst"] / ".env").write_text(
|
||||
"NODE_ID=node-7\nC2_URL=https://api.example.net\nIMAGE_REGISTRY=evil.example\n"
|
||||
)
|
||||
(env["inst"] / "configs" / "update-requested").write_text("")
|
||||
r = _run(env)
|
||||
assert r.returncode != 0
|
||||
assert "docker compose pull" not in _calls(env)
|
||||
|
||||
|
||||
def test_non_https_repo_in_conf_is_rejected(env):
|
||||
env["conf"].write_text(env["conf"].read_text().replace("https://", "http://"))
|
||||
r = _run(env)
|
||||
assert r.returncode != 0
|
||||
assert "git" not in _calls(env)
|
||||
|
||||
|
||||
def test_credentials_go_in_a_header_file_never_argv(env):
|
||||
_run(env)
|
||||
calls = _calls(env)
|
||||
assert "k" * 64 not in calls and "p" * 48 not in calls
|
||||
headers = (env["stub"] / "headers").read_text()
|
||||
assert f"Authorization: Bearer {'k' * 64}" in headers
|
||||
assert f"X-Pickup-Secret: {'p' * 48}" in headers
|
||||
|
||||
|
||||
def test_plain_http_c2_skips_checkin(env):
|
||||
(env["inst"] / ".env").write_text("NODE_ID=node-7\nC2_URL=http://api.example.net\nIMAGE_REGISTRY=git.example.net\n")
|
||||
r = _run(env)
|
||||
assert r.returncode == 0
|
||||
assert "curl" not in _calls(env)
|
||||
Reference in New Issue
Block a user