Node self-heals its API key; one-stop install.sh; real host-level OTA updater (#14, #15)

install.sh (#14): after enrolling it now BLOCKS until an admin approves, printing "Please have an admin approve node '<id>' at <app>/settings/nodes to continue", polling GET /nodes/{id}/credentials with the pickup secret (no default timeout; Ctrl-C safe; re-run resumes from configs/pickup_secret). A re-run with a key on disk re-checks it against C2 and replaces a stale (reissued) key. credentials.json is merged atomically so the dashboard auth salt/secret survive. edge-node is ALWAYS force-recreated at the end, so a key written after the container started is loaded. --wait-approval N caps the wait; 0 = don't wait. git fetch uses --force so a re-cut v1 tag is accepted. edge-node (#14): with no key at startup, or on a broker auth refusal (CONNACK 4/5/134/135), mqtt_manager re-reads credentials.json, else fetches the key over HTTPS (key_pickup.py, pickup secret from /configs), persists it and rebuilds the paho client in place. Single-flight, 30s..15min backoff. A key pushed over MQTT while connected is applied for the next reconnect. OTA (#15): node_update no longer os._exit()s. It drops configs/update-requested; install.sh installs scripts/install-updater.sh's drb-update.{service,timer,path}. The path unit fires on the sentinel, the timer (~15 min) does an HTTPS check-in (POST /nodes/{id}/update-check, server-26 PR) so offline/locked-out nodes still update. drb-update.sh only installs the repo/ref/registry in root-owned /etc/drb/updater.conf; C2's answer is just a timestamp; origin and registry mismatches abort; creds go to curl via a 0600 header file. Tests: test_credential_recovery.py, test_host_updater.py (runs the real updater script with stub git/docker/curl). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 01:09:39 -04:00
parent 6bf2c9dd93
commit c2c349043b
10 changed files with 1057 additions and 74 deletions
@@ -0,0 +1,205 @@
+"""
+node-26#14 — a node with no key, or a stale key, recovers on its own:
+re-read credentials.json, else fetch over HTTPS with the pickup secret, then
+rebuild the MQTT client in place. node-26#15 — node_update hands off to the
+host updater via a sentinel file.
+"""
+import asyncio
+import json
+from unittest.mock import AsyncMock
+
+import httpx
+import pytest
+
+from app.config import settings
+from app.internal import credentials, key_pickup, mqtt_manager as mm_mod, update_request
+from app.internal.mqtt_manager import MQTTManager, _is_auth_failure
+
+
+@pytest.fixture(autouse=True)
+def isolated(tmp_path, monkeypatch):
+    monkeypatch.setattr(credentials, "_CREDS_FILE", tmp_path / "credentials.json")
+    monkeypatch.setattr(credentials, "_PICKUP_FILE", tmp_path / "pickup_secret")
+    monkeypatch.setattr(credentials, "_api_key", None)
+    monkeypatch.setattr(settings, "config_path", str(tmp_path))
+    monkeypatch.setattr(settings, "c2_url", "https://c2.example")
+    monkeypatch.setattr(settings, "mqtt_user", None)
+    monkeypatch.setattr(mm_mod, "RECOVERY_MIN_DELAY", 0)
+    yield tmp_path
+
+
+def _manager(client_key=None):
+    m = MQTTManager()
+    m._client_key = client_key
+    m._rebuild_client = AsyncMock()
+    return m
+
+
+# --- credentials.py -----------------------------------------------------------
+
+def test_reload_picks_up_a_key_written_after_startup(isolated):
+    credentials.load()
+    assert credentials.get_api_key() is None
+    data = json.loads((isolated / "credentials.json").read_text())
+    data["api_key"] = "written-by-install-sh"
+    (isolated / "credentials.json").write_text(json.dumps(data))
+    assert credentials.reload_api_key() == "written-by-install-sh"
+
+
+def test_write_is_atomic_and_private(isolated):
+    credentials.save_api_key("k1")
+    f = isolated / "credentials.json"
+    assert json.loads(f.read_text())["api_key"] == "k1"
+    assert (f.stat().st_mode & 0o777) == 0o600
+    assert not (isolated / "credentials.json.tmp").exists()
+
+
+def test_pickup_secret_read_and_absent(isolated):
+    assert credentials.get_pickup_secret() is None
+    (isolated / "pickup_secret").write_text("abc123\n")
+    assert credentials.get_pickup_secret() == "abc123"
+
+
+# --- auth-failure classification -------------------------------------------------
+
+@pytest.mark.parametrize("code", [4, 5, 134, 135])
+def test_auth_failure_codes(code):
+    assert _is_auth_failure(code)
+
+
+def test_non_auth_failures_do_not_trigger_recovery():
+    assert not _is_auth_failure(3)  # server unavailable
+    assert not _is_auth_failure(136)
+
+
+def test_paho_reason_code_objects_classified():
+    from paho.mqtt.reasoncodes import ReasonCode
+    from paho.mqtt.packettypes import PacketTypes
+    assert _is_auth_failure(ReasonCode(PacketTypes.CONNACK, "Not authorized"))
+    assert _is_auth_failure(ReasonCode(PacketTypes.CONNACK, "Bad user name or password"))
+    assert not _is_auth_failure(ReasonCode(PacketTypes.CONNACK, "Server unavailable"))
+
+
+# --- recovery loop ----------------------------------------------------------------
+
+async def test_recovery_uses_key_already_on_disk_without_calling_c2(isolated, monkeypatch):
+    credentials.save_api_key("disk-key")
+    monkeypatch.setattr(credentials, "_api_key", None)  # process started before the write
+    fetch = AsyncMock()
+    monkeypatch.setattr(key_pickup, "fetch_api_key", fetch)
+    m = _manager(client_key=None)
+    await m._recover_credentials()
+    m._rebuild_client.assert_awaited_once()
+    fetch.assert_not_awaited()
+
+
+async def test_recovery_fetches_over_https_and_persists(isolated, monkeypatch):
+    monkeypatch.setattr(key_pickup, "fetch_api_key", AsyncMock(return_value="fresh-key"))
+    m = _manager(client_key="stale-key")
+    monkeypatch.setattr(credentials, "_api_key", "stale-key")
+    await m._recover_credentials()
+    m._rebuild_client.assert_awaited_once()
+    assert credentials.get_api_key() == "fresh-key"
+    assert json.loads((isolated / "credentials.json").read_text())["api_key"] == "fresh-key"
+
+
+async def test_recovery_backs_off_until_approved(isolated, monkeypatch):
+    fetch = AsyncMock(side_effect=[None, None, "approved-key"])
+    monkeypatch.setattr(key_pickup, "fetch_api_key", fetch)
+    m = _manager()
+    await asyncio.wait_for(m._recover_credentials(), timeout=5)
+    assert fetch.await_count == 3
+    m._rebuild_client.assert_awaited_once()
+
+
+async def test_recovery_stops_if_connection_recovers_meanwhile(isolated, monkeypatch):
+    monkeypatch.setattr(key_pickup, "fetch_api_key", AsyncMock(return_value=None))
+    m = _manager()
+    m._connected = True
+    await asyncio.wait_for(m._recover_credentials(), timeout=5)
+    m._rebuild_client.assert_not_awaited()
+
+
+async def test_start_recovery_is_single_flight(isolated, monkeypatch):
+    gate = asyncio.Event()
+
+    async def slow_fetch():
+        await gate.wait()
+        return "k"
+    monkeypatch.setattr(key_pickup, "fetch_api_key", slow_fetch)
+    m = _manager()
+    m._start_recovery("a")
+    first = m._recovery_task
+    m._start_recovery("b")
+    assert m._recovery_task is first
+    gate.set()
+    await first
+
+
+async def test_connect_without_key_starts_recovery(isolated, monkeypatch):
+    m = MQTTManager()
+    started = []
+    monkeypatch.setattr(m, "_connect_with_retry", AsyncMock())
+    monkeypatch.setattr(m, "_start_recovery", lambda why: started.append(why))
+    await m.connect()
+    assert started == ["no API key at startup"]
+
+
+def test_update_api_key_changes_creds_for_next_reconnect(isolated):
+    m = MQTTManager()
+    m._client = m._build_client()
+    m.update_api_key("rotated")
+    assert m._client._password == b"rotated"
+    assert m._client_key == "rotated"
+
+
+# --- key_pickup.fetch_api_key -----------------------------------------------------
+
+def _mock_http(monkeypatch, handler):
+    real = httpx.AsyncClient
+
+    def factory(*a, **kw):
+        kw["transport"] = httpx.MockTransport(handler)
+        return real(*a, **kw)
+    monkeypatch.setattr(key_pickup.httpx, "AsyncClient", factory)
+
+
+async def test_fetch_sends_pickup_secret_and_returns_key(isolated, monkeypatch):
+    (isolated / "pickup_secret").write_text("the-secret")
+    seen = {}
+
+    def handler(request):
+        seen["url"] = str(request.url)
+        seen["secret"] = request.headers.get("X-Pickup-Secret")
+        return httpx.Response(200, json={"approval_status": "approved", "api_key": "K"})
+    _mock_http(monkeypatch, handler)
+    assert await key_pickup.fetch_api_key() == "K"
+    assert seen == {"url": f"https://c2.example/nodes/{settings.node_id}/credentials", "secret": "the-secret"}
+
+
+@pytest.mark.parametrize("resp", [
+    httpx.Response(200, json={"approval_status": "pending", "api_key": None}),
+    httpx.Response(401, json={"detail": "Invalid pickup secret"}),
+    httpx.Response(404),
+    httpx.Response(500),
+    httpx.Response(200, text="not json"),
+])
+async def test_fetch_returns_none_when_no_key(isolated, monkeypatch, resp):
+    (isolated / "pickup_secret").write_text("s")
+    _mock_http(monkeypatch, lambda request: resp)
+    assert await key_pickup.fetch_api_key() is None
+
+
+async def test_fetch_without_pickup_secret_does_not_call_c2(isolated, monkeypatch):
+    def handler(request):
+        raise AssertionError("must not call C2 without a pickup secret")
+    _mock_http(monkeypatch, handler)
+    assert await key_pickup.fetch_api_key() is None
+
+
+# --- node_update -> host sentinel ---------------------------------------------------
+
+def test_node_update_writes_sentinel(isolated):
+    assert update_request.request_host_update() is True
+    data = json.loads((isolated / "update-requested").read_text())
+    assert "requested_at" in data