diff --git a/.env.example b/.env.example index 40f8613..513d3cf 100644 --- a/.env.example +++ b/.env.example @@ -7,11 +7,20 @@ # password file. Use different values in production — do NOT reuse defaults. # ----------------------------------------------------------------------- -# C2-core service account (full broker access) +# C2-core service account (full broker access via the "c2core" dynsec role) MQTT_C2_USER=drb-c2-core MQTT_C2_PASS=change-me-c2 -# Shared credential for all edge nodes (ACL scopes each node to its own -# nodes//# namespace via the MQTT client ID) -MQTT_NODE_USER=drb-node -MQTT_NODE_PASS=change-me-node +# Seeds mosquitto's built-in dynamic-security plugin's one-time "admin" +# bootstrap client on first boot (read directly by mosquitto, no entrypoint +# scripting involved). Must be >=12 chars. c2-core needs this SAME value as +# MQTT_DYNSEC_ADMIN_PASS in drb-c2-core/.env to log in as "admin" and +# administer node credentials — see app/internal/dynsec.py. +MOSQUITTO_DYNSEC_PASSWORD=change-me-dynsec-admin-min-12-chars + +# There is no shared node credential anymore. Each node authenticates as +# username=, password= — checked by +# mosquitto's dynamic-security plugin (not an HTTP backend — that was an +# earlier, since-rejected design using the now-archived mosquitto-go-auth). +# Nodes obtain that key via the enrollment flow — see ENROLLMENT_TOKEN in +# drb-c2-core/.env.example. diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 4c0a1d9..e2a4174 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -68,15 +68,21 @@ jobs: runs-on: ubuntu-latest steps: + - name: Check runner outbound IP + run: curl -s ifconfig.me + - name: Write SSH key run: | - echo "${{ secrets.SSH_PRIVATE_KEY }}" > /tmp/deploy_key + printf '%s\n' "${{ secrets.SSH_PRIVATE_KEY }}" > /tmp/deploy_key chmod 600 /tmp/deploy_key + ssh-keygen -l -f /tmp/deploy_key - name: Deploy run: | ssh -o StrictHostKeyChecking=no \ -o HostKeyAlgorithms=ssh-ed25519,rsa-sha2-256,rsa-sha2-512 \ + -o ConnectTimeout=15 \ + -v \ -i /tmp/deploy_key \ drb@${{ secrets.SERVER_IP }} << 'ENDSSH' set -e diff --git a/.gitignore b/.gitignore index 6583778..2a950b2 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,10 @@ infra/terraform.tfvars infra/tf.log infra/ansible/inventory.ini infra/ansible/group_vars/all.yml -infra/ansible/vault.yml +# Glob, not the bare filename: ansible-vault edit and manual backups leave +# siblings like vault.yml.locked.bak, which the exact-name rule left untracked +# but committable. +infra/ansible/vault.yml* # Python __pycache__/ diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 765a627..74c422d 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -8,9 +8,27 @@ # - restart: always (instead of unless-stopped) for hard reboots. services: + # ports AND volumes both need !override here, not !reset/a plain list — + # compose merges list-type fields by APPENDING across -f files. A plain + # list (or !reset on volumes) would leave dev's mosquitto_certs named + # volume mounted at /mosquitto/certs alongside this bind mount, and two + # mounts targeting the same path is exactly the "address already in use"- + # style footgun the c2-core override below already hit once with ports. + # mosquitto-data is now a host bind mount too (not just certs) — it holds + # dynamic-security.json, the broker's only record of node credentials + # (see app/internal/dynsec.py "TWO-SOURCES-OF-TRUTH"). A named Docker + # volume already survives normal redeploys (git pull && compose pull && + # up -d never passes -v), but the bind mount makes it inspectable/ + # backupable the same way the cert directory already is. NOT read-only — + # mosquitto writes dynamic-security.json here. mosquitto: restart: always - ports: !reset [] # Remove the dev 1883:1883 mapping — internal only + ports: !override + - "8883:8883" # TLS only, published. 1883 stays internal (docker bridge, c2-core's own login). + volumes: !override + - ./drb-c2-core/mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro + - /opt/drb/mosquitto-data:/mosquitto/data + - /opt/drb/mosquitto-certs:/mosquitto/certs:ro # fed by the cert-sync systemd unit, see infra/ansible # !override, not a plain list: compose MERGES `ports` by appending, so a plain # list leaves the base file's "8888:8000" in place alongside this one. The diff --git a/docker-compose.yml b/docker-compose.yml index 3509f53..ffa4424 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,20 +1,23 @@ services: + # Auth is mosquitto's own built-in dynamic-security plugin (see + # mosquitto.conf + app/internal/dynsec.py) — NOT mosquitto-go-auth, that + # project is archived upstream (no CVE patches), rejected for a + # public-internet broker. Stock official image, pinned to an exact patch + # (not the floating `:2` tag). MOSQUITTO_DYNSEC_PASSWORD seeds the + # plugin's own one-time "admin" bootstrap client on first boot — read + # directly by the plugin's C code, no entrypoint scripting needed for it. mosquitto: - image: eclipse-mosquitto:2 + image: eclipse-mosquitto:2.1.2-alpine restart: unless-stopped ports: - "1883:1883" - entrypoint: ["/bin/sh", "/mosquitto/config/entrypoint.sh"] + - "8883:8883" environment: - - MQTT_C2_USER=${MQTT_C2_USER} - - MQTT_C2_PASS=${MQTT_C2_PASS} - - MQTT_NODE_USER=${MQTT_NODE_USER} - - MQTT_NODE_PASS=${MQTT_NODE_PASS} + - MOSQUITTO_DYNSEC_PASSWORD=${MOSQUITTO_DYNSEC_PASSWORD} volumes: - ./drb-c2-core/mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro - - ./drb-c2-core/mosquitto/acl.conf:/mosquitto/config/acl.conf:ro - - ./drb-c2-core/mosquitto/entrypoint.sh:/mosquitto/config/entrypoint.sh:ro - mosquitto_data:/mosquitto/data + - mosquitto_certs:/mosquitto/certs c2-core: image: ${REGISTRY}/c2-core:${TAG:-latest} @@ -45,4 +48,11 @@ services: - c2-core volumes: + # Dev only for both. Prod overrides these to host bind mounts + # (/opt/drb/mosquitto-data, /opt/drb/mosquitto-certs — the latter fed by + # the Caddy cert-sync systemd unit) — see docker-compose.prod.yml and + # infra/ansible/roles/deploy/templates/. mosquitto_data holds + # dynamic-security.json (node MQTT credentials, see app/internal/dynsec.py) + # as well as the usual broker persistence state. mosquitto_data: + mosquitto_certs: diff --git a/drb-c2-core/.env.example b/drb-c2-core/.env.example index c7fbc8a..7e71827 100644 --- a/drb-c2-core/.env.example +++ b/drb-c2-core/.env.example @@ -2,10 +2,15 @@ MQTT_BROKER=mosquitto MQTT_PORT=1883 # Use the c2-core credential — must match MQTT_C2_USER/MQTT_C2_PASS in the -# top-level .env (which is passed to the mosquitto entrypoint) +# top-level .env MQTT_USER=drb-c2-core MQTT_PASS=change-me-c2 +# Same value as the top-level .env's MOSQUITTO_DYNSEC_PASSWORD — lets +# c2-core log in as mosquitto's built-in dynsec "admin" client to +# administer node MQTT credentials. See app/internal/dynsec.py. +MQTT_DYNSEC_ADMIN_PASS=change-me-dynsec-admin-min-12-chars + # GCP — path to service account JSON inside the container GCP_CREDENTIALS_PATH=/app/gcp-key.json @@ -28,6 +33,7 @@ SUMMARY_INTERVAL_MINUTES=15 CORRELATION_WINDOW_HOURS=4 EMBEDDING_SIMILARITY_THRESHOLD=0.82 -# Auth — static key that edge nodes send as Bearer token on /upload +# Fleet-wide token edge nodes present as X-Enrollment-Token on first boot +# (POST /nodes/enroll). Shared across every node — NOT a per-node secret. # Generate with: openssl rand -hex 32 -NODE_API_KEY= +ENROLLMENT_TOKEN= diff --git a/drb-c2-core/app/config.py b/drb-c2-core/app/config.py index 5284058..bd9795b 100644 --- a/drb-c2-core/app/config.py +++ b/drb-c2-core/app/config.py @@ -9,6 +9,15 @@ class Settings(BaseSettings): mqtt_user: Optional[str] = None mqtt_pass: Optional[str] = None + # mosquitto's built-in dynamic-security plugin (see app/internal/dynsec.py). + # "admin" is hardcoded by the plugin itself on first boot — not actually + # configurable — kept as a named setting rather than a literal for + # readability. mqtt_dynsec_admin_pass must equal the mosquitto + # container's own MOSQUITTO_DYNSEC_PASSWORD env var (root .env / + # root.env.j2) or c2-core can't administer node credentials at all. + mqtt_dynsec_admin_user: str = "admin" + mqtt_dynsec_admin_pass: Optional[str] = None + # GCP gcp_credentials_path: Optional[str] = None # None → uses ADC gcs_bucket: Optional[str] = None # None → audio upload disabled @@ -51,6 +60,11 @@ class Settings(BaseSettings): # Internal service key — allows server-side services (discord bot) to call C2 without Firebase service_key: Optional[str] = None + # Fleet-wide token edge nodes present to POST /nodes/enroll on first boot. + # Not a per-node secret — see routers/enrollment.py for why a leaked copy + # of this alone can't steal an already-approved node's key. + enrollment_token: Optional[str] = None + # Upload size limit — reject audio files larger than this (bytes). Default 100 MB. upload_max_bytes: int = 100 * 1024 * 1024 diff --git a/drb-c2-core/app/internal/dynsec.py b/drb-c2-core/app/internal/dynsec.py new file mode 100644 index 0000000..7b1d93f --- /dev/null +++ b/drb-c2-core/app/internal/dynsec.py @@ -0,0 +1,341 @@ +""" +Client for mosquitto's built-in dynamic-security plugin. + +WHY THIS EXISTS: MQTT-PUBLIC-AUTH-PLAN.md originally specced the +mosquitto-go-auth plugin (HTTP backend). That project was archived by its +maintainer 2025-08-06 ("no more changes") — unacceptable for a broker that's +about to be reachable from the public internet, no way to get a CVE fix. +Replaced with mosquitto 2.x's own `dynamic-security` plugin, which ships in +and is maintained alongside the official eclipse-mosquitto image itself. + +HOW DYNSEC WORKS (verified against plugin source on +github.com/eclipse-mosquitto/mosquitto, 2026-08-16 — see citations inline; +NOT verified by running anything, per instruction not to execute/deploy +anything from this machine): + + - The broker persists clients/roles/ACLs in a JSON file at + `plugin_opt_config_file` (we point this at /mosquitto/data/, the same + volume `persistence_location` already uses — one durable volume for all + broker state, see docker-compose.yml). + - Admin commands are plain MQTT publishes: JSON `{"commands": [...]}` to + `$CONTROL/dynamic-security/v1` (source: plugin.c, + `mosquitto_callback_register(plg_id, MOSQ_EVT_CONTROL, + dynsec_control_callback, "$CONTROL/dynamic-security/v1", ...)`). + Replies come back on `$CONTROL/dynamic-security/v1/response` + (source: control.c, `#define RESPONSE_TOPIC + "$CONTROL/dynamic-security/v1/response"`). + - Per-command JSON fields (verified against clients.c / roles.c handlers + and the plugin README): + createClient: username, password, clientid, textname, textdescription, + roles: [{rolename, priority}], groups: [...] + modifyClient: same fields, username identifies the existing client + deleteClient: username + createRole: rolename, textname, textdescription, + acls: [{acltype, topic, priority, allow}] + acltype values: publishClientSend, publishClientReceive, + subscribeLiteral, subscribePattern, unsubscribeLiteral, + unsubscribePattern. %u (username) and %c (clientid) are valid + substitutions in `topic` for every type except the two *Literal ones. + - On first boot, if `plugin_opt_config_file` doesn't exist, the plugin + bootstraps itself (config_init.c): reads env var + `MOSQUITTO_DYNSEC_PASSWORD` (or `plugin_opt_password_init_file`) and + creates a client literally named "admin" (hardcoded string, NOT + configurable — verified in config_init.c's `client_add_admin()`) with + three roles: `super-admin` (full pub/sub on `$CONTROL/#` — i.e. this is + what makes a client capable of issuing further dynsec commands, and + it's an ordinary role, nothing hardcoded beyond the initial grant), + `sys-observe` ($SYS/# read-only), `topic-observe` (# read-only, NOT + read-write). If MOSQUITTO_DYNSEC_PASSWORD is set (we always set it), + no `democlient` demo account gets created — that only happens in the + "no password provided, generate one randomly" path. + - This is a genuine backend swap, not just config: `allow_anonymous + false` plus the *absence* of `password_file`/`acl_file` directives + means dynsec is the only auth backend registered — nothing else is + there to conflict with it. (Inferred from plugin architecture — every + mosquitto auth backend, built-in or plugin, registers the same + basic-auth/ACL callback hooks; there's no "layering" mechanism, so + without password_file/acl_file directives there is nothing else to + check credentials or topics.) + +WHAT COULD NOT BE VERIFIED (see also the plan doc + final report): + - The exact JSON envelope of a *response* message (only individual + command outcomes were confirmed: `mosquitto_control_command_reply(cmd, + NULL)` for success, `mosquitto_control_command_reply(cmd, "error + string")` for failure — the wrapping object shape, e.g. whether it's + `{"responses": [{"command": ..., "error": ...}]}`, was not directly + read from source). This client parses defensively: it treats ANY + dict containing a non-null "error"/"Error" key anywhere in the + top-level response payload as failure, presence of "already exists" in + that string as an idempotent success, and a response with no such key + within the timeout as success. A response timeout is always a hard + failure (never assumed to mean success). + - "Client already exists" was confirmed verbatim as createClient's + error string; "already exists" for createRole is assumed analogous, + not directly confirmed. + +TWO-SOURCES-OF-TRUTH: Firestore's `node_keys` collection is the source of +truth for node credentials (nothing changes there); dynamic-security.json +is a derived cache the broker uses to authenticate. `reconcile_all()` +rebuilds every approved node's dynsec client from Firestore and is called +on every c2-core startup — so a lost/corrupted dynamic-security.json (e.g. +volume wiped) self-heals on the next restart instead of silently locking +out every node. `upsert_node_client()`/`delete_node_client()` are also +called synchronously from routers/nodes.py's approve/reissue/delete +handlers and raise on failure — those endpoints now fail loudly (502) +instead of updating Firestore while dynsec silently didn't get the memo. +""" +import asyncio +import json +import time +import uuid +import paho.mqtt.client as mqtt +from app.config import settings +from app.internal.logger import logger +from app.internal import firestore as fstore + +CONTROL_TOPIC = "$CONTROL/dynamic-security/v1" +RESPONSE_TOPIC = "$CONTROL/dynamic-security/v1/response" +_RESPONSE_TIMEOUT_SECONDS = 10 + +# Role every approved node's dynsec client is attached to. %u = the +# authenticated username (the node_id) — this is the fixed version of the +# old `pattern readwrite nodes/%c/#`, where %c was the client-supplied, +# spoofable client ID. +NODE_ROLE = "node" +# Role c2-core's own login gets, in addition to being handed the plugin's +# built-in `super-admin` role (see grant_c2core_admin()). Mirrors the old +# `topic readwrite #` superuser line. +C2CORE_ROLE = "c2core" + + +class DynsecError(Exception): + """A dynsec command was rejected, or no response arrived in time.""" + + +def _run_commands_sync(commands: list[dict], username: str, password: str) -> list[dict]: + """ + Blocking: open a short-lived MQTT connection, publish one or more dynsec + commands, wait for the matching replies, disconnect. Always called via + asyncio.to_thread — see the async wrappers below. A fresh connection per + call (rather than reusing mqtt_handler's long-lived client) keeps this + request/response exchange simple and isolated from that client's + async-callback-driven subscribe state. + """ + responses: list[dict] = [] + done = {"got": False, "error": None} + + def _on_connect(client, userdata, flags, reason_code, properties): + if reason_code != 0: + done["error"] = f"connect refused: {reason_code}" + done["got"] = True + return + client.subscribe(RESPONSE_TOPIC, qos=1) + client.publish(CONTROL_TOPIC, json.dumps({"commands": commands}), qos=1) + + def _on_message(client, userdata, msg): + try: + payload = json.loads(msg.payload.decode()) + except Exception: + return + responses.append(payload) + done["got"] = True + + client = mqtt.Client( + callback_api_version=mqtt.CallbackAPIVersion.VERSION2, + client_id=f"drb-c2-core-dynsec-{uuid.uuid4().hex[:8]}", + ) + client.username_pw_set(username, password) + client.on_connect = _on_connect + client.on_message = _on_message + + try: + client.connect(settings.mqtt_broker, settings.mqtt_port, keepalive=30) + except Exception as e: + raise DynsecError(f"could not connect to mosquitto for dynsec command: {e}") + + client.loop_start() + deadline = time.monotonic() + _RESPONSE_TIMEOUT_SECONDS + try: + while not done["got"] and time.monotonic() < deadline: + time.sleep(0.05) + finally: + client.loop_stop() + client.disconnect() + + if done["error"]: + raise DynsecError(str(done["error"])) + if not responses: + raise DynsecError( + f"no response on {RESPONSE_TOPIC} within {_RESPONSE_TIMEOUT_SECONDS}s for commands: " + f"{[c.get('command') for c in commands]}" + ) + return responses + + +def _check_responses_ok(responses: list[dict], tolerate_already_exists: bool = False) -> None: + """Raise DynsecError unless every response payload is error-free (or, + when tolerate_already_exists, only contains an 'already exists'-style + error — see the module docstring's "could not verify" note on why this + is a substring match rather than a structured error code check).""" + for payload in responses: + # Defensive: walk the payload looking for any *-cased "error" key + # with a non-empty value, since the exact envelope shape wasn't + # confirmed from source. Covers both a flat {"error": "..."} and a + # {"responses": [{"error": "..."}]}-style wrapper. + errors = _find_error_strings(payload) + for err in errors: + if tolerate_already_exists and "already exist" in err.lower(): + continue + raise DynsecError(f"dynsec command failed: {err}") + + +def _find_error_strings(obj) -> list[str]: + found = [] + if isinstance(obj, dict): + for k, v in obj.items(): + if k.lower() == "error" and v: + found.append(str(v)) + else: + found.extend(_find_error_strings(v)) + elif isinstance(obj, list): + for item in obj: + found.extend(_find_error_strings(item)) + return found + + +# --------------------------------------------------------------------------- +# Async wrappers (all real work happens in the thread pool) +# --------------------------------------------------------------------------- + +async def _admin_publish(commands: list[dict], tolerate_already_exists: bool = False) -> list[dict]: + if not settings.mqtt_dynsec_admin_pass: + raise DynsecError("MQTT_DYNSEC_ADMIN_PASS / mqtt_dynsec_admin_pass is not configured") + responses = await asyncio.to_thread( + _run_commands_sync, commands, settings.mqtt_dynsec_admin_user, settings.mqtt_dynsec_admin_pass + ) + _check_responses_ok(responses, tolerate_already_exists=tolerate_already_exists) + return responses + + +async def ensure_roles_and_c2core_grant() -> None: + """ + Idempotent, safe to run on every startup: + 1. createRole "node" — nodes/%u/# publish+subscribe (both directions) + 2. createRole "c2core" — full "#" publish+subscribe, same reach the + old `topic readwrite #` superuser line gave c2-core + 3. createClient/modifyClient drb-c2-core (settings.mqtt_user) with + BOTH roles above AND the plugin's built-in "super-admin" role — + i.e. c2-core's existing login is handed the actual dynsec admin + role, not a separate identity, per the design decision. + Runs over the dedicated "admin" bootstrap login (step 3 assigns + super-admin to c2-core's own login for the record / future use, but + THIS module still authenticates its own ongoing calls as "admin" — see + the module docstring for why: it's the one identity guaranteed by + mosquitto's own source to hold super-admin, so control-plane calls + don't depend on step 3's grant having actually landed). + """ + node_acl_types = ["publishClientSend", "publishClientReceive", "subscribePattern", "unsubscribePattern"] + await _admin_publish([{ + "command": "createRole", + "rolename": NODE_ROLE, + "textname": "DRB edge node — own namespace only", + "acls": [{"acltype": t, "topic": "nodes/%u/#", "priority": 0, "allow": True} for t in node_acl_types], + }], tolerate_already_exists=True) + + c2core_acl_types = ["publishClientSend", "publishClientReceive", "subscribePattern", "unsubscribePattern"] + await _admin_publish([{ + "command": "createRole", + "rolename": C2CORE_ROLE, + "textname": "DRB c2-core — full broker access", + "acls": [{"acltype": t, "topic": "#", "priority": 0, "allow": True} for t in c2core_acl_types], + }], tolerate_already_exists=True) + + if not settings.mqtt_user or not settings.mqtt_pass: + logger.warning("dynsec: MQTT_USER/MQTT_PASS not configured — skipping c2-core client grant") + return + + roles = [{"rolename": C2CORE_ROLE, "priority": 1}, {"rolename": "super-admin", "priority": 2}] + try: + await _admin_publish([{ + "command": "createClient", + "username": settings.mqtt_user, + "password": settings.mqtt_pass, + "roles": roles, + }]) + logger.info(f"dynsec: created client {settings.mqtt_user!r} with roles {C2CORE_ROLE}, super-admin") + except DynsecError as e: + if "already exist" in str(e).lower(): + await _admin_publish([{ + "command": "modifyClient", + "username": settings.mqtt_user, + "password": settings.mqtt_pass, + "roles": roles, + }]) + logger.info(f"dynsec: updated existing client {settings.mqtt_user!r} with roles {C2CORE_ROLE}, super-admin") + else: + raise + + +async def upsert_node_client(node_id: str, api_key: str) -> None: + """Create or update a node's dynsec client — called from + routers/nodes.py approve_node()/reissue_node_key(), and from + reconcile_all() on startup. Raises DynsecError on failure; callers + must not write Firestore as if this succeeded when it didn't.""" + try: + await _admin_publish([{ + "command": "createClient", + "username": node_id, + "password": api_key, + "roles": [{"rolename": NODE_ROLE, "priority": 1}], + }]) + except DynsecError as e: + if "already exist" not in str(e).lower(): + raise + await _admin_publish([{ + "command": "modifyClient", + "username": node_id, + "password": api_key, + "roles": [{"rolename": NODE_ROLE, "priority": 1}], + }]) + + +async def delete_node_client(node_id: str) -> None: + """Best-effort: a node that was never enrolled in dynsec (or already + removed) is treated as already-deleted, not an error.""" + try: + await _admin_publish([{"command": "deleteClient", "username": node_id}]) + except DynsecError as e: + if "not found" not in str(e).lower(): + raise + + +async def reconcile_all() -> None: + """ + Rebuild dynsec state for every approved node from Firestore + (node_keys is the source of truth). Called once at c2-core startup, + after ensure_roles_and_c2core_grant(). Self-heals a lost/corrupted + dynamic-security.json (e.g. volume wiped, or a prior approve/reissue's + dynsec publish silently failed to persist for some other reason) — + without this, a broker restart with an intact Firestore but an empty + dynsec store would lock out every previously-approved node until + someone noticed and manually re-approved each one. + """ + nodes = await fstore.collection_list("nodes", approval_status="approved") + if not nodes: + return + ok, failed = 0, 0 + for node in nodes: + node_id = node.get("node_id") + if not node_id: + continue + key_doc = await fstore.doc_get("node_keys", node_id) + if not key_doc or not key_doc.get("api_key"): + logger.warning(f"dynsec reconcile: node {node_id!r} is approved but has no node_keys entry — skipping") + continue + try: + await upsert_node_client(node_id, key_doc["api_key"]) + ok += 1 + except DynsecError as e: + failed += 1 + logger.error(f"dynsec reconcile: failed to sync node {node_id!r}: {e}") + logger.info(f"dynsec reconcile: {ok} node(s) synced, {failed} failed") diff --git a/drb-c2-core/app/internal/mqtt_handler.py b/drb-c2-core/app/internal/mqtt_handler.py index fc8a51a..ec2e867 100644 --- a/drb-c2-core/app/internal/mqtt_handler.py +++ b/drb-c2-core/app/internal/mqtt_handler.py @@ -33,6 +33,10 @@ class MQTTHandler: client.subscribe("nodes/+/checkin", qos=1) client.subscribe("nodes/+/status", qos=1) client.subscribe("nodes/+/metadata", qos=1) + # TODO(mqtt-cutover): drop this subscribe once the enrollment/HTTP + # credentials flow (routers/enrollment.py) is stable in prod and + # node-26 (the one live node) has been migrated. See + # MQTT-PUBLIC-AUTH-PLAN.md "Rollout order" step 6. client.subscribe("nodes/+/key_request", qos=1) logger.info("MQTT connected — subscribed to node topics.") else: @@ -117,17 +121,14 @@ class MQTTHandler: # Approved but not yet configured — restore reachable status after reboot updates["status"] = "unconfigured" - node_type = payload.get("node_type") or existing.get("node_type") or "fixed" - enforce_timeout = payload.get("enforce_override_timeout") - if enforce_timeout is None: - enforce_timeout = existing.get("enforce_override_timeout", True) - - updates["node_type"] = node_type - updates["enforce_override_timeout"] = enforce_timeout - + node_type = payload.get("node_type", existing.get("node_type", "fixed")) + enforce_timeout = payload.get("enforce_override_timeout", existing.get("enforce_override_timeout", True)) is_overridden = payload.get("is_overridden", False) override_system_id = payload.get("override_system_id") + updates["node_type"] = node_type + updates["enforce_override_timeout"] = enforce_timeout + if node_type == "portable": updates["is_overridden"] = False updates["override_system_id"] = None @@ -257,6 +258,11 @@ class MQTTHandler: # ------------------------------------------------------------------ # Key request — re-deliver an existing approved key to a node that # lost its credentials (e.g. after a directory move / fresh volume) + # TODO(mqtt-cutover): remove this handler + publish_node_key() below, + # and the key_request subscribe above, in the separate post-cutover + # pass called out in MQTT-PUBLIC-AUTH-PLAN.md. Left in place for now so + # node-26 (currently live, using the shared-password MQTT path) keeps + # working until the enrollment flow has replaced it in prod. # ------------------------------------------------------------------ async def _handle_key_request(self, node_id: str): @@ -289,7 +295,10 @@ class MQTTHandler: logger.warning(f"MQTT not connected — could not push config to {node_id}") def publish_node_key(self, node_id: str, api_key: str): - """Publish the provisioned API key to the node (retained so it survives reconnects).""" + """Publish the provisioned API key to the node (retained so it survives reconnects). + TODO(mqtt-cutover): dead once nodes.py's callers switch to the HTTP + credentials poll (routers/enrollment.py) exclusively. See note above + _handle_key_request.""" topic = f"nodes/{node_id}/api_key" if self._client and self._connected: self._client.publish(topic, json.dumps({"api_key": api_key}), qos=2, retain=True) diff --git a/drb-c2-core/app/main.py b/drb-c2-core/app/main.py index 07087f9..a5bdd3a 100644 --- a/drb-c2-core/app/main.py +++ b/drb-c2-core/app/main.py @@ -11,6 +11,8 @@ from app.internal.recorrelation_sweep import recorrelation_loop from app.config import settings from app.internal.auth import require_firebase_token, require_service_or_firebase_token from app.routers import nodes, systems, calls, upload, tokens, incidents, alerts, admin, trips, places, links, users +from app.routers import enrollment +from app.internal import dynsec from app.internal import firestore as fstore @@ -36,6 +38,22 @@ async def lifespan(app: FastAPI): logger.info("DRB C2 Core starting.") await _release_orphaned_tokens() + # dynsec bootstrap + reconcile — must happen before mqtt_handler.connect() + # so that by the time the app is serving requests, c2-core's own dynsec + # client/roles exist and every already-approved node's dynsec client + # matches Firestore (see app/internal/dynsec.py "TWO-SOURCES-OF-TRUTH"). + # Non-fatal by design: if the broker or MQTT_DYNSEC_ADMIN_PASS isn't + # reachable/configured yet (e.g. first-ever deploy, mosquitto still + # starting), log loudly and keep booting rather than crash-looping + # c2-core itself — mqtt_handler.connect() below has its own retry loop + # and node approval/reissue endpoints fail loudly on their own if dynsec + # calls fail later, so nothing here is silently swallowed forever. + try: + await dynsec.ensure_roles_and_c2core_grant() + await dynsec.reconcile_all() + except dynsec.DynsecError as e: + logger.error(f"dynsec bootstrap/reconcile failed — node approval/reissue will fail until this is resolved: {e}") + await mqtt_handler.connect() sweeper_task = asyncio.create_task(sweeper_loop()) summarizer_task = asyncio.create_task(summarizer_loop()) @@ -74,6 +92,16 @@ app.include_router(upload.router) # auth is per-node, handled inline app.include_router(admin.router) # auth is per-endpoint (read: firebase, write: admin) app.include_router(users.router) # auth: admin only app.include_router(links.router) # auth is per-endpoint (generate: firebase, resolve: service key) +app.include_router(enrollment.router) # public; auth is the enrollment/pickup-secret tokens, checked inline +# NOTE: there used to be an app.routers.mqtt_auth router here (an HTTP +# backend for the mosquitto-go-auth plugin). That plugin's upstream project +# is archived (no CVE patches) and was rejected for an internet-facing +# broker — see MQTT-PUBLIC-AUTH-PLAN.md. MQTT auth is now mosquitto's own +# built-in dynamic-security plugin (app/internal/dynsec.py talks to it over +# MQTT control topics, not HTTP), so there is nothing at /internal/mqtt/* +# anymore. Caddy's Caddyfile.j2 still 404s /internal/* on api. as +# defence in depth even though nothing calls it today — cheap insurance +# against a future /internal/* route being added and forgotten there. @app.get("/health") diff --git a/drb-c2-core/app/routers/enrollment.py b/drb-c2-core/app/routers/enrollment.py new file mode 100644 index 0000000..164d322 --- /dev/null +++ b/drb-c2-core/app/routers/enrollment.py @@ -0,0 +1,184 @@ +""" +Node self-enrollment — the public replacement for the old shared-MQTT- +password flow (see MQTT-PUBLIC-AUTH-PLAN.md "Enrollment flow"). + + 1. POST /nodes/enroll (X-Enrollment-Token: ) + First-boot node upserts itself as `approval_status: pending` and gets + back a one-time pickup_secret. Only its hash is persisted. + 2. GET /nodes/{id}/credentials (X-Pickup-Secret: ) + Node polls this with backoff until an admin approves it in the + frontend (existing nodes.py approve_node() flow — unchanged, still + writes node_keys/{id}.api_key) and then reads its api_key back. + +These two endpoints are meant to be public (unlike the dynsec control-plane +traffic in app/internal/dynsec.py, which never leaves the docker-internal +MQTT bridge) — that's the whole point of moving off WireGuard-per-node. +Auth is the token headers checked inline below, not the app-wide +Firebase/service-key dependency the rest of routers/nodes.py uses. +""" +import hashlib +import secrets +import time +from typing import Optional +from fastapi import APIRouter, HTTPException, Header, Request +from pydantic import BaseModel +from app.config import settings +from app.internal import firestore as fstore +from app.internal.logger import logger + +router = APIRouter(prefix="/nodes", tags=["enrollment"]) + + +# --------------------------------------------------------------------------- +# Per-source-IP token bucket for /nodes/enroll. +# +# c2-core has no rate-limiting dependency anywhere today (see +# MQTT-PUBLIC-AUTH-PLAN.md); this is a deliberately small (~30 line) +# in-memory limiter rather than a new library. Known limitations: +# - per-process: with more than one c2-core instance, each has its own +# bucket, so real throughput is (limit x instance count). Fine today — +# there is exactly one instance. +# - resets on every restart/redeploy — not persisted anywhere. +# Good enough to blunt casual guessing of node_ids against the fleet token; +# not a substitute for a real edge/WAF rate limiter if this endpoint is +# ever seriously targeted. +# --------------------------------------------------------------------------- +class _TokenBucket: + def __init__(self, capacity: int, refill_per_sec: float): + self.capacity = capacity + self.refill_per_sec = refill_per_sec + self._buckets: dict[str, tuple[float, float]] = {} # key -> (tokens, last_refill_ts) + + def allow(self, key: str) -> bool: + now = time.monotonic() + tokens, last_ts = self._buckets.get(key, (float(self.capacity), now)) + tokens = min(self.capacity, tokens + (now - last_ts) * self.refill_per_sec) + if tokens < 1: + self._buckets[key] = (tokens, now) + return False + self._buckets[key] = (tokens - 1, now) + return True + + +# Burst of 5, refilling 1/minute — enrollment is a first-boot, once-per-node +# event, so a legitimate node never needs more than a handful of attempts. +_enroll_limiter = _TokenBucket(capacity=5, refill_per_sec=1 / 60) + + +def _hash_secret(secret: str) -> str: + return hashlib.sha256(secret.encode()).hexdigest() + + +class EnrollRequest(BaseModel): + node_id: str + name: Optional[str] = None + lat: float = 0.0 + lon: float = 0.0 + + +class EnrollResponse(BaseModel): + node_id: str + pickup_secret: str + approval_status: str + + +@router.post("/enroll", response_model=EnrollResponse) +async def enroll_node( + body: EnrollRequest, + request: Request, + x_enrollment_token: Optional[str] = Header(None), +): + client_ip = request.client.host if request.client else "unknown" + if not _enroll_limiter.allow(client_ip): + raise HTTPException(429, "Too many enrollment attempts. Try again later.") + + if not settings.enrollment_token: + raise HTTPException(503, "Enrollment is not configured on this server.") + if not x_enrollment_token or not secrets.compare_digest(x_enrollment_token, settings.enrollment_token): + logger.warning(f"Enroll 401: bad/missing enrollment token from {client_ip} for node_id={body.node_id!r}") + raise HTTPException(401, "Invalid or missing X-Enrollment-Token") + + node_id = body.node_id.strip() + if not node_id: + raise HTTPException(400, "node_id is required") + + existing = await fstore.doc_get("nodes", node_id) + + # ------------------------------------------------------------------- + # CRITICAL GUARD — do not remove or weaken this check. + # + # An already-approved node_id must NEVER get a fresh pickup_secret off + # the fleet-wide enrollment token alone. The fleet token is shared by + # every node (it ships in every node's .env / setup.sh prompt), so it + # is the credential most likely to leak. Without this guard, a leaked + # fleet token plus a guessable node_id (node-001, node-002, ...) would + # let an attacker "re-enroll" a live, already-approved node and race + # the real node to GET /nodes/{id}/credentials — stealing its actual + # api_key before the legitimate device ever asks. + # + # Recovery for an approved node goes through the existing admin-only + # POST /nodes/{id}/reissue-key instead (routers/nodes.py), which + # requires a Firebase admin token, not the fleet token. + # ------------------------------------------------------------------- + if existing and existing.get("approval_status") == "approved": + logger.warning( + f"Enroll refused: node_id={node_id!r} is already approved — " + f"refusing to issue a new pickup_secret from the fleet token alone " + f"(source_ip={client_ip})" + ) + raise HTTPException( + 403, + "Node is already approved. This endpoint cannot re-issue credentials " + "for an approved node from the enrollment token alone — use admin " + "key reissue.", + ) + + pickup_secret = secrets.token_hex(24) + doc = { + "node_id": node_id, + "name": body.name or (existing or {}).get("name") or node_id, + "lat": body.lat or (existing or {}).get("lat", 0.0), + "lon": body.lon or (existing or {}).get("lon", 0.0), + "approval_status": (existing or {}).get("approval_status", "pending"), + "pickup_secret_hash": _hash_secret(pickup_secret), + } + # approval_status stays "pending" for a brand-new node; if it's an + # existing "pending" or "rejected" node re-enrolling (e.g. lost its + # pickup_secret before an admin ever approved it), leave whatever + # status it already has rather than silently flipping "rejected" back + # to "pending" — that decision belongs to an admin, not this endpoint. + await fstore.doc_set("nodes", node_id, doc, merge=True) + logger.info(f"Node enrolled: {node_id} (status={doc['approval_status']}, source_ip={client_ip})") + + return EnrollResponse(node_id=node_id, pickup_secret=pickup_secret, approval_status=doc["approval_status"]) + + +class CredentialsResponse(BaseModel): + approval_status: str + api_key: Optional[str] = None + + +@router.get("/{node_id}/credentials", response_model=CredentialsResponse) +async def get_node_credentials(node_id: str, x_pickup_secret: Optional[str] = Header(None)): + if not x_pickup_secret: + raise HTTPException(401, "Missing X-Pickup-Secret header") + + node = await fstore.doc_get("nodes", node_id) + if not node or not node.get("pickup_secret_hash"): + raise HTTPException(404, "Unknown node, or node was never enrolled via POST /nodes/enroll") + + if not secrets.compare_digest(_hash_secret(x_pickup_secret), node["pickup_secret_hash"]): + raise HTTPException(401, "Invalid pickup secret") + + approval_status = node.get("approval_status", "pending") + if approval_status != "approved": + return CredentialsResponse(approval_status=approval_status) + + key_doc = await fstore.doc_get("node_keys", node_id) + if not key_doc or not key_doc.get("api_key"): + # Approved but no key yet — shouldn't normally happen, approve_node() + # always writes node_keys in the same call that sets approved. Treat + # it as "keep polling" rather than erroring the node's retry loop. + return CredentialsResponse(approval_status=approval_status) + + return CredentialsResponse(approval_status=approval_status, api_key=key_doc["api_key"]) diff --git a/drb-c2-core/app/routers/nodes.py b/drb-c2-core/app/routers/nodes.py index 99422d3..764773d 100644 --- a/drb-c2-core/app/routers/nodes.py +++ b/drb-c2-core/app/routers/nodes.py @@ -5,6 +5,8 @@ from pydantic import BaseModel from app.models import CommandPayload from app.internal import firestore as fstore from app.internal.mqtt_handler import mqtt_handler +from app.internal import dynsec +from app.internal.logger import logger from app.internal.auth import require_admin_token, require_service_key_or_admin from app.routers.tokens import assign_token, release_token @@ -31,8 +33,23 @@ async def approve_node(node_id: str, _: dict = Depends(require_admin_token)): raise HTTPException(404, f"Node '{node_id}' not found.") api_key = secrets.token_hex(32) + + # dynsec FIRST, Firestore second: if the broker rejects/never confirms + # the new client, we must not tell Firestore (and the admin UI) the + # node is approved with a key mosquitto doesn't actually recognise — + # that's exactly the silent-drift the two-sources-of-truth problem + # warns about. See app/internal/dynsec.py. + try: + await dynsec.upsert_node_client(node_id, api_key) + except dynsec.DynsecError as e: + logger.error(f"Approve {node_id!r}: dynsec upsert failed, NOT writing Firestore: {e}") + raise HTTPException(502, f"Could not provision MQTT credentials for node: {e}") + await fstore.doc_set("node_keys", node_id, {"node_id": node_id, "api_key": api_key}, merge=False) await fstore.doc_update("nodes", node_id, {"approval_status": "approved"}) + # TODO(mqtt-cutover): drop this MQTT push once nodes pull their key via + # GET /nodes/{id}/credentials (routers/enrollment.py) exclusively — see + # MQTT-PUBLIC-AUTH-PLAN.md "Rollout order" step 6. Kept for node-26. mqtt_handler.publish_node_key(node_id, api_key) return {"ok": True} @@ -42,6 +59,11 @@ async def delete_node(node_id: str, _: dict = Depends(require_admin_token)): node = await fstore.doc_get("nodes", node_id) if not node: raise HTTPException(404, f"Node '{node_id}' not found.") + try: + await dynsec.delete_node_client(node_id) + except dynsec.DynsecError as e: + logger.error(f"Delete {node_id!r}: dynsec deleteClient failed, NOT deleting Firestore docs: {e}") + raise HTTPException(502, f"Could not revoke MQTT credentials for node: {e}") await fstore.doc_delete("node_keys", node_id) await fstore.doc_delete("nodes", node_id) @@ -102,7 +124,15 @@ async def reissue_node_key(node_id: str, _: dict = Depends(require_admin_token)) if not node: raise HTTPException(404, f"Node '{node_id}' not found.") api_key = secrets.token_hex(32) + try: + await dynsec.upsert_node_client(node_id, api_key) + except dynsec.DynsecError as e: + logger.error(f"Reissue {node_id!r}: dynsec upsert failed, NOT writing Firestore: {e}") + raise HTTPException(502, f"Could not update MQTT credentials for node: {e}") await fstore.doc_set("node_keys", node_id, {"node_id": node_id, "api_key": api_key}, merge=False) + # TODO(mqtt-cutover): drop this MQTT push once nodes pull their key via + # GET /nodes/{id}/credentials (routers/enrollment.py) exclusively — see + # MQTT-PUBLIC-AUTH-PLAN.md "Rollout order" step 6. Kept for node-26. mqtt_handler.publish_node_key(node_id, api_key) return {"ok": True} diff --git a/drb-c2-core/mosquitto/acl.conf b/drb-c2-core/mosquitto/acl.conf deleted file mode 100644 index c92cb27..0000000 --- a/drb-c2-core/mosquitto/acl.conf +++ /dev/null @@ -1,19 +0,0 @@ -# ----------------------------------------------------------------------- -# Mosquitto ACL — DRB C2 Server -# ----------------------------------------------------------------------- -# Two principals: -# drb-c2-core — the backend service; needs full broker access -# drb-node — shared credential for all edge nodes; scoped to their -# own namespace via MQTT client ID (%c = NODE_ID) -# ----------------------------------------------------------------------- - -# C2-core service — full read/write on every topic -user drb-c2-core -topic readwrite # - -# Edge nodes — each node may only read/write topics under nodes// -# Mosquitto substitutes %c with the connecting client's MQTT client ID at -# runtime. Edge nodes set client_id = NODE_ID in mqtt_manager.py, so this -# cryptographically prevents node-A from publishing to nodes/node-B/api_key -# or any other node's namespace. -pattern readwrite nodes/%c/# diff --git a/drb-c2-core/mosquitto/entrypoint.sh b/drb-c2-core/mosquitto/entrypoint.sh deleted file mode 100644 index e8b52fa..0000000 --- a/drb-c2-core/mosquitto/entrypoint.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Mosquitto entrypoint — generates /mosquitto/config/passwd from env vars -# before handing off to the broker process. -# -# Required environment variables (set in docker-compose.yml): -# MQTT_C2_USER — username for the drb-c2-core service -# MQTT_C2_PASS — password for the drb-c2-core service -# MQTT_NODE_USER — shared username for all edge nodes -# MQTT_NODE_PASS — shared password for all edge nodes -set -e - -PASSWD_FILE=/tmp/passwd - -# Remove any stale file so we start clean on every container start -rm -f "$PASSWD_FILE" - -if [ -z "$MQTT_C2_USER" ] || [ -z "$MQTT_C2_PASS" ]; then - echo "ERROR: MQTT_C2_USER and MQTT_C2_PASS must be set" >&2 - exit 1 -fi - -if [ -z "$MQTT_NODE_USER" ] || [ -z "$MQTT_NODE_PASS" ]; then - echo "ERROR: MQTT_NODE_USER and MQTT_NODE_PASS must be set" >&2 - exit 1 -fi - -# -c creates/overwrites the file; subsequent calls append without -c -mosquitto_passwd -c -b "$PASSWD_FILE" "$MQTT_C2_USER" "$MQTT_C2_PASS" -mosquitto_passwd -b "$PASSWD_FILE" "$MQTT_NODE_USER" "$MQTT_NODE_PASS" - -# mosquitto_passwd creates the file 0600 (root-only); mosquitto drops to -# the mosquitto user before reading it, so make it world-readable. -chmod 644 "$PASSWD_FILE" - -echo "Mosquitto: password file written for users: $MQTT_C2_USER, $MQTT_NODE_USER" - -exec /usr/sbin/mosquitto -c /mosquitto/config/mosquitto.conf diff --git a/drb-c2-core/mosquitto/mosquitto.conf b/drb-c2-core/mosquitto/mosquitto.conf index ef97cdf..9496ecf 100644 --- a/drb-c2-core/mosquitto/mosquitto.conf +++ b/drb-c2-core/mosquitto/mosquitto.conf @@ -1,11 +1,43 @@ -listener 1883 +# Auth: mosquitto's own built-in dynamic-security plugin — NOT +# mosquitto-go-auth (that project is archived upstream, no CVE patches; +# rejected for an internet-facing broker). This plugin ships in and is +# maintained alongside the official eclipse-mosquitto image itself. +# See MQTT-PUBLIC-AUTH-PLAN.md and app/internal/dynsec.py for the full +# design (bootstrap, roles, the two-sources-of-truth reconcile). +# +# Plugin path is DERIVED FROM SOURCE (docker/2.1-alpine/Dockerfile in +# eclipse-mosquitto/mosquitto), not observed by running the image — +# nothing in this project executes/pulls images from this machine. Verify +# it on first real deploy: `docker compose logs mosquitto` will say +# "Error: Unable to load plugin" at the exact path below if it's wrong for +# whatever patch tag ends up pinned. +plugin /usr/lib/mosquitto_dynamic_security.so +# Lives on the same persistent volume as `persistence_location` below — +# one durable volume for all broker state, survives redeploys. +plugin_opt_config_file /mosquitto/data/dynamic-security.json + allow_anonymous false +# No password_file/acl_file directive anywhere in this file — the plugin +# above is the only registered auth backend. There is no "coexist" mode: +# nothing else is registered to conflict with it. -# Credentials and ACLs are generated/mounted at container startup -password_file /tmp/passwd -acl_file /mosquitto/config/acl.conf +# Internal, plaintext — c2-core's own connection only (its dynsec-admin +# control-plane calls AND its regular data-plane pub/sub both use this). +# Never published to the host in prod (docker-compose.prod.yml removes the +# port mapping); external nodes use the TLS listener below instead. +listener 1883 + +# Public, TLS — edge nodes connect here as username=node_id, password=api_key +# (the same credential /upload already trusts via node_keys), authorized by +# the "node" dynsec role (nodes/%u/# — %u is the dynsec-authenticated +# username, fixing the old %c-based ACL's client-ID-spoofing hole). Cert/key +# come from infra/ansible's Caddy cert-sync unit; see +# MQTT-PUBLIC-AUTH-PLAN.md "Infra" and the "Rollout order" cert-verification +# step for what happens before that cert exists. +listener 8883 +certfile /mosquitto/certs/mqtt.crt +keyfile /mosquitto/certs/mqtt.key -# Persist retained messages (e.g. api_key, node status) across broker restarts persistence true persistence_location /mosquitto/data/ diff --git a/infra/ansible/roles/deploy/handlers/main.yml b/infra/ansible/roles/deploy/handlers/main.yml index 54b2a7d..7d71816 100644 --- a/infra/ansible/roles/deploy/handlers/main.yml +++ b/infra/ansible/roles/deploy/handlers/main.yml @@ -11,3 +11,9 @@ name: caddy state: reloaded enabled: true + +# For the mqtt-cert-sync.service/.path unit files — systemd won't pick up a +# new/changed unit file until the manager config is reloaded. +- name: Reload systemd daemon + ansible.builtin.systemd_service: + daemon_reload: true diff --git a/infra/ansible/roles/deploy/tasks/main.yml b/infra/ansible/roles/deploy/tasks/main.yml index 6e26773..c7ec660 100644 --- a/infra/ansible/roles/deploy/tasks/main.yml +++ b/infra/ansible/roles/deploy/tasks/main.yml @@ -66,6 +66,83 @@ mode: "0644" notify: Reload Caddy +# --- MQTT TLS cert sync (Caddy -> mosquitto) -------------------------------- +# See MQTT-PUBLIC-AUTH-PLAN.md "Infra". mosquitto reads its cert from this +# directory (docker-compose.prod.yml bind-mounts it in); nothing but root can +# read Caddy's own cert storage, so a systemd path unit + oneshot service +# copies a readable copy out and SIGHUPs the broker on every change. + +- name: Create mosquitto certs directory + file: + path: /opt/drb/mosquitto-certs + state: directory + owner: root + group: root + mode: "0700" + +# dynamic-security.json (node credentials — see app/internal/dynsec.py) +# lives here. Root-owned is fine: the mosquitto container itself runs as +# root (no `user` directive in mosquitto.conf, matching the pre-existing +# setup this project already ran before the dynsec change), so it can +# read/write this directory directly without any host-side chown dance. +- name: Create mosquitto data directory + file: + path: /opt/drb/mosquitto-data + state: directory + owner: root + group: root + mode: "0700" + +- name: Deploy MQTT cert-sync script + template: + src: sync-mqtt-cert.sh.j2 + dest: /opt/drb/sync-mqtt-cert.sh + owner: root + group: root + mode: "0700" + +- name: Deploy MQTT cert-sync systemd service unit + template: + src: mqtt-cert-sync.service.j2 + dest: /etc/systemd/system/mqtt-cert-sync.service + owner: root + group: root + mode: "0644" + notify: Reload systemd daemon + +- name: Deploy MQTT cert-sync systemd path unit + template: + src: mqtt-cert-sync.path.j2 + dest: /etc/systemd/system/mqtt-cert-sync.path + owner: root + group: root + mode: "0644" + notify: Reload systemd daemon + +# Flush the daemon-reload handler now (rather than at end-of-play) so the +# path unit is registered and actively watching BEFORE the "Reload Caddy" +# handler below fires and Caddy goes to obtain the mqtt.{{ domain }} cert — +# otherwise the unit could miss the very first PathChanged event. +- name: Apply pending handlers (systemd daemon-reload) + meta: flush_handlers + +- name: Enable and start MQTT cert-sync path unit + ansible.builtin.systemd_service: + name: mqtt-cert-sync.path + state: started + enabled: true + +# Best-effort initial sync in case Caddy already has a cert from a previous +# run (e.g. re-running this playbook after the first successful deploy) — +# the path unit only fires on a CHANGE, so it won't pick up a cert that was +# already sitting there unchanged before it started watching. Non-fatal if +# nothing exists yet (first-ever run, before Caddy has issued anything). +- name: Best-effort initial MQTT cert sync + command: /opt/drb/sync-mqtt-cert.sh + register: _initial_sync + changed_when: "'copied cert' in _initial_sync.stdout" + failed_when: false + - name: Log in to container registry command: > docker login {{ vault_registry_host }} diff --git a/infra/ansible/roles/deploy/templates/Caddyfile.j2 b/infra/ansible/roles/deploy/templates/Caddyfile.j2 index 31fa740..e83c061 100644 --- a/infra/ansible/roles/deploy/templates/Caddyfile.j2 +++ b/infra/ansible/roles/deploy/templates/Caddyfile.j2 @@ -1,11 +1,34 @@ # Managed by Ansible — do not edit manually. api.{{ domain }} { - reverse_proxy localhost:8888 { - header_up X-Forwarded-For {remote_host} + # MQTT auth is no longer an HTTP backend c2-core exposes (it moved to + # mosquitto's own built-in dynamic-security plugin, administered over MQTT + # control topics — see app/internal/dynsec.py) — there is currently no + # /internal/* route in c2-core at all. This block stays anyway as defence + # in depth: c2-core's app-wide reverse_proxy below forwards every path by + # default, so this guarantees any FUTURE /internal/* route (or a + # regression that reintroduces one) is still unreachable from the public + # internet unless someone also deliberately deletes this block. `route` + # forces top-to-bottom evaluation instead of Caddy's automatic directive + # sorting, so this is guaranteed to run before reverse_proxy. + route { + respond /internal/* 404 + reverse_proxy localhost:8888 { + header_up X-Forwarded-For {remote_host} + } } } +# mqtt.{{ domain }} has no application behind it — mosquitto's TLS listener +# (8883) is a raw MQTT socket, not HTTP, so Caddy can't reverse_proxy to it. +# This block's only job is to make Caddy request+manage a Let's Encrypt cert +# for the name via ACME HTTP-01, which infra/ansible's cert-sync unit then +# copies out to mosquitto. The DNS A record for mqtt.{{ domain }} must exist +# before this runs, or ACME issuance fails (see MQTT-PUBLIC-AUTH-PLAN.md). +mqtt.{{ domain }} { + respond 404 +} + # Frontend is served on the bare domain, not app.{{ domain }}: only drb and api # have public DNS records. A vhost for a name with no A record still starts, # but Caddy retries ACME against it forever and logs a failure each time. diff --git a/infra/ansible/roles/deploy/templates/c2-core.env.j2 b/infra/ansible/roles/deploy/templates/c2-core.env.j2 index 7e37a96..1f30633 100644 --- a/infra/ansible/roles/deploy/templates/c2-core.env.j2 +++ b/infra/ansible/roles/deploy/templates/c2-core.env.j2 @@ -5,6 +5,11 @@ MQTT_PORT=1883 MQTT_USER={{ vault_mqtt_c2_user }} MQTT_PASS={{ vault_mqtt_c2_pass }} +# Same value as mosquitto's MOSQUITTO_DYNSEC_PASSWORD (root.env.j2) — lets +# c2-core log in as the dynsec plugin's built-in "admin" client to +# administer node credentials. See app/internal/dynsec.py. +MQTT_DYNSEC_ADMIN_PASS={{ vault_mqtt_dynsec_admin_pass }} + # No GCP_CREDENTIALS_PATH — the VM uses Application Default Credentials # via the GCE metadata server. The Terraform IAM bindings grant the required roles. FIRESTORE_DATABASE={{ vault_firestore_database }} @@ -15,6 +20,11 @@ GOOGLE_MAPS_API_KEY={{ vault_google_maps_api_key }} GEMINI_API_KEY={{ vault_gemini_api_key }} SERVICE_KEY={{ vault_service_key }} -NODE_API_KEY={{ vault_node_api_key }} +ENROLLMENT_TOKEN={{ vault_enrollment_token }} -CORS_ORIGINS=["https://app.{{ domain }}"] +# Bare domain, not app.: the frontend is served on {{ domain }} itself +# (see Caddyfile.j2 — only api. and the bare name have DNS records). This said +# app.{{ domain }} while the browser origin was https://{{ domain }}, so every +# frontend call to the API would have failed CORS. If the frontend ever moves +# to app.{{ domain }}, change this at the same time. +CORS_ORIGINS=["https://{{ domain }}"] diff --git a/infra/ansible/roles/deploy/templates/mqtt-cert-sync.path.j2 b/infra/ansible/roles/deploy/templates/mqtt-cert-sync.path.j2 new file mode 100644 index 0000000..96cb4ed --- /dev/null +++ b/infra/ansible/roles/deploy/templates/mqtt-cert-sync.path.j2 @@ -0,0 +1,18 @@ +# Managed by Ansible — do not edit manually. +# +# Watches Caddy's on-disk cert for mqtt.{{ domain }} and fires +# mqtt-cert-sync.service on every change (initial issuance + every renewal). +# See sync-mqtt-cert.sh.j2 for the "unverified path" caveat — if Caddy's +# storage layout doesn't match, this unit simply never fires and mosquitto +# keeps using its self-signed placeholder cert (see mosquitto/entrypoint.sh) +# rather than failing loudly, so check `systemctl status mqtt-cert-sync.path` +# after the first deploy. +[Unit] +Description=Watch for a renewed MQTT TLS cert from Caddy (mqtt.{{ domain }}) + +[Path] +PathChanged=/var/lib/caddy/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/mqtt.{{ domain }}/mqtt.{{ domain }}.crt +Unit=mqtt-cert-sync.service + +[Install] +WantedBy=multi-user.target diff --git a/infra/ansible/roles/deploy/templates/mqtt-cert-sync.service.j2 b/infra/ansible/roles/deploy/templates/mqtt-cert-sync.service.j2 new file mode 100644 index 0000000..33f1e81 --- /dev/null +++ b/infra/ansible/roles/deploy/templates/mqtt-cert-sync.service.j2 @@ -0,0 +1,11 @@ +# Managed by Ansible — do not edit manually. +# +# Runs as root: it needs read access to Caddy's 0700 cert storage AND the +# ability to run `docker compose kill -s HUP` regardless of docker group +# membership. Triggered by mqtt-cert-sync.path, not run standalone. +[Unit] +Description=Sync Caddy-issued MQTT TLS cert to mosquitto and reload + +[Service] +Type=oneshot +ExecStart=/opt/drb/sync-mqtt-cert.sh diff --git a/infra/ansible/roles/deploy/templates/root.env.j2 b/infra/ansible/roles/deploy/templates/root.env.j2 index b8c307a..a0148b1 100644 --- a/infra/ansible/roles/deploy/templates/root.env.j2 +++ b/infra/ansible/roles/deploy/templates/root.env.j2 @@ -12,8 +12,15 @@ MQTT_C2_USER={{ vault_mqtt_c2_user }} MQTT_C2_PASS={{ vault_mqtt_c2_pass | replace('$', '$$') }} -MQTT_NODE_USER={{ vault_mqtt_node_user }} -MQTT_NODE_PASS={{ vault_mqtt_node_pass | replace('$', '$$') }} + +# Seeds mosquitto's built-in dynamic-security plugin's one-time "admin" +# bootstrap client on first boot (read directly by the plugin's C code via +# getenv — see app/internal/dynsec.py). Must be >=12 chars (plugin-enforced +# minimum). c2-core needs this SAME value as MQTT_DYNSEC_ADMIN_PASS in its +# own env (c2-core.env.j2) to log in as "admin" and administer node +# credentials — kept as one vault var (vault_mqtt_dynsec_admin_pass) so the +# two can't drift. +MOSQUITTO_DYNSEC_PASSWORD={{ vault_mqtt_dynsec_admin_pass | replace('$', '$$') }} # Container registry prefix — docker compose uses this for image: ${REGISTRY}/name:latest REGISTRY={{ vault_registry }} diff --git a/infra/ansible/roles/deploy/templates/sync-mqtt-cert.sh.j2 b/infra/ansible/roles/deploy/templates/sync-mqtt-cert.sh.j2 new file mode 100644 index 0000000..9b1197e --- /dev/null +++ b/infra/ansible/roles/deploy/templates/sync-mqtt-cert.sh.j2 @@ -0,0 +1,51 @@ +#!/bin/bash +# Managed by Ansible — do not edit manually. +# +# Copies Caddy's managed TLS cert for mqtt.{{ domain }} out of Caddy's +# storage (root:caddy, 0700 — nothing else can read it) into a location the +# mosquitto container can read, then SIGHUPs the broker so it picks up the +# new cert without a full restart. +# +# Triggered by mqtt-cert-sync.path.j2 (a systemd path unit) watching the +# source cert file for changes — a path unit rather than cron so this fires +# on the actual write instead of racing a polling interval. +# +# UNVERIFIED: the exact source path below assumes Caddy's default file +# storage layout and Let's Encrypt's production ACME directory name. This +# has not been confirmed against a real Caddy cert issuance for this +# project — check `caddy storage` / find the actual path under +# /var/lib/caddy the first time this runs, and correct CADDY_CERT_DIR below +# if it doesn't match. +# +# UNVERIFIED: mosquitto 2.x reloading TLS certs on SIGHUP without dropping +# connections is documented upstream but untested here. If listener 8883 +# doesn't pick up the new cert (check `docker compose logs mosquitto` after +# a sync), replace the `kill -s HUP` line below with a full +# `docker compose ... restart mosquitto` instead. +set -euo pipefail + +DOMAIN="mqtt.{{ domain }}" +CADDY_CERT_DIR="/var/lib/caddy/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/${DOMAIN}" +DEST_DIR="/opt/drb/mosquitto-certs" +APP_DIR="{{ app_dir }}" + +SRC_CERT="${CADDY_CERT_DIR}/${DOMAIN}.crt" +SRC_KEY="${CADDY_CERT_DIR}/${DOMAIN}.key" + +if [ ! -f "$SRC_CERT" ] || [ ! -f "$SRC_KEY" ]; then + echo "sync-mqtt-cert: source cert/key not found yet at $CADDY_CERT_DIR — Caddy may not have issued it yet." >&2 + exit 0 +fi + +mkdir -p "$DEST_DIR" +# Copy, don't symlink — nothing outside the caddy user can read the +# originals (0700-owned), so mosquitto (running as a different container/ +# user) needs its own readable copy, not a pointer to an unreadable file. +cp "$SRC_CERT" "$DEST_DIR/mqtt.crt" +cp "$SRC_KEY" "$DEST_DIR/mqtt.key" +chmod 600 "$DEST_DIR/mqtt.crt" "$DEST_DIR/mqtt.key" +chown root:root "$DEST_DIR/mqtt.crt" "$DEST_DIR/mqtt.key" + +cd "$APP_DIR" +docker compose -f docker-compose.yml -f docker-compose.prod.yml kill -s HUP mosquitto +echo "sync-mqtt-cert: copied cert for ${DOMAIN} and sent SIGHUP to mosquitto." diff --git a/infra/ansible/vault.yml.example b/infra/ansible/vault.yml.example index 81ca1dc..56711f1 100644 --- a/infra/ansible/vault.yml.example +++ b/infra/ansible/vault.yml.example @@ -12,14 +12,18 @@ # Generate with: openssl rand -hex 32 (hex output has no shell metacharacters) # ── MQTT ───────────────────────────────────────────────────────────────────── +# No more shared node credential (vault_mqtt_node_user/pass) — nodes now +# authenticate as username=, password=, +# checked by mosquitto's built-in dynamic-security plugin (c2-core +# administers it — see app/internal/dynsec.py). See vault_enrollment_token +# below for how a node gets that key in the first place. vault_mqtt_c2_user: drb-c2-core vault_mqtt_c2_pass: "CHANGE_ME" -vault_mqtt_node_user: drb-node -vault_mqtt_node_pass: "CHANGE_ME" +vault_mqtt_dynsec_admin_pass: "CHANGE_ME" # openssl rand -hex 32 — must be >=12 chars, plugin-enforced minimum # ── C2 Core ─────────────────────────────────────────────────────────────────── vault_service_key: "" # openssl rand -hex 32 -vault_node_api_key: "" # openssl rand -hex 32 +vault_enrollment_token: "" # openssl rand -hex 32 — fleet-wide, shared by every node's POST /nodes/enroll vault_openai_api_key: "" vault_google_maps_api_key: "" vault_gemini_api_key: "" diff --git a/infra/main.tf b/infra/main.tf index 73f34a1..d099d19 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -64,20 +64,25 @@ resource "google_compute_firewall" "allow_ssh" { target_tags = ["drb-server"] } -# MQTT is NOT exposed externally — edge nodes connect via WireGuard (see below) -# If you need to temporarily allow direct MQTT access for testing, uncomment and -# restrict source_ranges to your node IPs. -# -# resource "google_compute_firewall" "allow_mqtt" { -# name = "drb-allow-mqtt" -# network = "default" -# allow { -# protocol = "tcp" -# ports = ["8883"] # TLS MQTT, not 1883 -# } -# source_ranges = ["YOUR_NODE_CIDR"] -# target_tags = ["drb-server"] -# } +# MQTT is now publicly exposed on 8883 (TLS) — nodes get deployed to +# arbitrary locations by arbitrary people, so there is no fixed CIDR to +# restrict this to (WireGuard-per-node was evaluated and rejected; see +# MQTT-PUBLIC-AUTH-PLAN.md). Security is enforced by mosquitto-go-auth +# (per-node api_key over TLS), not by network ACL. 1883 (plaintext) is +# intentionally NOT opened here — it stays on the docker bridge for +# c2-core's own connection only. +resource "google_compute_firewall" "allow_mqtt" { + name = "drb-allow-mqtt" + network = "default" + + allow { + protocol = "tcp" + ports = ["8883"] # TLS MQTT only, not 1883 + } + + source_ranges = ["0.0.0.0/0"] + target_tags = ["drb-server"] +} # --------------------------------------------------------------------------- # Compute Engine VM @@ -185,5 +190,9 @@ resource "google_storage_bucket" "audio" { # After terraform apply, add these A records in Route 53: # app.drb.cusano.net → server_ip output # api.drb.cusano.net → server_ip output +# mqtt.drb.cusano.net → server_ip output — MUST exist before the ansible +# deploy that adds the Caddy +# mqtt. block, or ACME +# issuance for it fails. # Or use a single wildcard: *.drb.cusano.net → server_ip # ---------------------------------------------------------------------------