Let edge nodes read /systems with their own api_key

The node builds its OP25 config from GET /systems, but that router only
accepted a Firebase token or the shared service key — a node holds neither.
Every fetch returned 401 and the node fell back to its stale offline cache,
so a system edited in the UI never reached the field. Confirmed on node-002
against the live server: "Failed to fetch systems from C2: 401 Unauthorized
... Offline cache will be used."

The node sends no node_id with the request, only the bearer token, so the
key is matched by querying node_keys for the value instead of fetching a
known document the way /upload does.

Read access only: the mutating routes in this router each carry their own
require_admin_token, so widening the router-level gate doesn't let a node
create, edit or delete a system.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

.env.example

@@ -7,11 +7,20 @@
 # password file. Use different values in production — do NOT reuse defaults.
 # -----------------------------------------------------------------------
 
-# C2-core service account (full broker access)
+# C2-core service account (full broker access via the "c2core" dynsec role)
 MQTT_C2_USER=drb-c2-core
 MQTT_C2_PASS=change-me-c2
 
-# Shared credential for all edge nodes (ACL scopes each node to its own
-# nodes/<NODE_ID>/# namespace via the MQTT client ID)
-MQTT_NODE_USER=drb-node
-MQTT_NODE_PASS=change-me-node
+# Seeds mosquitto's built-in dynamic-security plugin's one-time "admin"
+# bootstrap client on first boot (read directly by mosquitto, no entrypoint
+# scripting involved). Must be >=12 chars. c2-core needs this SAME value as
+# MQTT_DYNSEC_ADMIN_PASS in drb-c2-core/.env to log in as "admin" and
+# administer node credentials — see app/internal/dynsec.py.
+MOSQUITTO_DYNSEC_PASSWORD=change-me-dynsec-admin-min-12-chars
+
+# There is no shared node credential anymore. Each node authenticates as
+# username=<node_id>, password=<its node_keys.api_key> — checked by
+# mosquitto's dynamic-security plugin (not an HTTP backend — that was an
+# earlier, since-rejected design using the now-archived mosquitto-go-auth).
+# Nodes obtain that key via the enrollment flow — see ENROLLMENT_TOKEN in
+# drb-c2-core/.env.example.

.gitea/workflows/deploy.yml

@@ -52,6 +52,15 @@ jobs:
           tags: |
             ${{ env.REGISTRY }}/frontend:latest
             ${{ env.REGISTRY }}/frontend:${{ gitea.sha }}
+          build-args: |
+            NEXT_PUBLIC_C2_URL=https://api.${{ secrets.DRB_DOMAIN }}
+            NEXT_PUBLIC_FIREBASE_API_KEY=${{ secrets.FIREBASE_API_KEY }}
+            NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN=${{ secrets.FIREBASE_AUTH_DOMAIN }}
+            NEXT_PUBLIC_FIREBASE_PROJECT_ID=${{ secrets.FIREBASE_PROJECT_ID }}
+            NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET=${{ secrets.FIREBASE_STORAGE_BUCKET }}
+            NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID=${{ secrets.FIREBASE_MESSAGING_SENDER_ID }}
+            NEXT_PUBLIC_FIREBASE_APP_ID=${{ secrets.FIREBASE_APP_ID }}
+            NEXT_PUBLIC_FIRESTORE_DATABASE=${{ secrets.FIRESTORE_DATABASE }}
 
   deploy:
     name: Deploy to VM
@@ -59,15 +68,21 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Check runner outbound IP
+        run: curl -s ifconfig.me
+
       - name: Write SSH key
         run: |
-          echo "${{ secrets.SSH_PRIVATE_KEY }}" > /tmp/deploy_key
+          printf '%s\n' "${{ secrets.SSH_PRIVATE_KEY }}" > /tmp/deploy_key
           chmod 600 /tmp/deploy_key
+          ssh-keygen -l -f /tmp/deploy_key
 
       - name: Deploy
         run: |
           ssh -o StrictHostKeyChecking=no \
               -o HostKeyAlgorithms=ssh-ed25519,rsa-sha2-256,rsa-sha2-512 \
+              -o ConnectTimeout=15 \
+              -v \
               -i /tmp/deploy_key \
               drb@${{ secrets.SERVER_IP }} << 'ENDSSH'
           set -e

.gitignore

@@ -15,7 +15,10 @@ infra/terraform.tfvars
 infra/tf.log
 infra/ansible/inventory.ini
 infra/ansible/group_vars/all.yml
-infra/ansible/vault.yml
+# Glob, not the bare filename: ansible-vault edit and manual backups leave
+# siblings like vault.yml.locked.bak, which the exact-name rule left untracked
+# but committable.
+infra/ansible/vault.yml*
 
 # Python
 __pycache__/

docker-compose.prod.yml

@@ -8,13 +8,36 @@
 #   - restart: always (instead of unless-stopped) for hard reboots.
 
 services:
+  # ports AND volumes both need !override here, not !reset/a plain list —
+  # compose merges list-type fields by APPENDING across -f files. A plain
+  # list (or !reset on volumes) would leave dev's mosquitto_certs named
+  # volume mounted at /mosquitto/certs alongside this bind mount, and two
+  # mounts targeting the same path is exactly the "address already in use"-
+  # style footgun the c2-core override below already hit once with ports.
+  # mosquitto-data is now a host bind mount too (not just certs) — it holds
+  # dynamic-security.json, the broker's only record of node credentials
+  # (see app/internal/dynsec.py "TWO-SOURCES-OF-TRUTH"). A named Docker
+  # volume already survives normal redeploys (git pull && compose pull &&
+  # up -d never passes -v), but the bind mount makes it inspectable/
+  # backupable the same way the cert directory already is. NOT read-only —
+  # mosquitto writes dynamic-security.json here.
   mosquitto:
     restart: always
-    ports: !reset []   # Remove the dev 1883:1883 mapping — internal only
+    ports: !override
+      - "8883:8883"   # TLS only, published. 1883 stays internal (docker bridge, c2-core's own login).
+    volumes: !override
+      - ./drb-c2-core/mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro
+      - /opt/drb/mosquitto-data:/mosquitto/data
+      - /opt/drb/mosquitto-certs:/mosquitto/certs:ro  # fed by the cert-sync systemd unit, see infra/ansible
 
+  # !override, not a plain list: compose MERGES `ports` by appending, so a plain
+  # list leaves the base file's "8888:8000" in place alongside this one. The
+  # container then tries to bind 8888 twice — 0.0.0.0 and 127.0.0.1 — and the
+  # second bind fails with "address already in use". It also silently defeated
+  # the whole point of this override, publishing the port on every interface.
   c2-core:
     restart: always
-    ports:
+    ports: !override
       - "127.0.0.1:8888:8000"  # Caddy proxies, not exposed publicly
 
   discord-bot:
@@ -22,5 +45,5 @@ services:
 
   frontend:
     restart: always
-    ports:
+    ports: !override
       - "127.0.0.1:3000:3000"  # Caddy proxies, not exposed publicly

docker-compose.yml

@@ -1,20 +1,23 @@
 services:
+  # Auth is mosquitto's own built-in dynamic-security plugin (see
+  # mosquitto.conf + app/internal/dynsec.py) — NOT mosquitto-go-auth, that
+  # project is archived upstream (no CVE patches), rejected for a
+  # public-internet broker. Stock official image, pinned to an exact patch
+  # (not the floating `:2` tag). MOSQUITTO_DYNSEC_PASSWORD seeds the
+  # plugin's own one-time "admin" bootstrap client on first boot — read
+  # directly by the plugin's C code, no entrypoint scripting needed for it.
   mosquitto:
-    image: eclipse-mosquitto:2
+    image: eclipse-mosquitto:2.1.2-alpine
     restart: unless-stopped
     ports:
       - "1883:1883"
-    entrypoint: ["/bin/sh", "/mosquitto/config/entrypoint.sh"]
+      - "8883:8883"
     environment:
-      - MQTT_C2_USER=${MQTT_C2_USER}
-      - MQTT_C2_PASS=${MQTT_C2_PASS}
-      - MQTT_NODE_USER=${MQTT_NODE_USER}
-      - MQTT_NODE_PASS=${MQTT_NODE_PASS}
+      - MOSQUITTO_DYNSEC_PASSWORD=${MOSQUITTO_DYNSEC_PASSWORD}
     volumes:
       - ./drb-c2-core/mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro
-      - ./drb-c2-core/mosquitto/acl.conf:/mosquitto/config/acl.conf:ro
-      - ./drb-c2-core/mosquitto/entrypoint.sh:/mosquitto/config/entrypoint.sh:ro
       - mosquitto_data:/mosquitto/data
+      - mosquitto_certs:/mosquitto/certs
 
   c2-core:
     image: ${REGISTRY}/c2-core:${TAG:-latest}
@@ -45,4 +48,11 @@ services:
       - c2-core
 
 volumes:
+  # Dev only for both. Prod overrides these to host bind mounts
+  # (/opt/drb/mosquitto-data, /opt/drb/mosquitto-certs — the latter fed by
+  # the Caddy cert-sync systemd unit) — see docker-compose.prod.yml and
+  # infra/ansible/roles/deploy/templates/. mosquitto_data holds
+  # dynamic-security.json (node MQTT credentials, see app/internal/dynsec.py)
+  # as well as the usual broker persistence state.
   mosquitto_data:
+  mosquitto_certs:

drb-c2-core/.env.example

@@ -2,10 +2,15 @@
 MQTT_BROKER=mosquitto
 MQTT_PORT=1883
 # Use the c2-core credential — must match MQTT_C2_USER/MQTT_C2_PASS in the
-# top-level .env (which is passed to the mosquitto entrypoint)
+# top-level .env
 MQTT_USER=drb-c2-core
 MQTT_PASS=change-me-c2
 
+# Same value as the top-level .env's MOSQUITTO_DYNSEC_PASSWORD — lets
+# c2-core log in as mosquitto's built-in dynsec "admin" client to
+# administer node MQTT credentials. See app/internal/dynsec.py.
+MQTT_DYNSEC_ADMIN_PASS=change-me-dynsec-admin-min-12-chars
+
 # GCP — path to service account JSON inside the container
 GCP_CREDENTIALS_PATH=/app/gcp-key.json
 
@@ -28,6 +33,7 @@ SUMMARY_INTERVAL_MINUTES=15
 CORRELATION_WINDOW_HOURS=4
 EMBEDDING_SIMILARITY_THRESHOLD=0.82
 
-# Auth — static key that edge nodes send as Bearer token on /upload
+# Fleet-wide token edge nodes present as X-Enrollment-Token on first boot
+# (POST /nodes/enroll). Shared across every node — NOT a per-node secret.
 # Generate with: openssl rand -hex 32
-NODE_API_KEY=
+ENROLLMENT_TOKEN=

drb-c2-core/app/config.py

@@ -9,6 +9,15 @@ class Settings(BaseSettings):
     mqtt_user: Optional[str] = None
     mqtt_pass: Optional[str] = None
 
+    # mosquitto's built-in dynamic-security plugin (see app/internal/dynsec.py).
+    # "admin" is hardcoded by the plugin itself on first boot — not actually
+    # configurable — kept as a named setting rather than a literal for
+    # readability. mqtt_dynsec_admin_pass must equal the mosquitto
+    # container's own MOSQUITTO_DYNSEC_PASSWORD env var (root .env /
+    # root.env.j2) or c2-core can't administer node credentials at all.
+    mqtt_dynsec_admin_user: str = "admin"
+    mqtt_dynsec_admin_pass: Optional[str] = None
+
     # GCP
     gcp_credentials_path: Optional[str] = None  # None → uses ADC
     gcs_bucket: Optional[str] = None             # None → audio upload disabled
@@ -51,6 +60,11 @@ class Settings(BaseSettings):
     # Internal service key — allows server-side services (discord bot) to call C2 without Firebase
     service_key: Optional[str] = None
 
+    # Fleet-wide token edge nodes present to POST /nodes/enroll on first boot.
+    # Not a per-node secret — see routers/enrollment.py for why a leaked copy
+    # of this alone can't steal an already-approved node's key.
+    enrollment_token: Optional[str] = None
+
     # Upload size limit — reject audio files larger than this (bytes). Default 100 MB.
     upload_max_bytes: int = 100 * 1024 * 1024
 

drb-c2-core/app/internal/auth.py

@@ -37,6 +37,41 @@ async def require_service_or_firebase_token(
         raise HTTPException(status_code=401, detail="Invalid or expired token")
 
 
+async def require_node_service_or_firebase_token(
+    credentials: Optional[HTTPAuthorizationCredentials] = Security(_bearer),
+) -> dict:
+    """Accept a node's own API key in addition to a service key / Firebase token.
+
+    Edge nodes need to read ``/systems`` to build their OP25 config, but they
+    hold neither a Firebase token nor the shared service key — only the
+    per-node api_key that ``/upload`` already trusts. Without this they got a
+    flat 401 and silently fell back to their stale offline cache, so a system
+    edited in the UI never reached the node.
+
+    Unlike ``/upload``, the node sends no node_id alongside the bearer token,
+    so the key is matched by querying ``node_keys`` for the value rather than
+    fetching a known document. Mutating routes are unaffected: they carry
+    their own ``require_admin_token`` dependency, so widening the router-level
+    gate grants nodes read access only.
+    """
+    if not credentials:
+        raise HTTPException(status_code=401, detail="Missing authorization token")
+    token = credentials.credentials
+    if settings.service_key and secrets.compare_digest(token, settings.service_key):
+        return {"service": True}
+    try:
+        return firebase_auth.verify_id_token(token)
+    except Exception:
+        pass
+    # Deferred import: app.internal.firestore initialises firebase-admin at
+    # import time, and auth.py is imported from module scope in the routers.
+    from app.internal import firestore as fstore
+    matches = await fstore.collection_list("node_keys", api_key=token)
+    if matches:
+        return {"node": True, "node_id": matches[0].get("node_id")}
+    raise HTTPException(status_code=401, detail="Invalid or expired token")
+
+
 def get_role(decoded: dict) -> str:
     """Extract the effective role from a decoded Firebase token.
 

drb-c2-core/app/internal/dynsec.py

@@ -0,0 +1,341 @@
+"""
+Client for mosquitto's built-in dynamic-security plugin.
+
+WHY THIS EXISTS: MQTT-PUBLIC-AUTH-PLAN.md originally specced the
+mosquitto-go-auth plugin (HTTP backend). That project was archived by its
+maintainer 2025-08-06 ("no more changes") — unacceptable for a broker that's
+about to be reachable from the public internet, no way to get a CVE fix.
+Replaced with mosquitto 2.x's own `dynamic-security` plugin, which ships in
+and is maintained alongside the official eclipse-mosquitto image itself.
+
+HOW DYNSEC WORKS (verified against plugin source on
+github.com/eclipse-mosquitto/mosquitto, 2026-08-16 — see citations inline;
+NOT verified by running anything, per instruction not to execute/deploy
+anything from this machine):
+
+  - The broker persists clients/roles/ACLs in a JSON file at
+    `plugin_opt_config_file` (we point this at /mosquitto/data/, the same
+    volume `persistence_location` already uses — one durable volume for all
+    broker state, see docker-compose.yml).
+  - Admin commands are plain MQTT publishes: JSON `{"commands": [...]}` to
+    `$CONTROL/dynamic-security/v1` (source: plugin.c,
+    `mosquitto_callback_register(plg_id, MOSQ_EVT_CONTROL,
+    dynsec_control_callback, "$CONTROL/dynamic-security/v1", ...)`).
+    Replies come back on `$CONTROL/dynamic-security/v1/response`
+    (source: control.c, `#define RESPONSE_TOPIC
+    "$CONTROL/dynamic-security/v1/response"`).
+  - Per-command JSON fields (verified against clients.c / roles.c handlers
+    and the plugin README):
+      createClient: username, password, clientid, textname, textdescription,
+                    roles: [{rolename, priority}], groups: [...]
+      modifyClient: same fields, username identifies the existing client
+      deleteClient: username
+      createRole:   rolename, textname, textdescription,
+                    acls: [{acltype, topic, priority, allow}]
+    acltype values: publishClientSend, publishClientReceive,
+    subscribeLiteral, subscribePattern, unsubscribeLiteral,
+    unsubscribePattern. %u (username) and %c (clientid) are valid
+    substitutions in `topic` for every type except the two *Literal ones.
+  - On first boot, if `plugin_opt_config_file` doesn't exist, the plugin
+    bootstraps itself (config_init.c): reads env var
+    `MOSQUITTO_DYNSEC_PASSWORD` (or `plugin_opt_password_init_file`) and
+    creates a client literally named "admin" (hardcoded string, NOT
+    configurable — verified in config_init.c's `client_add_admin()`) with
+    three roles: `super-admin` (full pub/sub on `$CONTROL/#` — i.e. this is
+    what makes a client capable of issuing further dynsec commands, and
+    it's an ordinary role, nothing hardcoded beyond the initial grant),
+    `sys-observe` ($SYS/# read-only), `topic-observe` (# read-only, NOT
+    read-write). If MOSQUITTO_DYNSEC_PASSWORD is set (we always set it),
+    no `democlient` demo account gets created — that only happens in the
+    "no password provided, generate one randomly" path.
+  - This is a genuine backend swap, not just config: `allow_anonymous
+    false` plus the *absence* of `password_file`/`acl_file` directives
+    means dynsec is the only auth backend registered — nothing else is
+    there to conflict with it. (Inferred from plugin architecture — every
+    mosquitto auth backend, built-in or plugin, registers the same
+    basic-auth/ACL callback hooks; there's no "layering" mechanism, so
+    without password_file/acl_file directives there is nothing else to
+    check credentials or topics.)
+
+WHAT COULD NOT BE VERIFIED (see also the plan doc + final report):
+  - The exact JSON envelope of a *response* message (only individual
+    command outcomes were confirmed: `mosquitto_control_command_reply(cmd,
+    NULL)` for success, `mosquitto_control_command_reply(cmd, "error
+    string")` for failure — the wrapping object shape, e.g. whether it's
+    `{"responses": [{"command": ..., "error": ...}]}`, was not directly
+    read from source). This client parses defensively: it treats ANY
+    dict containing a non-null "error"/"Error" key anywhere in the
+    top-level response payload as failure, presence of "already exists" in
+    that string as an idempotent success, and a response with no such key
+    within the timeout as success. A response timeout is always a hard
+    failure (never assumed to mean success).
+  - "Client already exists" was confirmed verbatim as createClient's
+    error string; "already exists" for createRole is assumed analogous,
+    not directly confirmed.
+
+TWO-SOURCES-OF-TRUTH: Firestore's `node_keys` collection is the source of
+truth for node credentials (nothing changes there); dynamic-security.json
+is a derived cache the broker uses to authenticate. `reconcile_all()`
+rebuilds every approved node's dynsec client from Firestore and is called
+on every c2-core startup — so a lost/corrupted dynamic-security.json (e.g.
+volume wiped) self-heals on the next restart instead of silently locking
+out every node. `upsert_node_client()`/`delete_node_client()` are also
+called synchronously from routers/nodes.py's approve/reissue/delete
+handlers and raise on failure — those endpoints now fail loudly (502)
+instead of updating Firestore while dynsec silently didn't get the memo.
+"""
+import asyncio
+import json
+import time
+import uuid
+import paho.mqtt.client as mqtt
+from app.config import settings
+from app.internal.logger import logger
+from app.internal import firestore as fstore
+
+CONTROL_TOPIC = "$CONTROL/dynamic-security/v1"
+RESPONSE_TOPIC = "$CONTROL/dynamic-security/v1/response"
+_RESPONSE_TIMEOUT_SECONDS = 10
+
+# Role every approved node's dynsec client is attached to. %u = the
+# authenticated username (the node_id) — this is the fixed version of the
+# old `pattern readwrite nodes/%c/#`, where %c was the client-supplied,
+# spoofable client ID.
+NODE_ROLE = "node"
+# Role c2-core's own login gets, in addition to being handed the plugin's
+# built-in `super-admin` role (see grant_c2core_admin()). Mirrors the old
+# `topic readwrite #` superuser line.
+C2CORE_ROLE = "c2core"
+
+
+class DynsecError(Exception):
+    """A dynsec command was rejected, or no response arrived in time."""
+
+
+def _run_commands_sync(commands: list[dict], username: str, password: str) -> list[dict]:
+    """
+    Blocking: open a short-lived MQTT connection, publish one or more dynsec
+    commands, wait for the matching replies, disconnect. Always called via
+    asyncio.to_thread — see the async wrappers below. A fresh connection per
+    call (rather than reusing mqtt_handler's long-lived client) keeps this
+    request/response exchange simple and isolated from that client's
+    async-callback-driven subscribe state.
+    """
+    responses: list[dict] = []
+    done = {"got": False, "error": None}
+
+    def _on_connect(client, userdata, flags, reason_code, properties):
+        if reason_code != 0:
+            done["error"] = f"connect refused: {reason_code}"
+            done["got"] = True
+            return
+        client.subscribe(RESPONSE_TOPIC, qos=1)
+        client.publish(CONTROL_TOPIC, json.dumps({"commands": commands}), qos=1)
+
+    def _on_message(client, userdata, msg):
+        try:
+            payload = json.loads(msg.payload.decode())
+        except Exception:
+            return
+        responses.append(payload)
+        done["got"] = True
+
+    client = mqtt.Client(
+        callback_api_version=mqtt.CallbackAPIVersion.VERSION2,
+        client_id=f"drb-c2-core-dynsec-{uuid.uuid4().hex[:8]}",
+    )
+    client.username_pw_set(username, password)
+    client.on_connect = _on_connect
+    client.on_message = _on_message
+
+    try:
+        client.connect(settings.mqtt_broker, settings.mqtt_port, keepalive=30)
+    except Exception as e:
+        raise DynsecError(f"could not connect to mosquitto for dynsec command: {e}")
+
+    client.loop_start()
+    deadline = time.monotonic() + _RESPONSE_TIMEOUT_SECONDS
+    try:
+        while not done["got"] and time.monotonic() < deadline:
+            time.sleep(0.05)
+    finally:
+        client.loop_stop()
+        client.disconnect()
+
+    if done["error"]:
+        raise DynsecError(str(done["error"]))
+    if not responses:
+        raise DynsecError(
+            f"no response on {RESPONSE_TOPIC} within {_RESPONSE_TIMEOUT_SECONDS}s for commands: "
+            f"{[c.get('command') for c in commands]}"
+        )
+    return responses
+
+
+def _check_responses_ok(responses: list[dict], tolerate_already_exists: bool = False) -> None:
+    """Raise DynsecError unless every response payload is error-free (or,
+    when tolerate_already_exists, only contains an 'already exists'-style
+    error — see the module docstring's "could not verify" note on why this
+    is a substring match rather than a structured error code check)."""
+    for payload in responses:
+        # Defensive: walk the payload looking for any *-cased "error" key
+        # with a non-empty value, since the exact envelope shape wasn't
+        # confirmed from source. Covers both a flat {"error": "..."} and a
+        # {"responses": [{"error": "..."}]}-style wrapper.
+        errors = _find_error_strings(payload)
+        for err in errors:
+            if tolerate_already_exists and "already exist" in err.lower():
+                continue
+            raise DynsecError(f"dynsec command failed: {err}")
+
+
+def _find_error_strings(obj) -> list[str]:
+    found = []
+    if isinstance(obj, dict):
+        for k, v in obj.items():
+            if k.lower() == "error" and v:
+                found.append(str(v))
+            else:
+                found.extend(_find_error_strings(v))
+    elif isinstance(obj, list):
+        for item in obj:
+            found.extend(_find_error_strings(item))
+    return found
+
+
+# ---------------------------------------------------------------------------
+# Async wrappers (all real work happens in the thread pool)
+# ---------------------------------------------------------------------------
+
+async def _admin_publish(commands: list[dict], tolerate_already_exists: bool = False) -> list[dict]:
+    if not settings.mqtt_dynsec_admin_pass:
+        raise DynsecError("MQTT_DYNSEC_ADMIN_PASS / mqtt_dynsec_admin_pass is not configured")
+    responses = await asyncio.to_thread(
+        _run_commands_sync, commands, settings.mqtt_dynsec_admin_user, settings.mqtt_dynsec_admin_pass
+    )
+    _check_responses_ok(responses, tolerate_already_exists=tolerate_already_exists)
+    return responses
+
+
+async def ensure_roles_and_c2core_grant() -> None:
+    """
+    Idempotent, safe to run on every startup:
+      1. createRole "node"    — nodes/%u/# publish+subscribe (both directions)
+      2. createRole "c2core"  — full "#" publish+subscribe, same reach the
+         old `topic readwrite #` superuser line gave c2-core
+      3. createClient/modifyClient drb-c2-core (settings.mqtt_user) with
+         BOTH roles above AND the plugin's built-in "super-admin" role —
+         i.e. c2-core's existing login is handed the actual dynsec admin
+         role, not a separate identity, per the design decision.
+    Runs over the dedicated "admin" bootstrap login (step 3 assigns
+    super-admin to c2-core's own login for the record / future use, but
+    THIS module still authenticates its own ongoing calls as "admin" — see
+    the module docstring for why: it's the one identity guaranteed by
+    mosquitto's own source to hold super-admin, so control-plane calls
+    don't depend on step 3's grant having actually landed).
+    """
+    node_acl_types = ["publishClientSend", "publishClientReceive", "subscribePattern", "unsubscribePattern"]
+    await _admin_publish([{
+        "command": "createRole",
+        "rolename": NODE_ROLE,
+        "textname": "DRB edge node — own namespace only",
+        "acls": [{"acltype": t, "topic": "nodes/%u/#", "priority": 0, "allow": True} for t in node_acl_types],
+    }], tolerate_already_exists=True)
+
+    c2core_acl_types = ["publishClientSend", "publishClientReceive", "subscribePattern", "unsubscribePattern"]
+    await _admin_publish([{
+        "command": "createRole",
+        "rolename": C2CORE_ROLE,
+        "textname": "DRB c2-core — full broker access",
+        "acls": [{"acltype": t, "topic": "#", "priority": 0, "allow": True} for t in c2core_acl_types],
+    }], tolerate_already_exists=True)
+
+    if not settings.mqtt_user or not settings.mqtt_pass:
+        logger.warning("dynsec: MQTT_USER/MQTT_PASS not configured — skipping c2-core client grant")
+        return
+
+    roles = [{"rolename": C2CORE_ROLE, "priority": 1}, {"rolename": "super-admin", "priority": 2}]
+    try:
+        await _admin_publish([{
+            "command": "createClient",
+            "username": settings.mqtt_user,
+            "password": settings.mqtt_pass,
+            "roles": roles,
+        }])
+        logger.info(f"dynsec: created client {settings.mqtt_user!r} with roles {C2CORE_ROLE}, super-admin")
+    except DynsecError as e:
+        if "already exist" in str(e).lower():
+            await _admin_publish([{
+                "command": "modifyClient",
+                "username": settings.mqtt_user,
+                "password": settings.mqtt_pass,
+                "roles": roles,
+            }])
+            logger.info(f"dynsec: updated existing client {settings.mqtt_user!r} with roles {C2CORE_ROLE}, super-admin")
+        else:
+            raise
+
+
+async def upsert_node_client(node_id: str, api_key: str) -> None:
+    """Create or update a node's dynsec client — called from
+    routers/nodes.py approve_node()/reissue_node_key(), and from
+    reconcile_all() on startup. Raises DynsecError on failure; callers
+    must not write Firestore as if this succeeded when it didn't."""
+    try:
+        await _admin_publish([{
+            "command": "createClient",
+            "username": node_id,
+            "password": api_key,
+            "roles": [{"rolename": NODE_ROLE, "priority": 1}],
+        }])
+    except DynsecError as e:
+        if "already exist" not in str(e).lower():
+            raise
+        await _admin_publish([{
+            "command": "modifyClient",
+            "username": node_id,
+            "password": api_key,
+            "roles": [{"rolename": NODE_ROLE, "priority": 1}],
+        }])
+
+
+async def delete_node_client(node_id: str) -> None:
+    """Best-effort: a node that was never enrolled in dynsec (or already
+    removed) is treated as already-deleted, not an error."""
+    try:
+        await _admin_publish([{"command": "deleteClient", "username": node_id}])
+    except DynsecError as e:
+        if "not found" not in str(e).lower():
+            raise
+
+
+async def reconcile_all() -> None:
+    """
+    Rebuild dynsec state for every approved node from Firestore
+    (node_keys is the source of truth). Called once at c2-core startup,
+    after ensure_roles_and_c2core_grant(). Self-heals a lost/corrupted
+    dynamic-security.json (e.g. volume wiped, or a prior approve/reissue's
+    dynsec publish silently failed to persist for some other reason) —
+    without this, a broker restart with an intact Firestore but an empty
+    dynsec store would lock out every previously-approved node until
+    someone noticed and manually re-approved each one.
+    """
+    nodes = await fstore.collection_list("nodes", approval_status="approved")
+    if not nodes:
+        return
+    ok, failed = 0, 0
+    for node in nodes:
+        node_id = node.get("node_id")
+        if not node_id:
+            continue
+        key_doc = await fstore.doc_get("node_keys", node_id)
+        if not key_doc or not key_doc.get("api_key"):
+            logger.warning(f"dynsec reconcile: node {node_id!r} is approved but has no node_keys entry — skipping")
+            continue
+        try:
+            await upsert_node_client(node_id, key_doc["api_key"])
+            ok += 1
+        except DynsecError as e:
+            failed += 1
+            logger.error(f"dynsec reconcile: failed to sync node {node_id!r}: {e}")
+    logger.info(f"dynsec reconcile: {ok} node(s) synced, {failed} failed")

drb-c2-core/app/internal/mqtt_handler.py

@@ -1,6 +1,6 @@
 import asyncio
 import json
-from datetime import datetime, timezone
+from datetime import datetime, timezone, timedelta
 from typing import Optional
 import paho.mqtt.client as mqtt
 from app.config import settings
@@ -33,6 +33,10 @@ class MQTTHandler:
             client.subscribe("nodes/+/checkin",     qos=1)
             client.subscribe("nodes/+/status",      qos=1)
             client.subscribe("nodes/+/metadata",    qos=1)
+            # TODO(mqtt-cutover): drop this subscribe once the enrollment/HTTP
+            # credentials flow (routers/enrollment.py) is stable in prod and
+            # node-26 (the one live node) has been migrated. See
+            # MQTT-PUBLIC-AUTH-PLAN.md "Rollout order" step 6.
             client.subscribe("nodes/+/key_request", qos=1)
             logger.info("MQTT connected — subscribed to node topics.")
         else:
@@ -94,6 +98,11 @@ class MQTTHandler:
                 "last_seen": now.isoformat(),
                 "assigned_system_id": None,
                 "approval_status": "pending",
+                "node_type": payload.get("node_type", "fixed"),
+                "enforce_override_timeout": payload.get("enforce_override_timeout", True),
+                "is_overridden": False,
+                "override_system_id": None,
+                "override_timeout_at": None,
             }
             await fstore.doc_set("nodes", node_id, doc, merge=False)
             logger.info(f"New node registered: {node_id} — pending admin approval.")
@@ -111,6 +120,34 @@ class MQTTHandler:
                 elif existing.get("approval_status") == "approved":
                     # Approved but not yet configured — restore reachable status after reboot
                     updates["status"] = "unconfigured"
+            
+            node_type = payload.get("node_type", existing.get("node_type", "fixed"))
+            enforce_timeout = payload.get("enforce_override_timeout", existing.get("enforce_override_timeout", True))
+            is_overridden = payload.get("is_overridden", False)
+            override_system_id = payload.get("override_system_id")
+            
+            updates["node_type"] = node_type
+            updates["enforce_override_timeout"] = enforce_timeout
+
+            if node_type == "portable":
+                updates["is_overridden"] = False
+                updates["override_system_id"] = None
+                updates["override_timeout_at"] = None
+            else:
+                updates["is_overridden"] = is_overridden
+                updates["override_system_id"] = override_system_id
+                
+                if is_overridden:
+                    existing_timeout = existing.get("override_timeout_at")
+                    existing_override_id = existing.get("override_system_id")
+                    if enforce_timeout:
+                        if not existing_timeout or existing_override_id != override_system_id:
+                            updates["override_timeout_at"] = (now + timedelta(hours=24)).isoformat()
+                    else:
+                        updates["override_timeout_at"] = None
+                else:
+                    updates["override_timeout_at"] = None
+
             await fstore.doc_update("nodes", node_id, updates)
 
         # NOTE: discord_connected in checkins is informational only — do NOT release the
@@ -221,6 +258,11 @@ class MQTTHandler:
     # ------------------------------------------------------------------
     # Key request — re-deliver an existing approved key to a node that
     # lost its credentials (e.g. after a directory move / fresh volume)
+    # TODO(mqtt-cutover): remove this handler + publish_node_key() below,
+    # and the key_request subscribe above, in the separate post-cutover
+    # pass called out in MQTT-PUBLIC-AUTH-PLAN.md. Left in place for now so
+    # node-26 (currently live, using the shared-password MQTT path) keeps
+    # working until the enrollment flow has replaced it in prod.
     # ------------------------------------------------------------------
 
     async def _handle_key_request(self, node_id: str):
@@ -253,7 +295,10 @@ class MQTTHandler:
             logger.warning(f"MQTT not connected — could not push config to {node_id}")
 
     def publish_node_key(self, node_id: str, api_key: str):
-        """Publish the provisioned API key to the node (retained so it survives reconnects)."""
+        """Publish the provisioned API key to the node (retained so it survives reconnects).
+        TODO(mqtt-cutover): dead once nodes.py's callers switch to the HTTP
+        credentials poll (routers/enrollment.py) exclusively. See note above
+        _handle_key_request."""
         topic = f"nodes/{node_id}/api_key"
         if self._client and self._connected:
             self._client.publish(topic, json.dumps({"api_key": api_key}), qos=2, retain=True)

drb-c2-core/app/internal/node_sweeper.py

@@ -55,3 +55,35 @@ async def _sweep():
             logger.info(f"Node {node_id} marked offline (last seen: {last_seen.isoformat()})")
             from app.routers.tokens import release_token
             await release_token(node_id)
+            continue
+
+        # Check for expired system overrides (only for fixed nodes with timeout enforced)
+        override_timeout_raw = node.get("override_timeout_at")
+        enforce_timeout = node.get("enforce_override_timeout", True)
+        node_type = node.get("node_type", "fixed")
+
+        if override_timeout_raw and enforce_timeout and node_type != "portable":
+            if isinstance(override_timeout_raw, str):
+                override_timeout = datetime.fromisoformat(override_timeout_raw)
+            else:
+                override_timeout = override_timeout_raw
+            if override_timeout.tzinfo is None:
+                override_timeout = override_timeout.replace(tzinfo=timezone.utc)
+
+            if datetime.now(timezone.utc) > override_timeout:
+                node_id = node.get("node_id")
+                assigned_system_id = node.get("assigned_system_id")
+                logger.info(f"Node {node_id} override has expired. Reverting to system {assigned_system_id}.")
+
+                # Push the original assigned config if it exists
+                if assigned_system_id:
+                    system_doc = await fstore.doc_get("systems", assigned_system_id)
+                    if system_doc:
+                        from app.internal.mqtt_handler import mqtt_handler
+                        mqtt_handler.push_config(node_id, system_doc)
+
+                await fstore.doc_update("nodes", node_id, {
+                    "is_overridden": False,
+                    "override_system_id": None,
+                    "override_timeout_at": None,
+                })

drb-c2-core/app/main.py

@@ -9,8 +9,14 @@ from app.internal.summarizer import summarizer_loop
 from app.internal.vocabulary_learner import vocabulary_induction_loop
 from app.internal.recorrelation_sweep import recorrelation_loop
 from app.config import settings
-from app.internal.auth import require_firebase_token, require_service_or_firebase_token
+from app.internal.auth import (
+    require_firebase_token,
+    require_service_or_firebase_token,
+    require_node_service_or_firebase_token,
+)
 from app.routers import nodes, systems, calls, upload, tokens, incidents, alerts, admin, trips, places, links, users
+from app.routers import enrollment
+from app.internal import dynsec
 from app.internal import firestore as fstore
 
 
@@ -36,6 +42,22 @@ async def lifespan(app: FastAPI):
     logger.info("DRB C2 Core starting.")
     await _release_orphaned_tokens()
 
+    # dynsec bootstrap + reconcile — must happen before mqtt_handler.connect()
+    # so that by the time the app is serving requests, c2-core's own dynsec
+    # client/roles exist and every already-approved node's dynsec client
+    # matches Firestore (see app/internal/dynsec.py "TWO-SOURCES-OF-TRUTH").
+    # Non-fatal by design: if the broker or MQTT_DYNSEC_ADMIN_PASS isn't
+    # reachable/configured yet (e.g. first-ever deploy, mosquitto still
+    # starting), log loudly and keep booting rather than crash-looping
+    # c2-core itself — mqtt_handler.connect() below has its own retry loop
+    # and node approval/reissue endpoints fail loudly on their own if dynsec
+    # calls fail later, so nothing here is silently swallowed forever.
+    try:
+        await dynsec.ensure_roles_and_c2core_grant()
+        await dynsec.reconcile_all()
+    except dynsec.DynsecError as e:
+        logger.error(f"dynsec bootstrap/reconcile failed — node approval/reissue will fail until this is resolved: {e}")
+
     await mqtt_handler.connect()
     sweeper_task      = asyncio.create_task(sweeper_loop())
     summarizer_task   = asyncio.create_task(summarizer_loop())
@@ -63,7 +85,11 @@ app.add_middleware(
 )
 
 app.include_router(nodes.router,     dependencies=[Depends(require_service_or_firebase_token)])
-app.include_router(systems.router,   dependencies=[Depends(require_service_or_firebase_token)])
+# systems is the one router edge nodes read directly (system_cacher.py builds
+# the OP25 config from it), so its gate also accepts a per-node api_key. The
+# write routes inside carry their own require_admin_token, so nodes get read
+# access only.
+app.include_router(systems.router,   dependencies=[Depends(require_node_service_or_firebase_token)])
 app.include_router(calls.router,     dependencies=[Depends(require_service_or_firebase_token)])
 app.include_router(tokens.router,    dependencies=[Depends(require_service_or_firebase_token)])
 app.include_router(incidents.router, dependencies=[Depends(require_service_or_firebase_token)])
@@ -74,6 +100,16 @@ app.include_router(upload.router)    # auth is per-node, handled inline
 app.include_router(admin.router)     # auth is per-endpoint (read: firebase, write: admin)
 app.include_router(users.router)     # auth: admin only
 app.include_router(links.router)     # auth is per-endpoint (generate: firebase, resolve: service key)
+app.include_router(enrollment.router)  # public; auth is the enrollment/pickup-secret tokens, checked inline
+# NOTE: there used to be an app.routers.mqtt_auth router here (an HTTP
+# backend for the mosquitto-go-auth plugin). That plugin's upstream project
+# is archived (no CVE patches) and was rejected for an internet-facing
+# broker — see MQTT-PUBLIC-AUTH-PLAN.md. MQTT auth is now mosquitto's own
+# built-in dynamic-security plugin (app/internal/dynsec.py talks to it over
+# MQTT control topics, not HTTP), so there is nothing at /internal/mqtt/*
+# anymore. Caddy's Caddyfile.j2 still 404s /internal/* on api.<domain> as
+# defence in depth even though nothing calls it today — cheap insurance
+# against a future /internal/* route being added and forgotten there.
 
 
 @app.get("/health")

drb-c2-core/app/models.py

@@ -16,6 +16,11 @@ class NodeRecord(BaseModel):
     configured: bool = False
     last_seen: Optional[datetime] = None
     assigned_system_id: Optional[str] = None
+    node_type: str = "fixed"  # fixed or portable
+    enforce_override_timeout: bool = True
+    is_overridden: bool = False
+    override_system_id: Optional[str] = None
+    override_timeout_at: Optional[datetime] = None
 
 
 class CommandPayload(BaseModel):

drb-c2-core/app/routers/enrollment.py

@@ -0,0 +1,184 @@
+"""
+Node self-enrollment — the public replacement for the old shared-MQTT-
+password flow (see MQTT-PUBLIC-AUTH-PLAN.md "Enrollment flow").
+
+  1. POST /nodes/enroll          (X-Enrollment-Token: <fleet-wide token>)
+     First-boot node upserts itself as `approval_status: pending` and gets
+     back a one-time pickup_secret. Only its hash is persisted.
+  2. GET  /nodes/{id}/credentials (X-Pickup-Secret: <secret from step 1>)
+     Node polls this with backoff until an admin approves it in the
+     frontend (existing nodes.py approve_node() flow — unchanged, still
+     writes node_keys/{id}.api_key) and then reads its api_key back.
+
+These two endpoints are meant to be public (unlike the dynsec control-plane
+traffic in app/internal/dynsec.py, which never leaves the docker-internal
+MQTT bridge) — that's the whole point of moving off WireGuard-per-node.
+Auth is the token headers checked inline below, not the app-wide
+Firebase/service-key dependency the rest of routers/nodes.py uses.
+"""
+import hashlib
+import secrets
+import time
+from typing import Optional
+from fastapi import APIRouter, HTTPException, Header, Request
+from pydantic import BaseModel
+from app.config import settings
+from app.internal import firestore as fstore
+from app.internal.logger import logger
+
+router = APIRouter(prefix="/nodes", tags=["enrollment"])
+
+
+# ---------------------------------------------------------------------------
+# Per-source-IP token bucket for /nodes/enroll.
+#
+# c2-core has no rate-limiting dependency anywhere today (see
+# MQTT-PUBLIC-AUTH-PLAN.md); this is a deliberately small (~30 line)
+# in-memory limiter rather than a new library. Known limitations:
+#   - per-process: with more than one c2-core instance, each has its own
+#     bucket, so real throughput is (limit x instance count). Fine today —
+#     there is exactly one instance.
+#   - resets on every restart/redeploy — not persisted anywhere.
+# Good enough to blunt casual guessing of node_ids against the fleet token;
+# not a substitute for a real edge/WAF rate limiter if this endpoint is
+# ever seriously targeted.
+# ---------------------------------------------------------------------------
+class _TokenBucket:
+    def __init__(self, capacity: int, refill_per_sec: float):
+        self.capacity = capacity
+        self.refill_per_sec = refill_per_sec
+        self._buckets: dict[str, tuple[float, float]] = {}  # key -> (tokens, last_refill_ts)
+
+    def allow(self, key: str) -> bool:
+        now = time.monotonic()
+        tokens, last_ts = self._buckets.get(key, (float(self.capacity), now))
+        tokens = min(self.capacity, tokens + (now - last_ts) * self.refill_per_sec)
+        if tokens < 1:
+            self._buckets[key] = (tokens, now)
+            return False
+        self._buckets[key] = (tokens - 1, now)
+        return True
+
+
+# Burst of 5, refilling 1/minute — enrollment is a first-boot, once-per-node
+# event, so a legitimate node never needs more than a handful of attempts.
+_enroll_limiter = _TokenBucket(capacity=5, refill_per_sec=1 / 60)
+
+
+def _hash_secret(secret: str) -> str:
+    return hashlib.sha256(secret.encode()).hexdigest()
+
+
+class EnrollRequest(BaseModel):
+    node_id: str
+    name: Optional[str] = None
+    lat: float = 0.0
+    lon: float = 0.0
+
+
+class EnrollResponse(BaseModel):
+    node_id: str
+    pickup_secret: str
+    approval_status: str
+
+
+@router.post("/enroll", response_model=EnrollResponse)
+async def enroll_node(
+    body: EnrollRequest,
+    request: Request,
+    x_enrollment_token: Optional[str] = Header(None),
+):
+    client_ip = request.client.host if request.client else "unknown"
+    if not _enroll_limiter.allow(client_ip):
+        raise HTTPException(429, "Too many enrollment attempts. Try again later.")
+
+    if not settings.enrollment_token:
+        raise HTTPException(503, "Enrollment is not configured on this server.")
+    if not x_enrollment_token or not secrets.compare_digest(x_enrollment_token, settings.enrollment_token):
+        logger.warning(f"Enroll 401: bad/missing enrollment token from {client_ip} for node_id={body.node_id!r}")
+        raise HTTPException(401, "Invalid or missing X-Enrollment-Token")
+
+    node_id = body.node_id.strip()
+    if not node_id:
+        raise HTTPException(400, "node_id is required")
+
+    existing = await fstore.doc_get("nodes", node_id)
+
+    # -------------------------------------------------------------------
+    # CRITICAL GUARD — do not remove or weaken this check.
+    #
+    # An already-approved node_id must NEVER get a fresh pickup_secret off
+    # the fleet-wide enrollment token alone. The fleet token is shared by
+    # every node (it ships in every node's .env / setup.sh prompt), so it
+    # is the credential most likely to leak. Without this guard, a leaked
+    # fleet token plus a guessable node_id (node-001, node-002, ...) would
+    # let an attacker "re-enroll" a live, already-approved node and race
+    # the real node to GET /nodes/{id}/credentials — stealing its actual
+    # api_key before the legitimate device ever asks.
+    #
+    # Recovery for an approved node goes through the existing admin-only
+    # POST /nodes/{id}/reissue-key instead (routers/nodes.py), which
+    # requires a Firebase admin token, not the fleet token.
+    # -------------------------------------------------------------------
+    if existing and existing.get("approval_status") == "approved":
+        logger.warning(
+            f"Enroll refused: node_id={node_id!r} is already approved — "
+            f"refusing to issue a new pickup_secret from the fleet token alone "
+            f"(source_ip={client_ip})"
+        )
+        raise HTTPException(
+            403,
+            "Node is already approved. This endpoint cannot re-issue credentials "
+            "for an approved node from the enrollment token alone — use admin "
+            "key reissue.",
+        )
+
+    pickup_secret = secrets.token_hex(24)
+    doc = {
+        "node_id": node_id,
+        "name": body.name or (existing or {}).get("name") or node_id,
+        "lat": body.lat or (existing or {}).get("lat", 0.0),
+        "lon": body.lon or (existing or {}).get("lon", 0.0),
+        "approval_status": (existing or {}).get("approval_status", "pending"),
+        "pickup_secret_hash": _hash_secret(pickup_secret),
+    }
+    # approval_status stays "pending" for a brand-new node; if it's an
+    # existing "pending" or "rejected" node re-enrolling (e.g. lost its
+    # pickup_secret before an admin ever approved it), leave whatever
+    # status it already has rather than silently flipping "rejected" back
+    # to "pending" — that decision belongs to an admin, not this endpoint.
+    await fstore.doc_set("nodes", node_id, doc, merge=True)
+    logger.info(f"Node enrolled: {node_id} (status={doc['approval_status']}, source_ip={client_ip})")
+
+    return EnrollResponse(node_id=node_id, pickup_secret=pickup_secret, approval_status=doc["approval_status"])
+
+
+class CredentialsResponse(BaseModel):
+    approval_status: str
+    api_key: Optional[str] = None
+
+
+@router.get("/{node_id}/credentials", response_model=CredentialsResponse)
+async def get_node_credentials(node_id: str, x_pickup_secret: Optional[str] = Header(None)):
+    if not x_pickup_secret:
+        raise HTTPException(401, "Missing X-Pickup-Secret header")
+
+    node = await fstore.doc_get("nodes", node_id)
+    if not node or not node.get("pickup_secret_hash"):
+        raise HTTPException(404, "Unknown node, or node was never enrolled via POST /nodes/enroll")
+
+    if not secrets.compare_digest(_hash_secret(x_pickup_secret), node["pickup_secret_hash"]):
+        raise HTTPException(401, "Invalid pickup secret")
+
+    approval_status = node.get("approval_status", "pending")
+    if approval_status != "approved":
+        return CredentialsResponse(approval_status=approval_status)
+
+    key_doc = await fstore.doc_get("node_keys", node_id)
+    if not key_doc or not key_doc.get("api_key"):
+        # Approved but no key yet — shouldn't normally happen, approve_node()
+        # always writes node_keys in the same call that sets approved. Treat
+        # it as "keep polling" rather than erroring the node's retry loop.
+        return CredentialsResponse(approval_status=approval_status)
+
+    return CredentialsResponse(approval_status=approval_status, api_key=key_doc["api_key"])

drb-c2-core/app/routers/nodes.py

@@ -1,9 +1,12 @@
 import secrets
 from typing import Optional
 from fastapi import APIRouter, HTTPException, Depends, Query
+from pydantic import BaseModel
 from app.models import CommandPayload
 from app.internal import firestore as fstore
 from app.internal.mqtt_handler import mqtt_handler
+from app.internal import dynsec
+from app.internal.logger import logger
 from app.internal.auth import require_admin_token, require_service_key_or_admin
 from app.routers.tokens import assign_token, release_token
 
@@ -30,8 +33,23 @@ async def approve_node(node_id: str, _: dict = Depends(require_admin_token)):
         raise HTTPException(404, f"Node '{node_id}' not found.")
 
     api_key = secrets.token_hex(32)
+
+    # dynsec FIRST, Firestore second: if the broker rejects/never confirms
+    # the new client, we must not tell Firestore (and the admin UI) the
+    # node is approved with a key mosquitto doesn't actually recognise —
+    # that's exactly the silent-drift the two-sources-of-truth problem
+    # warns about. See app/internal/dynsec.py.
+    try:
+        await dynsec.upsert_node_client(node_id, api_key)
+    except dynsec.DynsecError as e:
+        logger.error(f"Approve {node_id!r}: dynsec upsert failed, NOT writing Firestore: {e}")
+        raise HTTPException(502, f"Could not provision MQTT credentials for node: {e}")
+
     await fstore.doc_set("node_keys", node_id, {"node_id": node_id, "api_key": api_key}, merge=False)
     await fstore.doc_update("nodes", node_id, {"approval_status": "approved"})
+    # TODO(mqtt-cutover): drop this MQTT push once nodes pull their key via
+    # GET /nodes/{id}/credentials (routers/enrollment.py) exclusively — see
+    # MQTT-PUBLIC-AUTH-PLAN.md "Rollout order" step 6. Kept for node-26.
     mqtt_handler.publish_node_key(node_id, api_key)
     return {"ok": True}
 
@@ -41,6 +59,11 @@ async def delete_node(node_id: str, _: dict = Depends(require_admin_token)):
     node = await fstore.doc_get("nodes", node_id)
     if not node:
         raise HTTPException(404, f"Node '{node_id}' not found.")
+    try:
+        await dynsec.delete_node_client(node_id)
+    except dynsec.DynsecError as e:
+        logger.error(f"Delete {node_id!r}: dynsec deleteClient failed, NOT deleting Firestore docs: {e}")
+        raise HTTPException(502, f"Could not revoke MQTT credentials for node: {e}")
     await fstore.doc_delete("node_keys", node_id)
     await fstore.doc_delete("nodes", node_id)
 
@@ -101,7 +124,15 @@ async def reissue_node_key(node_id: str, _: dict = Depends(require_admin_token))
     if not node:
         raise HTTPException(404, f"Node '{node_id}' not found.")
     api_key = secrets.token_hex(32)
+    try:
+        await dynsec.upsert_node_client(node_id, api_key)
+    except dynsec.DynsecError as e:
+        logger.error(f"Reissue {node_id!r}: dynsec upsert failed, NOT writing Firestore: {e}")
+        raise HTTPException(502, f"Could not update MQTT credentials for node: {e}")
     await fstore.doc_set("node_keys", node_id, {"node_id": node_id, "api_key": api_key}, merge=False)
+    # TODO(mqtt-cutover): drop this MQTT push once nodes pull their key via
+    # GET /nodes/{id}/credentials (routers/enrollment.py) exclusively — see
+    # MQTT-PUBLIC-AUTH-PLAN.md "Rollout order" step 6. Kept for node-26.
     mqtt_handler.publish_node_key(node_id, api_key)
     return {"ok": True}
 
@@ -126,10 +157,13 @@ async def assign_system(
     if not system:
         raise HTTPException(404, f"System '{system_id}' not found.")
 
-    # Include hardware preset in the push so the edge node applies it when
-    # generating the OP25 config. Strip it from the system doc first so it
-    # doesn't collide with SystemConfig field validation on the node side.
-    push_payload = {**system, "hardware_preset": hardware_preset}
+    # Include hardware preset, node type, and enforce timeout in the push
+    push_payload = {
+        **system, 
+        "hardware_preset": hardware_preset,
+        "node_type": node.get("node_type", "fixed"),
+        "enforce_override_timeout": node.get("enforce_override_timeout", True),
+    }
     if ppm_override is not None:
         push_payload["ppm_override"] = ppm_override
     mqtt_handler.push_config(node_id, push_payload)
@@ -145,3 +179,89 @@ async def assign_system(
     await fstore.doc_update("nodes", node_id, node_updates)
 
     return {"ok": True}
+
+
+class NodeUpdateBody(BaseModel):
+    node_type: Optional[str] = None
+    enforce_override_timeout: Optional[bool] = None
+
+
+@router.patch("/{node_id}")
+async def update_node(
+    node_id: str,
+    body: NodeUpdateBody,
+    _: dict = Depends(require_admin_token),
+):
+    node = await fstore.doc_get("nodes", node_id)
+    if not node:
+        raise HTTPException(404, f"Node '{node_id}' not found.")
+
+    updates = body.model_dump(exclude_unset=True)
+    if not updates:
+        return {"ok": True}
+
+    await fstore.doc_update("nodes", node_id, updates)
+
+    # Re-push config to apply new node settings locally
+    updated_node = await fstore.doc_get("nodes", node_id)
+    assigned_system_id = updated_node.get("assigned_system_id")
+    if assigned_system_id:
+        system = await fstore.doc_get("systems", assigned_system_id)
+        if system:
+            push_payload = {
+                **system,
+                "hardware_preset": updated_node.get("hardware_preset", "rtl-sdr-v3"),
+                "node_type": updated_node.get("node_type", "fixed"),
+                "enforce_override_timeout": updated_node.get("enforce_override_timeout", True),
+            }
+            if updated_node.get("ppm_override") is not None:
+                push_payload["ppm_override"] = updated_node["ppm_override"]
+            mqtt_handler.push_config(node_id, push_payload)
+
+    return {"ok": True}
+
+
+class AckOverrideBody(BaseModel):
+    timeout_minutes: int = 1440
+
+
+@router.post("/{node_id}/override/ack")
+async def ack_override(
+    node_id: str,
+    body: AckOverrideBody,
+    _: dict = Depends(require_service_key_or_admin),
+):
+    node = await fstore.doc_get("nodes", node_id)
+    if not node:
+        raise HTTPException(404, f"Node '{node_id}' not found.")
+
+    from datetime import datetime, timezone, timedelta
+    new_timeout = datetime.now(timezone.utc) + timedelta(minutes=body.timeout_minutes)
+    
+    await fstore.doc_update("nodes", node_id, {
+        "override_timeout_at": new_timeout.isoformat()
+    })
+    return {"ok": True, "override_timeout_at": new_timeout.isoformat()}
+
+
+@router.post("/{node_id}/override/reset")
+async def reset_override(
+    node_id: str,
+    _: dict = Depends(require_service_key_or_admin),
+):
+    node = await fstore.doc_get("nodes", node_id)
+    if not node:
+        raise HTTPException(404, f"Node '{node_id}' not found.")
+
+    assigned_system_id = node.get("assigned_system_id")
+    if assigned_system_id:
+        system = await fstore.doc_get("systems", assigned_system_id)
+        if system:
+            mqtt_handler.push_config(node_id, system)
+
+    await fstore.doc_update("nodes", node_id, {
+        "is_overridden": False,
+        "override_system_id": None,
+        "override_timeout_at": None,
+    })
+    return {"ok": True}

drb-c2-core/mosquitto/acl.conf

@@ -1,19 +0,0 @@
-# -----------------------------------------------------------------------
-# Mosquitto ACL — DRB C2 Server
-# -----------------------------------------------------------------------
-# Two principals:
-#   drb-c2-core  — the backend service; needs full broker access
-#   drb-node     — shared credential for all edge nodes; scoped to their
-#                  own namespace via MQTT client ID (%c = NODE_ID)
-# -----------------------------------------------------------------------
-
-# C2-core service — full read/write on every topic
-user drb-c2-core
-topic readwrite #
-
-# Edge nodes — each node may only read/write topics under nodes/<its-own-ID>/
-# Mosquitto substitutes %c with the connecting client's MQTT client ID at
-# runtime. Edge nodes set client_id = NODE_ID in mqtt_manager.py, so this
-# cryptographically prevents node-A from publishing to nodes/node-B/api_key
-# or any other node's namespace.
-pattern readwrite nodes/%c/#

drb-c2-core/mosquitto/entrypoint.sh

@@ -1,37 +0,0 @@
-#!/bin/sh
-# Mosquitto entrypoint — generates /mosquitto/config/passwd from env vars
-# before handing off to the broker process.
-#
-# Required environment variables (set in docker-compose.yml):
-#   MQTT_C2_USER   — username for the drb-c2-core service
-#   MQTT_C2_PASS   — password for the drb-c2-core service
-#   MQTT_NODE_USER — shared username for all edge nodes
-#   MQTT_NODE_PASS — shared password for all edge nodes
-set -e
-
-PASSWD_FILE=/tmp/passwd
-
-# Remove any stale file so we start clean on every container start
-rm -f "$PASSWD_FILE"
-
-if [ -z "$MQTT_C2_USER" ] || [ -z "$MQTT_C2_PASS" ]; then
-    echo "ERROR: MQTT_C2_USER and MQTT_C2_PASS must be set" >&2
-    exit 1
-fi
-
-if [ -z "$MQTT_NODE_USER" ] || [ -z "$MQTT_NODE_PASS" ]; then
-    echo "ERROR: MQTT_NODE_USER and MQTT_NODE_PASS must be set" >&2
-    exit 1
-fi
-
-# -c creates/overwrites the file; subsequent calls append without -c
-mosquitto_passwd -c -b "$PASSWD_FILE" "$MQTT_C2_USER"   "$MQTT_C2_PASS"
-mosquitto_passwd    -b "$PASSWD_FILE" "$MQTT_NODE_USER" "$MQTT_NODE_PASS"
-
-# mosquitto_passwd creates the file 0600 (root-only); mosquitto drops to
-# the mosquitto user before reading it, so make it world-readable.
-chmod 644 "$PASSWD_FILE"
-
-echo "Mosquitto: password file written for users: $MQTT_C2_USER, $MQTT_NODE_USER"
-
-exec /usr/sbin/mosquitto -c /mosquitto/config/mosquitto.conf

drb-c2-core/mosquitto/mosquitto.conf

@@ -1,11 +1,43 @@
-listener 1883
+# Auth: mosquitto's own built-in dynamic-security plugin — NOT
+# mosquitto-go-auth (that project is archived upstream, no CVE patches;
+# rejected for an internet-facing broker). This plugin ships in and is
+# maintained alongside the official eclipse-mosquitto image itself.
+# See MQTT-PUBLIC-AUTH-PLAN.md and app/internal/dynsec.py for the full
+# design (bootstrap, roles, the two-sources-of-truth reconcile).
+#
+# Plugin path is DERIVED FROM SOURCE (docker/2.1-alpine/Dockerfile in
+# eclipse-mosquitto/mosquitto), not observed by running the image —
+# nothing in this project executes/pulls images from this machine. Verify
+# it on first real deploy: `docker compose logs mosquitto` will say
+# "Error: Unable to load plugin" at the exact path below if it's wrong for
+# whatever patch tag ends up pinned.
+plugin /usr/lib/mosquitto_dynamic_security.so
+# Lives on the same persistent volume as `persistence_location` below —
+# one durable volume for all broker state, survives redeploys.
+plugin_opt_config_file /mosquitto/data/dynamic-security.json
+
 allow_anonymous false
+# No password_file/acl_file directive anywhere in this file — the plugin
+# above is the only registered auth backend. There is no "coexist" mode:
+# nothing else is registered to conflict with it.
 
-# Credentials and ACLs are generated/mounted at container startup
-password_file /tmp/passwd
-acl_file      /mosquitto/config/acl.conf
+# Internal, plaintext — c2-core's own connection only (its dynsec-admin
+# control-plane calls AND its regular data-plane pub/sub both use this).
+# Never published to the host in prod (docker-compose.prod.yml removes the
+# port mapping); external nodes use the TLS listener below instead.
+listener 1883
+
+# Public, TLS — edge nodes connect here as username=node_id, password=api_key
+# (the same credential /upload already trusts via node_keys), authorized by
+# the "node" dynsec role (nodes/%u/# — %u is the dynsec-authenticated
+# username, fixing the old %c-based ACL's client-ID-spoofing hole). Cert/key
+# come from infra/ansible's Caddy cert-sync unit; see
+# MQTT-PUBLIC-AUTH-PLAN.md "Infra" and the "Rollout order" cert-verification
+# step for what happens before that cert exists.
+listener 8883
+certfile /mosquitto/certs/mqtt.crt
+keyfile  /mosquitto/certs/mqtt.key
 
-# Persist retained messages (e.g. api_key, node status) across broker restarts
 persistence true
 persistence_location /mosquitto/data/
 

drb-frontend/Dockerfile

@@ -4,6 +4,17 @@ WORKDIR /app
 COPY package.json ./
 RUN npm install
 COPY . .
+
+# Build-time public vars — baked into the Next.js bundle by the CI workflow
+ARG NEXT_PUBLIC_C2_URL
+ARG NEXT_PUBLIC_FIREBASE_API_KEY
+ARG NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN
+ARG NEXT_PUBLIC_FIREBASE_PROJECT_ID
+ARG NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET
+ARG NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID
+ARG NEXT_PUBLIC_FIREBASE_APP_ID
+ARG NEXT_PUBLIC_FIRESTORE_DATABASE
+
 RUN npm run build
 
 FROM node:20-slim AS runner

drb-frontend/app/nodes/[id]/page.tsx

@@ -192,6 +192,54 @@ export default function NodeDetailPage() {
         <StatusBadge status={node.status} />
       </div>
 
+      {/* Override Warning */}
+      {node.is_overridden && (
+        <div className="bg-yellow-950/40 border border-yellow-800/60 rounded-lg p-4 font-mono text-sm space-y-3">
+          <div className="flex items-center gap-2 text-yellow-400 font-semibold">
+            <span className="text-base">⚠</span>
+            <span>Local System Override Active</span>
+          </div>
+          <p className="text-gray-400 text-xs leading-relaxed">
+            This node is operating on a local system override. 
+            {node.override_timeout_at ? (
+              <> Resets automatically on: <span className="text-white font-bold">{new Date(node.override_timeout_at).toLocaleString()}</span>.</>
+            ) : (
+              <> No timeout is currently enforced (permanent override).</>
+            )}
+          </p>
+          <div className="flex gap-2">
+            {node.override_timeout_at && (
+              <button
+                onClick={async () => {
+                  try {
+                    await c2api.ackOverride(id, 1440);
+                  } catch (e) {
+                    alert("Failed to extend timer.");
+                  }
+                }}
+                className="px-3 py-1 bg-yellow-800 hover:bg-yellow-700 text-white rounded text-xs transition-colors"
+              >
+                Ack (Reset 24h Timer)
+              </button>
+            )}
+            <button
+              onClick={async () => {
+                if (confirm("Force this node to revert back to its assigned system config?")) {
+                  try {
+                    await c2api.resetOverride(id);
+                  } catch (e) {
+                    alert("Failed to reset override.");
+                  }
+                }
+              }}
+              className="px-3 py-1 bg-red-900 hover:bg-red-800 text-red-200 rounded text-xs transition-colors"
+            >
+              Force Revert Config
+            </button>
+          </div>
+        </div>
+      )}
+
       {/* Info */}
       <div className="bg-gray-900 border border-gray-800 rounded-lg divide-y divide-gray-800 font-mono text-sm">
         {[
@@ -199,6 +247,8 @@ export default function NodeDetailPage() {
           ["Location",  `${node.lat}, ${node.lon}`],
           ["Last Seen", node.last_seen ? new Date(node.last_seen).toLocaleString() : "never"],
           ["Configured", node.configured ? "Yes" : "No"],
+          ["Node Type", node.node_type ?? "fixed"],
+          ...(node.node_type !== "portable" ? [["Enforce Timeout", node.enforce_override_timeout ? "Yes" : "No"]] : []),
         ].map(([label, value]) => (
           <div key={label} className="flex justify-between px-4 py-2.5">
             <span className="text-gray-500">{label}</span>

drb-frontend/components/NodeCard.tsx

@@ -45,6 +45,18 @@ export function NodeCard({ node, system }: Props) {
             ⚠ Needs configuration
           </div>
         )}
+        {node.is_overridden && (
+          <div className="mt-3 text-xs text-yellow-500 font-mono border-t border-gray-800 pt-2 flex justify-between">
+            <span>⚠ Local Override</span>
+            {node.override_timeout_at ? (
+              <span className="text-gray-500">
+                Resets: {new Date(node.override_timeout_at).toLocaleTimeString()}
+              </span>
+            ) : (
+              <span className="text-gray-500">Permanent</span>
+            )}
+          </div>
+        )}
       </div>
     </Link>
   );

drb-frontend/components/NodeConfigModal.tsx

@@ -17,9 +17,11 @@ const PRESETS = [
 ];
 
 export function NodeConfigModal({ node, systems, onClose }: Props) {
-  const [systemId, setSystemId] = useState("");
-  const [preset, setPreset] = useState("rtl-sdr-v3");
-  const [ppm, setPpm] = useState("0");
+  const [systemId, setSystemId] = useState(node.assigned_system_id ?? "");
+  const [preset, setPreset] = useState(node.hardware_preset ?? "rtl-sdr-v3");
+  const [ppm, setPpm] = useState(node.ppm_override ? String(node.ppm_override) : "0");
+  const [nodeType, setNodeType] = useState(node.node_type ?? "");
+  const [enforceTimeout, setEnforceTimeout] = useState(node.enforce_override_timeout ?? true);
   const [saving, setSaving] = useState(false);
   const [error, setError] = useState<string | null>(null);
 
@@ -32,6 +34,10 @@ export function NodeConfigModal({ node, systems, onClose }: Props) {
     setSaving(true);
     setError(null);
     try {
+      await c2api.updateNode(node.node_id, {
+        node_type: nodeType,
+        enforce_override_timeout: enforceTimeout,
+      });
       await c2api.assignSystem(node.node_id, systemId, preset, ppmOverride);
       onClose();
     } catch (err) {
@@ -100,12 +106,41 @@ export function NodeConfigModal({ node, systems, onClose }: Props) {
             />
           </div>
 
+          <div>
+            <label className="block text-xs text-gray-400 mb-1">Node Type *</label>
+            <select
+              value={nodeType}
+              onChange={(e) => setNodeType(e.target.value)}
+              className="w-full bg-gray-800 border border-gray-700 rounded-lg px-3 py-2 text-white text-sm focus:outline-none focus:border-indigo-500"
+              required
+            >
+              <option value="">Select node type...</option>
+              <option value="fixed">Fixed Node (Standard)</option>
+              <option value="portable">Portable Node (Handheld)</option>
+            </select>
+          </div>
+
+          {nodeType === "fixed" && (
+            <div className="flex items-center gap-2 py-1">
+              <input
+                type="checkbox"
+                id="enforceTimeout"
+                checked={enforceTimeout}
+                onChange={(e) => setEnforceTimeout(e.target.checked)}
+                className="rounded bg-gray-800 border-gray-700 text-indigo-600 focus:ring-indigo-500 focus:ring-offset-gray-900"
+              />
+              <label htmlFor="enforceTimeout" className="text-xs text-gray-400 cursor-pointer select-none">
+                Enforce Local Override Timeout (24 hours)
+              </label>
+            </div>
+          )}
+
           {error && <p className="text-red-400 text-xs">{error}</p>}
 
           <div className="flex gap-3 pt-1">
             <button
               type="submit"
-              disabled={saving || !systemId}
+              disabled={saving || !systemId || !nodeType}
               className="flex-1 bg-indigo-600 hover:bg-indigo-500 disabled:opacity-50 text-white rounded-lg py-2 text-sm font-semibold transition-colors"
             >
               {saving ? "Saving…" : "Assign & Configure"}

drb-frontend/lib/c2api.ts

@@ -30,6 +30,12 @@ export const c2api = {
     if (ppmOverride !== undefined) params.set("ppm_override", String(ppmOverride));
     return request(`/nodes/${nodeId}/config/${systemId}?${params}`, { method: "POST" });
   },
+  ackOverride: (nodeId: string, timeoutMinutes: number = 1440) =>
+    request(`/nodes/${nodeId}/override/ack`, { method: "POST", body: JSON.stringify({ timeout_minutes: timeoutMinutes }) }),
+  resetOverride: (nodeId: string) =>
+    request(`/nodes/${nodeId}/override/reset`, { method: "POST" }),
+  updateNode: (id: string, body: { node_type?: string; enforce_override_timeout?: boolean }) =>
+    request(`/nodes/${id}`, { method: "PATCH", body: JSON.stringify(body) }),
 
   // Systems
   getSystems: () => request<unknown[]>("/systems"),

drb-frontend/lib/types.ts

@@ -52,6 +52,11 @@ export interface NodeRecord {
   approval_status: ApprovalStatus | null;
   hardware_preset?: string;
   ppm_override?: number | null;
+  node_type?: string;
+  enforce_override_timeout?: boolean;
+  is_overridden?: boolean;
+  override_system_id?: string | null;
+  override_timeout_at?: string | null;
 }
 
 export interface VocabularyPendingTerm {

infra/Caddyfile

@@ -1,14 +0,0 @@
-# Managed by CI — deployed to /etc/caddy/Caddyfile on the server.
-# Caddy handles TLS automatically via Let's Encrypt.
-
-api.{$DRB_DOMAIN} {
-  reverse_proxy localhost:8888 {
-    header_up X-Forwarded-For {remote_host}
-  }
-}
-
-app.{$DRB_DOMAIN} {
-  reverse_proxy localhost:3000 {
-    header_up X-Forwarded-For {remote_host}
-  }
-}

infra/ansible/roles/deploy/handlers/main.yml

@@ -0,0 +1,19 @@
+---
+# The "Deploy Caddyfile" task notifies this. Without this file the play aborts
+# with "The requested handler 'Reload Caddy' was not found" — notify does not
+# tolerate a missing handler.
+#
+# reloaded, not restarted: caddy reload swaps config with zero downtime and
+# keeps existing TLS certs/connections; a restart drops every in-flight request.
+
+- name: Reload Caddy
+  ansible.builtin.systemd_service:
+    name: caddy
+    state: reloaded
+    enabled: true
+
+# For the mqtt-cert-sync.service/.path unit files — systemd won't pick up a
+# new/changed unit file until the manager config is reloaded.
+- name: Reload systemd daemon
+  ansible.builtin.systemd_service:
+    daemon_reload: true

infra/ansible/roles/deploy/tasks/main.yml

@@ -2,12 +2,19 @@
 # First-time setup: clone repo, write secrets, pull pre-built images and start stack.
 # Images are built and pushed by Gitea CI — this role never builds on the VM.
 
-- name: Clone repo (skipped if already present)
+# update: true (was false) — with update disabled, every re-run of this playbook
+# redeployed the code that happened to be on the VM at first clone, so any fix
+# pushed to main was invisible here and the only way to ship one was CI or a
+# manual pull. force: true discards local edits made on the VM; the templated
+# .env files and Caddyfile live outside git tracking, so nothing generated by
+# this role is at risk.
+- name: Clone or update repo
   git:
     repo: "{{ repo_url }}"
     dest: "{{ app_dir }}"
     version: main
-    update: false
+    update: true
+    force: true
   become: false
 
 - name: Set ownership of app directory
@@ -59,6 +66,93 @@
     mode: "0644"
   notify: Reload Caddy
 
+# --- MQTT TLS cert sync (Caddy -> mosquitto) --------------------------------
+# See MQTT-PUBLIC-AUTH-PLAN.md "Infra". mosquitto reads its cert from this
+# directory (docker-compose.prod.yml bind-mounts it in); nothing but root can
+# read Caddy's own cert storage, so a systemd path unit + oneshot service
+# copies a readable copy out and SIGHUPs the broker on every change.
+
+# root:1883 0750, not root:root 0700. The stock eclipse-mosquitto entrypoint
+# drops privileges to the in-image `mosquitto` user (uid/gid 1883), so a
+# root-only directory makes the broker fail to read its own cert and
+# crash-loop: "Unable to load server certificate ... Permission denied".
+# The host has no `mosquitto` user, hence the numeric gid.
+- name: Create mosquitto certs directory
+  file:
+    path: /opt/drb/mosquitto-certs
+    state: directory
+    owner: root
+    group: "1883"
+    mode: "0750"
+
+# dynamic-security.json (node credentials — see app/internal/dynsec.py)
+# lives here, and mosquitto WRITES it, so this must be owned by the uid the
+# broker actually runs as (1883), not root. The earlier assumption that the
+# container runs as root was wrong — the image's entrypoint drops privileges
+# to the `mosquitto` user, which a real deploy proved by failing to read a
+# root-owned cert. Same numeric-gid reasoning as the certs directory above.
+- name: Create mosquitto data directory
+  file:
+    path: /opt/drb/mosquitto-data
+    state: directory
+    owner: "1883"
+    group: "1883"
+    mode: "0700"
+    # recurse so an existing root-owned dynamic-security.json / mosquitto.db
+    # left behind by the earlier root-owned deploy gets fixed too — chowning
+    # only the directory would leave the broker unable to rewrite them.
+    recurse: true
+
+- name: Deploy MQTT cert-sync script
+  template:
+    src: sync-mqtt-cert.sh.j2
+    dest: /opt/drb/sync-mqtt-cert.sh
+    owner: root
+    group: root
+    mode: "0700"
+
+- name: Deploy MQTT cert-sync systemd service unit
+  template:
+    src: mqtt-cert-sync.service.j2
+    dest: /etc/systemd/system/mqtt-cert-sync.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Reload systemd daemon
+
+- name: Deploy MQTT cert-sync systemd path unit
+  template:
+    src: mqtt-cert-sync.path.j2
+    dest: /etc/systemd/system/mqtt-cert-sync.path
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Reload systemd daemon
+
+# Flush the daemon-reload handler now (rather than at end-of-play) so the
+# path unit is registered and actively watching BEFORE the "Reload Caddy"
+# handler below fires and Caddy goes to obtain the mqtt.{{ domain }} cert —
+# otherwise the unit could miss the very first PathChanged event.
+- name: Apply pending handlers (systemd daemon-reload)
+  meta: flush_handlers
+
+- name: Enable and start MQTT cert-sync path unit
+  ansible.builtin.systemd_service:
+    name: mqtt-cert-sync.path
+    state: started
+    enabled: true
+
+# Best-effort initial sync in case Caddy already has a cert from a previous
+# run (e.g. re-running this playbook after the first successful deploy) —
+# the path unit only fires on a CHANGE, so it won't pick up a cert that was
+# already sitting there unchanged before it started watching. Non-fatal if
+# nothing exists yet (first-ever run, before Caddy has issued anything).
+- name: Best-effort initial MQTT cert sync
+  command: /opt/drb/sync-mqtt-cert.sh
+  register: _initial_sync
+  changed_when: "'copied cert' in _initial_sync.stdout"
+  failed_when: false
+
 - name: Log in to container registry
   command: >
     docker login {{ vault_registry_host }}

infra/ansible/roles/deploy/templates/Caddyfile.j2

@@ -1,12 +1,40 @@
-# Managed by Ansible — do not edit manually on the server.
+# Managed by Ansible — do not edit manually.
 
 api.{{ domain }} {
+  # MQTT auth is no longer an HTTP backend c2-core exposes (it moved to
+  # mosquitto's own built-in dynamic-security plugin, administered over MQTT
+  # control topics — see app/internal/dynsec.py) — there is currently no
+  # /internal/* route in c2-core at all. This block stays anyway as defence
+  # in depth: c2-core's app-wide reverse_proxy below forwards every path by
+  # default, so this guarantees any FUTURE /internal/* route (or a
+  # regression that reintroduces one) is still unreachable from the public
+  # internet unless someone also deliberately deletes this block. `route`
+  # forces top-to-bottom evaluation instead of Caddy's automatic directive
+  # sorting, so this is guaranteed to run before reverse_proxy.
+  route {
+    respond /internal/* 404
     reverse_proxy localhost:8888 {
       header_up X-Forwarded-For {remote_host}
     }
   }
+}
 
-app.{{ domain }} {
+# mqtt.{{ domain }} has no application behind it — mosquitto's TLS listener
+# (8883) is a raw MQTT socket, not HTTP, so Caddy can't reverse_proxy to it.
+# This block's only job is to make Caddy request+manage a Let's Encrypt cert
+# for the name via ACME HTTP-01, which infra/ansible's cert-sync unit then
+# copies out to mosquitto. The DNS A record for mqtt.{{ domain }} must exist
+# before this runs, or ACME issuance fails (see MQTT-PUBLIC-AUTH-PLAN.md).
+mqtt.{{ domain }} {
+  respond 404
+}
+
+# Frontend is served on the bare domain, not app.{{ domain }}: only drb and api
+# have public DNS records. A vhost for a name with no A record still starts,
+# but Caddy retries ACME against it forever and logs a failure each time.
+# To move it to app.{{ domain }}, create the A record first, then change this
+# line — the reverse_proxy target stays the same either way.
+{{ domain }} {
   reverse_proxy localhost:3000 {
     header_up X-Forwarded-For {remote_host}
   }

infra/ansible/roles/deploy/templates/c2-core.env.j2

@@ -5,6 +5,11 @@ MQTT_PORT=1883
 MQTT_USER={{ vault_mqtt_c2_user }}
 MQTT_PASS={{ vault_mqtt_c2_pass }}
 
+# Same value as mosquitto's MOSQUITTO_DYNSEC_PASSWORD (root.env.j2) — lets
+# c2-core log in as the dynsec plugin's built-in "admin" client to
+# administer node credentials. See app/internal/dynsec.py.
+MQTT_DYNSEC_ADMIN_PASS={{ vault_mqtt_dynsec_admin_pass }}
+
 # No GCP_CREDENTIALS_PATH — the VM uses Application Default Credentials
 # via the GCE metadata server. The Terraform IAM bindings grant the required roles.
 FIRESTORE_DATABASE={{ vault_firestore_database }}
@@ -15,6 +20,11 @@ GOOGLE_MAPS_API_KEY={{ vault_google_maps_api_key }}
 GEMINI_API_KEY={{ vault_gemini_api_key }}
 
 SERVICE_KEY={{ vault_service_key }}
-NODE_API_KEY={{ vault_node_api_key }}
+ENROLLMENT_TOKEN={{ vault_enrollment_token }}
 
-CORS_ORIGINS=["https://app.{{ domain }}"]
+# Bare domain, not app.<domain>: the frontend is served on {{ domain }} itself
+# (see Caddyfile.j2 — only api. and the bare name have DNS records). This said
+# app.{{ domain }} while the browser origin was https://{{ domain }}, so every
+# frontend call to the API would have failed CORS. If the frontend ever moves
+# to app.{{ domain }}, change this at the same time.
+CORS_ORIGINS=["https://{{ domain }}"]

infra/ansible/roles/deploy/templates/mqtt-cert-sync.path.j2

@@ -0,0 +1,18 @@
+# Managed by Ansible — do not edit manually.
+#
+# Watches Caddy's on-disk cert for mqtt.{{ domain }} and fires
+# mqtt-cert-sync.service on every change (initial issuance + every renewal).
+# See sync-mqtt-cert.sh.j2 for the "unverified path" caveat — if Caddy's
+# storage layout doesn't match, this unit simply never fires and mosquitto
+# keeps using its self-signed placeholder cert (see mosquitto/entrypoint.sh)
+# rather than failing loudly, so check `systemctl status mqtt-cert-sync.path`
+# after the first deploy.
+[Unit]
+Description=Watch for a renewed MQTT TLS cert from Caddy (mqtt.{{ domain }})
+
+[Path]
+PathChanged=/var/lib/caddy/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/mqtt.{{ domain }}/mqtt.{{ domain }}.crt
+Unit=mqtt-cert-sync.service
+
+[Install]
+WantedBy=multi-user.target

infra/ansible/roles/deploy/templates/mqtt-cert-sync.service.j2

@@ -0,0 +1,11 @@
+# Managed by Ansible — do not edit manually.
+#
+# Runs as root: it needs read access to Caddy's 0700 cert storage AND the
+# ability to run `docker compose kill -s HUP` regardless of docker group
+# membership. Triggered by mqtt-cert-sync.path, not run standalone.
+[Unit]
+Description=Sync Caddy-issued MQTT TLS cert to mosquitto and reload
+
+[Service]
+Type=oneshot
+ExecStart=/opt/drb/sync-mqtt-cert.sh

infra/ansible/roles/deploy/templates/root.env.j2

@@ -1,10 +1,26 @@
 # Top-level docker-compose environment — MQTT credentials and registry prefix.
 # Managed by Ansible. Do not edit manually.
+#
+# The passwords are $-escaped ($ -> $$). Compose INTERPOLATES this file, so a
+# raw "$fP" in a password is read as the variable $fP, warned about, and
+# replaced with an empty string. The env_file templates (c2-core.env.j2 etc.)
+# are NOT interpolated, so they keep the literal value — which means an
+# unescaped $ here silently gives mosquitto and c2-core two different
+# passwords and MQTT auth fails. Compose collapses $$ back to a single $, so
+# both sides end up with the real password.
+# Do not add the same escaping to the env_file templates; it would be literal.
 
 MQTT_C2_USER={{ vault_mqtt_c2_user }}
-MQTT_C2_PASS={{ vault_mqtt_c2_pass }}
-MQTT_NODE_USER={{ vault_mqtt_node_user }}
-MQTT_NODE_PASS={{ vault_mqtt_node_pass }}
+MQTT_C2_PASS={{ vault_mqtt_c2_pass | replace('$', '$$') }}
+
+# Seeds mosquitto's built-in dynamic-security plugin's one-time "admin"
+# bootstrap client on first boot (read directly by the plugin's C code via
+# getenv — see app/internal/dynsec.py). Must be >=12 chars (plugin-enforced
+# minimum). c2-core needs this SAME value as MQTT_DYNSEC_ADMIN_PASS in its
+# own env (c2-core.env.j2) to log in as "admin" and administer node
+# credentials — kept as one vault var (vault_mqtt_dynsec_admin_pass) so the
+# two can't drift.
+MOSQUITTO_DYNSEC_PASSWORD={{ vault_mqtt_dynsec_admin_pass | replace('$', '$$') }}
 
 # Container registry prefix — docker compose uses this for image: ${REGISTRY}/name:latest
 REGISTRY={{ vault_registry }}

infra/ansible/roles/deploy/templates/sync-mqtt-cert.sh.j2

@@ -0,0 +1,56 @@
+#!/bin/bash
+# Managed by Ansible — do not edit manually.
+#
+# Copies Caddy's managed TLS cert for mqtt.{{ domain }} out of Caddy's
+# storage (root:caddy, 0700 — nothing else can read it) into a location the
+# mosquitto container can read, then SIGHUPs the broker so it picks up the
+# new cert without a full restart.
+#
+# Triggered by mqtt-cert-sync.path.j2 (a systemd path unit) watching the
+# source cert file for changes — a path unit rather than cron so this fires
+# on the actual write instead of racing a polling interval.
+#
+# CONFIRMED 2026-08-16 against a real issuance on drb-server: this path is
+# correct, and the copied cert came out as CN=mqtt.drb.cusano.net issued by
+# Let's Encrypt. Was previously flagged unverified.
+#
+# UNVERIFIED: mosquitto 2.x reloading TLS certs on SIGHUP without dropping
+# connections is documented upstream but untested here. If listener 8883
+# doesn't pick up the new cert (check `docker compose logs mosquitto` after
+# a sync), replace the `kill -s HUP` line below with a full
+# `docker compose ... restart mosquitto` instead.
+set -euo pipefail
+
+DOMAIN="mqtt.{{ domain }}"
+CADDY_CERT_DIR="/var/lib/caddy/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/${DOMAIN}"
+DEST_DIR="/opt/drb/mosquitto-certs"
+APP_DIR="{{ app_dir }}"
+
+SRC_CERT="${CADDY_CERT_DIR}/${DOMAIN}.crt"
+SRC_KEY="${CADDY_CERT_DIR}/${DOMAIN}.key"
+
+if [ ! -f "$SRC_CERT" ] || [ ! -f "$SRC_KEY" ]; then
+    echo "sync-mqtt-cert: source cert/key not found yet at $CADDY_CERT_DIR — Caddy may not have issued it yet." >&2
+    exit 0
+fi
+
+mkdir -p "$DEST_DIR"
+# Copy, don't symlink — nothing outside the caddy user can read the
+# originals (0700-owned), so mosquitto (running as a different container/
+# user) needs its own readable copy, not a pointer to an unreadable file.
+cp "$SRC_CERT" "$DEST_DIR/mqtt.crt"
+cp "$SRC_KEY"  "$DEST_DIR/mqtt.key"
+# Ownership matters: the stock eclipse-mosquitto entrypoint drops privileges
+# to the in-image `mosquitto` user (uid/gid 1883) — the broker does NOT run
+# as root, despite what an earlier note in DEFERRED.md claimed. Proof from a
+# real deploy: "running mosquitto as user: mosquitto", immediately followed
+# by "Unable to load server certificate ... Permission denied" on a
+# 600 root:root cert. The host has no such user, so use the numeric gid.
+# The cert is public material (0644); the key is group-read only (0640).
+chown root:1883 "$DEST_DIR/mqtt.crt" "$DEST_DIR/mqtt.key"
+chmod 644 "$DEST_DIR/mqtt.crt"
+chmod 640 "$DEST_DIR/mqtt.key"
+
+cd "$APP_DIR"
+docker compose -f docker-compose.yml -f docker-compose.prod.yml kill -s HUP mosquitto
+echo "sync-mqtt-cert: copied cert for ${DOMAIN} and sent SIGHUP to mosquitto."

infra/ansible/site.yml

@@ -36,16 +36,33 @@
         path: /swapfile
         mode: "0600"
 
+    # mkswap refuses to touch a file that is already active as swap, so a
+    # re-run would fail here without this guard. The swap file survives
+    # reboots via the fstab entry below, so on any second run it IS active.
+    - name: Check whether the swap file is already active
+      command: swapon --show=NAME --noheadings
+      register: _active_swaps
+      changed_when: false
+      failed_when: false
+
     - name: Format swap file
       command: mkswap /swapfile
+      when: "'/swapfile' not in _active_swaps.stdout"
       register: _mkswap
       changed_when: _mkswap.rc == 0
 
+    # Guarded by the same check as mkswap above. The stderr test alone was not
+    # enough: an already-active swap file reports "Device or resource busy",
+    # not "already", so the original failed_when never matched it.
     - name: Enable swap
       command: swapon /swapfile
+      when: "'/swapfile' not in _active_swaps.stdout"
       register: _swapon
-      failed_when: _swapon.rc != 0 and 'already' not in _swapon.stderr
-      changed_when: _swapon.rc == 0
+      failed_when: >
+        _swapon.rc is defined and _swapon.rc != 0
+        and 'already' not in _swapon.stderr
+        and 'busy' not in _swapon.stderr
+      changed_when: _swapon.rc is defined and _swapon.rc == 0
 
     - name: Persist swap in fstab
       lineinfile:

infra/ansible/vault.yml.example

@@ -4,15 +4,26 @@
 # Edit later with:
 #   ansible-vault edit vault.yml
 
+# DO NOT put a literal "$" in any value here. Docker compose interpolates the
+# top-level .env, and depending on version it also interpolates env_file, so a
+# password like "aB$fPx" is read as the variable $fPx and silently replaced
+# with an empty string — on one side of the connection but not the other.
+# That produced "MQTT connect refused: Not authorized" with no obvious cause.
+# Generate with: openssl rand -hex 32  (hex output has no shell metacharacters)
+
 # ── MQTT ─────────────────────────────────────────────────────────────────────
+# No more shared node credential (vault_mqtt_node_user/pass) — nodes now
+# authenticate as username=<node_id>, password=<their node_keys.api_key>,
+# checked by mosquitto's built-in dynamic-security plugin (c2-core
+# administers it — see app/internal/dynsec.py). See vault_enrollment_token
+# below for how a node gets that key in the first place.
 vault_mqtt_c2_user: drb-c2-core
 vault_mqtt_c2_pass: "CHANGE_ME"
-vault_mqtt_node_user: drb-node
-vault_mqtt_node_pass: "CHANGE_ME"
+vault_mqtt_dynsec_admin_pass: "CHANGE_ME"  # openssl rand -hex 32 — must be >=12 chars, plugin-enforced minimum
 
 # ── C2 Core ───────────────────────────────────────────────────────────────────
 vault_service_key: ""            # openssl rand -hex 32
-vault_node_api_key: ""           # openssl rand -hex 32
+vault_enrollment_token: ""       # openssl rand -hex 32 — fleet-wide, shared by every node's POST /nodes/enroll
 vault_openai_api_key: ""
 vault_google_maps_api_key: ""
 vault_gemini_api_key: ""
@@ -22,7 +33,14 @@ vault_firestore_database: "c2-server"
 # ── Gitea Container Registry ──────────────────────────────────────────────────
 vault_registry_host: "git.vpn.cusano.net"
 vault_registry_user: "logan"
-vault_registry_token: ""                                   # Gitea access token with package:write scope
+vault_registry_token: ""                                   # Gitea access token, READ-ONLY package scope.
+                                                           # The VM only pulls (roles/deploy/tasks/main.yml:62-72);
+                                                           # nothing here pushes. Pushing is CI's job and uses a
+                                                           # separate write-scoped token (BUILD_TOKEN in Gitea
+                                                           # repo secrets). Keep them separate: this token sits on
+                                                           # an internet-facing VM, and a write-scoped one there
+                                                           # would let an attacker publish a poisoned image that
+                                                           # every future deploy and edge node would install.
 vault_registry: "git.vpn.cusano.net/logan"                # full image prefix
 
 # ── Discord Bot ───────────────────────────────────────────────────────────────

infra/main.tf

@@ -64,20 +64,25 @@ resource "google_compute_firewall" "allow_ssh" {
   target_tags   = ["drb-server"]
 }
 
-# MQTT is NOT exposed externally — edge nodes connect via WireGuard (see below)
-# If you need to temporarily allow direct MQTT access for testing, uncomment and
-# restrict source_ranges to your node IPs.
-#
-# resource "google_compute_firewall" "allow_mqtt" {
-#   name    = "drb-allow-mqtt"
-#   network = "default"
-#   allow {
-#     protocol = "tcp"
-#     ports    = ["8883"]  # TLS MQTT, not 1883
-#   }
-#   source_ranges = ["YOUR_NODE_CIDR"]
-#   target_tags   = ["drb-server"]
-# }
+# MQTT is now publicly exposed on 8883 (TLS) — nodes get deployed to
+# arbitrary locations by arbitrary people, so there is no fixed CIDR to
+# restrict this to (WireGuard-per-node was evaluated and rejected; see
+# MQTT-PUBLIC-AUTH-PLAN.md). Security is enforced by mosquitto-go-auth
+# (per-node api_key over TLS), not by network ACL. 1883 (plaintext) is
+# intentionally NOT opened here — it stays on the docker bridge for
+# c2-core's own connection only.
+resource "google_compute_firewall" "allow_mqtt" {
+  name    = "drb-allow-mqtt"
+  network = "default"
+
+  allow {
+    protocol = "tcp"
+    ports    = ["8883"]  # TLS MQTT only, not 1883
+  }
+
+  source_ranges = ["0.0.0.0/0"]
+  target_tags   = ["drb-server"]
+}
 
 # ---------------------------------------------------------------------------
 # Compute Engine VM
@@ -185,5 +190,9 @@ resource "google_storage_bucket" "audio" {
 # After terraform apply, add these A records in Route 53:
 #   app.drb.cusano.net  → server_ip output
 #   api.drb.cusano.net  → server_ip output
+#   mqtt.drb.cusano.net → server_ip output  — MUST exist before the ansible
+#                                              deploy that adds the Caddy
+#                                              mqtt.<domain> block, or ACME
+#                                              issuance for it fails.
 # Or use a single wildcard: *.drb.cusano.net → server_ip
 # ---------------------------------------------------------------------------

infra/outputs.tf

@@ -3,12 +3,22 @@ output "server_ip" {
   description = "Static external IP of the DRB server VM"
 }
 
+# Bare domain, not app.<domain> — the frontend is served on the domain itself
+# (see infra/ansible/roles/deploy/templates/Caddyfile.j2). Only the bare name,
+# api. and mqtt. have DNS records; app. has never resolved.
 output "app_url" {
-  value = "https://app.${var.domain}"
+  value       = "https://${var.domain}"
+  description = "Frontend / portal URL"
 }
 
 output "api_url" {
   value       = "https://api.${var.domain}"
+  description = "c2-core REST API URL"
+}
+
+output "mqtt_host" {
+  value       = "mqtt.${var.domain}"
+  description = "Broker hostname edge nodes connect to on TCP 8883 (TLS)"
 }
 
 output "project_number" {