From 55cd1110dfdcd9e2be6bcd874dfd0c8895c6d3e9 Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Sun, 16 Aug 2026 13:29:26 -0400 Subject: [PATCH] Give mosquitto's bind-mounted dirs to uid 1883, not root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The broker crash-looped on every deploy: "Unable to load server certificate /mosquitto/certs/mqtt.crt ... Permission denied". The cert-sync script wrote 600 root:root into a 0700 root:root directory, on the assumption that mosquitto runs as root inside its container. It does not — the stock eclipse-mosquitto entrypoint drops privileges to the in-image mosquitto user, confirmed on the server as uid=1883(mosquitto) gid=1883(mosquitto), and the broker's own log says so on every start. Certs dir is now root:1883 0750 with the cert 0644 and the key 0640, and the data dir is 1883:1883 recursively — recursively because mosquitto WRITES dynamic-security.json there, and a root-owned file left by an earlier deploy would still be unwritable after a directory-only chown. Also drops the "unverified Caddy cert path" note: a real issuance confirmed the path, producing CN=mqtt.drb.cusano.net signed by Let's Encrypt. Co-Authored-By: Claude Opus 5 --- infra/ansible/roles/deploy/tasks/main.yml | 26 +++++++++++++------ .../deploy/templates/sync-mqtt-cert.sh.j2 | 21 +++++++++------ 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/infra/ansible/roles/deploy/tasks/main.yml b/infra/ansible/roles/deploy/tasks/main.yml index c7ec660..5446c51 100644 --- a/infra/ansible/roles/deploy/tasks/main.yml +++ b/infra/ansible/roles/deploy/tasks/main.yml @@ -72,26 +72,36 @@ # read Caddy's own cert storage, so a systemd path unit + oneshot service # copies a readable copy out and SIGHUPs the broker on every change. +# root:1883 0750, not root:root 0700. The stock eclipse-mosquitto entrypoint +# drops privileges to the in-image `mosquitto` user (uid/gid 1883), so a +# root-only directory makes the broker fail to read its own cert and +# crash-loop: "Unable to load server certificate ... Permission denied". +# The host has no `mosquitto` user, hence the numeric gid. - name: Create mosquitto certs directory file: path: /opt/drb/mosquitto-certs state: directory owner: root - group: root - mode: "0700" + group: "1883" + mode: "0750" # dynamic-security.json (node credentials — see app/internal/dynsec.py) -# lives here. Root-owned is fine: the mosquitto container itself runs as -# root (no `user` directive in mosquitto.conf, matching the pre-existing -# setup this project already ran before the dynsec change), so it can -# read/write this directory directly without any host-side chown dance. +# lives here, and mosquitto WRITES it, so this must be owned by the uid the +# broker actually runs as (1883), not root. The earlier assumption that the +# container runs as root was wrong — the image's entrypoint drops privileges +# to the `mosquitto` user, which a real deploy proved by failing to read a +# root-owned cert. Same numeric-gid reasoning as the certs directory above. - name: Create mosquitto data directory file: path: /opt/drb/mosquitto-data state: directory - owner: root - group: root + owner: "1883" + group: "1883" mode: "0700" + # recurse so an existing root-owned dynamic-security.json / mosquitto.db + # left behind by the earlier root-owned deploy gets fixed too — chowning + # only the directory would leave the broker unable to rewrite them. + recurse: true - name: Deploy MQTT cert-sync script template: diff --git a/infra/ansible/roles/deploy/templates/sync-mqtt-cert.sh.j2 b/infra/ansible/roles/deploy/templates/sync-mqtt-cert.sh.j2 index 9b1197e..92405fd 100644 --- a/infra/ansible/roles/deploy/templates/sync-mqtt-cert.sh.j2 +++ b/infra/ansible/roles/deploy/templates/sync-mqtt-cert.sh.j2 @@ -10,12 +10,9 @@ # source cert file for changes — a path unit rather than cron so this fires # on the actual write instead of racing a polling interval. # -# UNVERIFIED: the exact source path below assumes Caddy's default file -# storage layout and Let's Encrypt's production ACME directory name. This -# has not been confirmed against a real Caddy cert issuance for this -# project — check `caddy storage` / find the actual path under -# /var/lib/caddy the first time this runs, and correct CADDY_CERT_DIR below -# if it doesn't match. +# CONFIRMED 2026-08-16 against a real issuance on drb-server: this path is +# correct, and the copied cert came out as CN=mqtt.drb.cusano.net issued by +# Let's Encrypt. Was previously flagged unverified. # # UNVERIFIED: mosquitto 2.x reloading TLS certs on SIGHUP without dropping # connections is documented upstream but untested here. If listener 8883 @@ -43,8 +40,16 @@ mkdir -p "$DEST_DIR" # user) needs its own readable copy, not a pointer to an unreadable file. cp "$SRC_CERT" "$DEST_DIR/mqtt.crt" cp "$SRC_KEY" "$DEST_DIR/mqtt.key" -chmod 600 "$DEST_DIR/mqtt.crt" "$DEST_DIR/mqtt.key" -chown root:root "$DEST_DIR/mqtt.crt" "$DEST_DIR/mqtt.key" +# Ownership matters: the stock eclipse-mosquitto entrypoint drops privileges +# to the in-image `mosquitto` user (uid/gid 1883) — the broker does NOT run +# as root, despite what an earlier note in DEFERRED.md claimed. Proof from a +# real deploy: "running mosquitto as user: mosquitto", immediately followed +# by "Unable to load server certificate ... Permission denied" on a +# 600 root:root cert. The host has no such user, so use the numeric gid. +# The cert is public material (0644); the key is group-read only (0640). +chown root:1883 "$DEST_DIR/mqtt.crt" "$DEST_DIR/mqtt.key" +chmod 644 "$DEST_DIR/mqtt.crt" +chmod 640 "$DEST_DIR/mqtt.key" cd "$APP_DIR" docker compose -f docker-compose.yml -f docker-compose.prod.yml kill -s HUP mosquitto