From 71ee6101a135d82dd5b2fe0f29aea9fa303a1d05 Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Sun, 4 Oct 2026 18:01:53 -0400 Subject: [PATCH 1/2] deploy.yml: serialise prod deploys and skip superseded runs (server-26#167) Two merges minutes apart ran two "Deploy to VM" jobs on the VM at once (616/617, 652/656): racing compose pull/up, one run's prune breaking the other's retry, health checks judging the wrong containers. - deploy job: concurrency group drb-prod-deploy, cancel-in-progress false (Gitea >= 1.26 honours it, go-gitea/gitea#32751; server is 1.27.3). - Deploy step exits clean, before touching the VM, when its SHA is no longer origin/main's tip; Health check is skipped for a superseded run. Stops an older run landing after a newer one and rolling prod back. - deploy-firestore-rules: its own group plus the same tip check. Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/deploy.yml | 51 +++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 93ab700..e59f408 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -69,6 +69,21 @@ jobs: name: Deploy to VM needs: build runs-on: ubuntu-latest + # server-26#167: without this, two merges a few minutes apart had two + # "Deploy to VM" jobs on the VM at once (runs 616/617, 652/656): racing + # `compose pull && up -d`, one run's prune breaking the other's + # prune-and-retry, and one run's health check judging the other run's + # containers. Gitea >= 1.26 honours `concurrency` (go-gitea/gitea#32751; + # older versions silently ignored it, and 1.26 also dropped the old + # implicit auto-cancel), the server runs 1.27.3. Job-level so builds still + # overlap, only the VM part is serialised. cancel-in-progress: false + # queues instead of killing a run mid-compose; a newer run joining the + # group replaces any run still waiting, and the Deploy step below skips a + # run that has stopped being main's tip, so an older SHA never lands after + # a newer one. + concurrency: + group: drb-prod-deploy + cancel-in-progress: false outputs: prev_sha: ${{ steps.deploy.outputs.prev_sha }} rollback_status: ${{ steps.rollback.outputs.status }} @@ -97,6 +112,20 @@ jobs: set -e cd /opt/drb + # server-26#167: only main's tip deploys. Serialising (the job's + # concurrency group) stops two runs overlapping, but not an older + # run going AFTER a newer one (its build finished later, or someone + # re-ran it), which would silently roll prod back. A superseded run + # exits clean before touching anything; the newer run carries this + # commit's changes anyway. To deliberately put an older build back, + # use TAG= by hand on the VM, not a re-run. + git fetch -q origin main + MAIN_TIP=$(git rev-parse origin/main) + if [ "$MAIN_TIP" != "${{ gitea.sha }}" ]; then + echo "SUPERSEDED=$MAIN_TIP" + exit 0 + fi + # server-26#129: every deploy pushes 3 freshly SHA-tagged images and # nothing ever removed the old ones except a prune that only ran # AFTER a successful `compose pull` -- so a run that never got that @@ -183,6 +212,12 @@ jobs: ENDSSH ) echo "$OUTPUT" + SUPERSEDED=$(printf '%s\n' "$OUTPUT" | grep '^SUPERSEDED=' | tail -n1 | cut -d'=' -f2 || true) + if [ -n "$SUPERSEDED" ]; then + echo "main has moved on to $SUPERSEDED - this run (${{ gitea.sha }}) is superseded, nothing deployed." + echo "superseded=true" >> "$GITHUB_OUTPUT" + exit 0 + fi PREV_TAG=$(printf '%s\n' "$OUTPUT" | grep '^PREV_TAG=' | tail -n1 | cut -d'=' -f2) if [ -z "$PREV_TAG" ]; then echo "Could not determine the previous tag from deploy output - rollback target unknown." @@ -192,6 +227,9 @@ jobs: - name: Health check id: health + # A superseded run deployed nothing, so there is nothing of its own to + # check, and /health legitimately answers with the newer SHA. + if: steps.deploy.outputs.superseded != 'true' run: | # Poll rather than sleep-once: the container has to finish starting, # and a fixed sleep is either too short (flaky red) or wastes time on @@ -295,6 +333,12 @@ jobs: name: Deploy Firestore rules & indexes needs: build runs-on: ubuntu-latest + # server-26#167: same race as the app deploy, smaller blast radius: an + # older run's `firebase deploy` finishing last would put stale rules back. + # Its own group, so it never waits on the VM deploy. + concurrency: + group: drb-prod-firestore-rules + cancel-in-progress: false # Deliberately independent of the `deploy` job (app containers) and its # health-check/rollback chain above: a rules/indexes deploy failure has # nothing to roll back (there is no previous "build" of a ruleset to @@ -310,6 +354,13 @@ jobs: FIREBASE_SA_KEY: ${{ secrets.FIREBASE_SA_KEY }} run: | set -e + # server-26#167: only main's tip deploys rules (see the app Deploy + # step). An unreadable remote falls through to deploying, as before. + MAIN_TIP=$(git ls-remote origin refs/heads/main | cut -f1) + if [ -n "$MAIN_TIP" ] && [ "$MAIN_TIP" != "${{ gitea.sha }}" ]; then + echo "main has moved on to $MAIN_TIP - this run is superseded, rules not deployed." + exit 0 + fi # server-26#51: this used to run over SSH on the deploy VM, gated # on the VM having firebase-tools installed. It never did, so it # silently warned-and-skipped on every single deploy for weeks. -- 2.54.0 From 06f01ebbe01279728df5b189c37a33ee448e2bfc Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Sun, 4 Oct 2026 18:07:08 -0400 Subject: [PATCH 2/2] deploy.yml: run the superseded check after the #129 pre-deploy prune (server-26#167) The tip check's git fetch ran before the disk-relief prune that #129 says must be the first thing a deploy does; on a full disk the fetch could fail and skip the prune. Prune only removes unreferenced images, so running it on a run that then turns out superseded is harmless. Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/deploy.yml | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index e59f408..e186f80 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -112,20 +112,6 @@ jobs: set -e cd /opt/drb - # server-26#167: only main's tip deploys. Serialising (the job's - # concurrency group) stops two runs overlapping, but not an older - # run going AFTER a newer one (its build finished later, or someone - # re-ran it), which would silently roll prod back. A superseded run - # exits clean before touching anything; the newer run carries this - # commit's changes anyway. To deliberately put an older build back, - # use TAG= by hand on the VM, not a re-run. - git fetch -q origin main - MAIN_TIP=$(git rev-parse origin/main) - if [ "$MAIN_TIP" != "${{ gitea.sha }}" ]; then - echo "SUPERSEDED=$MAIN_TIP" - exit 0 - fi - # server-26#129: every deploy pushes 3 freshly SHA-tagged images and # nothing ever removed the old ones except a prune that only ran # AFTER a successful `compose pull` -- so a run that never got that @@ -141,6 +127,22 @@ jobs: # block a deploy that doesn't actually need the space this time. docker image prune -af || echo "WARNING: pre-deploy image prune failed (server-26#129) -- disk pressure may persist" + # server-26#167: only main's tip deploys. Serialising (the job's + # concurrency group) stops two runs overlapping, but not an older + # run going AFTER a newer one (its build finished later, or someone + # re-ran it), which would silently roll prod back. A superseded run + # exits clean before pulling or restarting anything; the newer run carries this + # commit's changes anyway. To deliberately put an older build back, + # use TAG= by hand on the VM, not a re-run. Runs AFTER the + # server-26#129 prune above on purpose: on a full disk `git fetch` can + # fail too, and that prune must stay the first thing a deploy does. + git fetch -q origin main + MAIN_TIP=$(git rev-parse origin/main) + if [ "$MAIN_TIP" != "${{ gitea.sha }}" ]; then + echo "SUPERSEDED=$MAIN_TIP" + exit 0 + fi + # Update compose files + mosquitto config git pull origin main -- 2.54.0