Merge pull request 'deploy.yml: serialise prod deploys and skip superseded runs (#167)' (#210) from fix/167-deploy-concurrency into main

Reviewed-on: #210

.gitea/workflows/deploy.yml

@@ -69,6 +69,21 @@ jobs:
     name: Deploy to VM
     needs: build
     runs-on: ubuntu-latest
+    # server-26#167: without this, two merges a few minutes apart had two
+    # "Deploy to VM" jobs on the VM at once (runs 616/617, 652/656): racing
+    # `compose pull && up -d`, one run's prune breaking the other's
+    # prune-and-retry, and one run's health check judging the other run's
+    # containers. Gitea >= 1.26 honours `concurrency` (go-gitea/gitea#32751;
+    # older versions silently ignored it, and 1.26 also dropped the old
+    # implicit auto-cancel), the server runs 1.27.3. Job-level so builds still
+    # overlap, only the VM part is serialised. cancel-in-progress: false
+    # queues instead of killing a run mid-compose; a newer run joining the
+    # group replaces any run still waiting, and the Deploy step below skips a
+    # run that has stopped being main's tip, so an older SHA never lands after
+    # a newer one.
+    concurrency:
+      group: drb-prod-deploy
+      cancel-in-progress: false
     outputs:
       prev_sha: ${{ steps.deploy.outputs.prev_sha }}
       rollback_status: ${{ steps.rollback.outputs.status }}
@@ -112,6 +127,22 @@ jobs:
           # block a deploy that doesn't actually need the space this time.
           docker image prune -af || echo "WARNING: pre-deploy image prune failed (server-26#129) -- disk pressure may persist"
 
+          # server-26#167: only main's tip deploys. Serialising (the job's
+          # concurrency group) stops two runs overlapping, but not an older
+          # run going AFTER a newer one (its build finished later, or someone
+          # re-ran it), which would silently roll prod back. A superseded run
+          # exits clean before pulling or restarting anything; the newer run carries this
+          # commit's changes anyway. To deliberately put an older build back,
+          # use TAG=<sha> by hand on the VM, not a re-run. Runs AFTER the
+          # server-26#129 prune above on purpose: on a full disk `git fetch` can
+          # fail too, and that prune must stay the first thing a deploy does.
+          git fetch -q origin main
+          MAIN_TIP=$(git rev-parse origin/main)
+          if [ "$MAIN_TIP" != "${{ gitea.sha }}" ]; then
+            echo "SUPERSEDED=$MAIN_TIP"
+            exit 0
+          fi
+
           # Update compose files + mosquitto config
           git pull origin main
 
@@ -183,6 +214,12 @@ jobs:
           ENDSSH
           )
           echo "$OUTPUT"
+          SUPERSEDED=$(printf '%s\n' "$OUTPUT" | grep '^SUPERSEDED=' | tail -n1 | cut -d'=' -f2 || true)
+          if [ -n "$SUPERSEDED" ]; then
+            echo "main has moved on to $SUPERSEDED - this run (${{ gitea.sha }}) is superseded, nothing deployed."
+            echo "superseded=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
           PREV_TAG=$(printf '%s\n' "$OUTPUT" | grep '^PREV_TAG=' | tail -n1 | cut -d'=' -f2)
           if [ -z "$PREV_TAG" ]; then
             echo "Could not determine the previous tag from deploy output - rollback target unknown."
@@ -192,6 +229,9 @@ jobs:
 
       - name: Health check
         id: health
+        # A superseded run deployed nothing, so there is nothing of its own to
+        # check, and /health legitimately answers with the newer SHA.
+        if: steps.deploy.outputs.superseded != 'true'
         run: |
           # Poll rather than sleep-once: the container has to finish starting,
           # and a fixed sleep is either too short (flaky red) or wastes time on
@@ -295,6 +335,12 @@ jobs:
     name: Deploy Firestore rules & indexes
     needs: build
     runs-on: ubuntu-latest
+    # server-26#167: same race as the app deploy, smaller blast radius: an
+    # older run's `firebase deploy` finishing last would put stale rules back.
+    # Its own group, so it never waits on the VM deploy.
+    concurrency:
+      group: drb-prod-firestore-rules
+      cancel-in-progress: false
     # Deliberately independent of the `deploy` job (app containers) and its
     # health-check/rollback chain above: a rules/indexes deploy failure has
     # nothing to roll back (there is no previous "build" of a ruleset to
@@ -310,6 +356,13 @@ jobs:
           FIREBASE_SA_KEY: ${{ secrets.FIREBASE_SA_KEY }}
         run: |
           set -e
+          # server-26#167: only main's tip deploys rules (see the app Deploy
+          # step). An unreadable remote falls through to deploying, as before.
+          MAIN_TIP=$(git ls-remote origin refs/heads/main | cut -f1)
+          if [ -n "$MAIN_TIP" ] && [ "$MAIN_TIP" != "${{ gitea.sha }}" ]; then
+            echo "main has moved on to $MAIN_TIP - this run is superseded, rules not deployed."
+            exit 0
+          fi
           # server-26#51: this used to run over SSH on the deploy VM, gated
           # on the VM having firebase-tools installed. It never did, so it
           # silently warned-and-skipped on every single deploy for weeks.