diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 93ab700..e59f408 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -69,6 +69,21 @@ jobs: name: Deploy to VM needs: build runs-on: ubuntu-latest + # server-26#167: without this, two merges a few minutes apart had two + # "Deploy to VM" jobs on the VM at once (runs 616/617, 652/656): racing + # `compose pull && up -d`, one run's prune breaking the other's + # prune-and-retry, and one run's health check judging the other run's + # containers. Gitea >= 1.26 honours `concurrency` (go-gitea/gitea#32751; + # older versions silently ignored it, and 1.26 also dropped the old + # implicit auto-cancel), the server runs 1.27.3. Job-level so builds still + # overlap, only the VM part is serialised. cancel-in-progress: false + # queues instead of killing a run mid-compose; a newer run joining the + # group replaces any run still waiting, and the Deploy step below skips a + # run that has stopped being main's tip, so an older SHA never lands after + # a newer one. + concurrency: + group: drb-prod-deploy + cancel-in-progress: false outputs: prev_sha: ${{ steps.deploy.outputs.prev_sha }} rollback_status: ${{ steps.rollback.outputs.status }} @@ -97,6 +112,20 @@ jobs: set -e cd /opt/drb + # server-26#167: only main's tip deploys. Serialising (the job's + # concurrency group) stops two runs overlapping, but not an older + # run going AFTER a newer one (its build finished later, or someone + # re-ran it), which would silently roll prod back. A superseded run + # exits clean before touching anything; the newer run carries this + # commit's changes anyway. To deliberately put an older build back, + # use TAG= by hand on the VM, not a re-run. + git fetch -q origin main + MAIN_TIP=$(git rev-parse origin/main) + if [ "$MAIN_TIP" != "${{ gitea.sha }}" ]; then + echo "SUPERSEDED=$MAIN_TIP" + exit 0 + fi + # server-26#129: every deploy pushes 3 freshly SHA-tagged images and # nothing ever removed the old ones except a prune that only ran # AFTER a successful `compose pull` -- so a run that never got that @@ -183,6 +212,12 @@ jobs: ENDSSH ) echo "$OUTPUT" + SUPERSEDED=$(printf '%s\n' "$OUTPUT" | grep '^SUPERSEDED=' | tail -n1 | cut -d'=' -f2 || true) + if [ -n "$SUPERSEDED" ]; then + echo "main has moved on to $SUPERSEDED - this run (${{ gitea.sha }}) is superseded, nothing deployed." + echo "superseded=true" >> "$GITHUB_OUTPUT" + exit 0 + fi PREV_TAG=$(printf '%s\n' "$OUTPUT" | grep '^PREV_TAG=' | tail -n1 | cut -d'=' -f2) if [ -z "$PREV_TAG" ]; then echo "Could not determine the previous tag from deploy output - rollback target unknown." @@ -192,6 +227,9 @@ jobs: - name: Health check id: health + # A superseded run deployed nothing, so there is nothing of its own to + # check, and /health legitimately answers with the newer SHA. + if: steps.deploy.outputs.superseded != 'true' run: | # Poll rather than sleep-once: the container has to finish starting, # and a fixed sleep is either too short (flaky red) or wastes time on @@ -295,6 +333,12 @@ jobs: name: Deploy Firestore rules & indexes needs: build runs-on: ubuntu-latest + # server-26#167: same race as the app deploy, smaller blast radius: an + # older run's `firebase deploy` finishing last would put stale rules back. + # Its own group, so it never waits on the VM deploy. + concurrency: + group: drb-prod-firestore-rules + cancel-in-progress: false # Deliberately independent of the `deploy` job (app containers) and its # health-check/rollback chain above: a rules/indexes deploy failure has # nothing to roll back (there is no previous "build" of a ruleset to @@ -310,6 +354,13 @@ jobs: FIREBASE_SA_KEY: ${{ secrets.FIREBASE_SA_KEY }} run: | set -e + # server-26#167: only main's tip deploys rules (see the app Deploy + # step). An unreadable remote falls through to deploying, as before. + MAIN_TIP=$(git ls-remote origin refs/heads/main | cut -f1) + if [ -n "$MAIN_TIP" ] && [ "$MAIN_TIP" != "${{ gitea.sha }}" ]; then + echo "main has moved on to $MAIN_TIP - this run is superseded, rules not deployed." + exit 0 + fi # server-26#51: this used to run over SSH on the deploy VM, gated # on the VM having firebase-tools installed. It never did, so it # silently warned-and-skipped on every single deploy for weeks.