deploy.yml: serialise prod deploys and skip superseded runs (server-26#167)
Two merges minutes apart ran two "Deploy to VM" jobs on the VM at once (616/617, 652/656): racing compose pull/up, one run's prune breaking the other's retry, health checks judging the wrong containers. - deploy job: concurrency group drb-prod-deploy, cancel-in-progress false (Gitea >= 1.26 honours it, go-gitea/gitea#32751; server is 1.27.3). - Deploy step exits clean, before touching the VM, when its SHA is no longer origin/main's tip; Health check is skipped for a superseded run. Stops an older run landing after a newer one and rolling prod back. - deploy-firestore-rules: its own group plus the same tip check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
d57738a8f6
commit
71ee6101a1
@@ -69,6 +69,21 @@ jobs:
|
|||||||
name: Deploy to VM
|
name: Deploy to VM
|
||||||
needs: build
|
needs: build
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
# server-26#167: without this, two merges a few minutes apart had two
|
||||||
|
# "Deploy to VM" jobs on the VM at once (runs 616/617, 652/656): racing
|
||||||
|
# `compose pull && up -d`, one run's prune breaking the other's
|
||||||
|
# prune-and-retry, and one run's health check judging the other run's
|
||||||
|
# containers. Gitea >= 1.26 honours `concurrency` (go-gitea/gitea#32751;
|
||||||
|
# older versions silently ignored it, and 1.26 also dropped the old
|
||||||
|
# implicit auto-cancel), the server runs 1.27.3. Job-level so builds still
|
||||||
|
# overlap, only the VM part is serialised. cancel-in-progress: false
|
||||||
|
# queues instead of killing a run mid-compose; a newer run joining the
|
||||||
|
# group replaces any run still waiting, and the Deploy step below skips a
|
||||||
|
# run that has stopped being main's tip, so an older SHA never lands after
|
||||||
|
# a newer one.
|
||||||
|
concurrency:
|
||||||
|
group: drb-prod-deploy
|
||||||
|
cancel-in-progress: false
|
||||||
outputs:
|
outputs:
|
||||||
prev_sha: ${{ steps.deploy.outputs.prev_sha }}
|
prev_sha: ${{ steps.deploy.outputs.prev_sha }}
|
||||||
rollback_status: ${{ steps.rollback.outputs.status }}
|
rollback_status: ${{ steps.rollback.outputs.status }}
|
||||||
@@ -97,6 +112,20 @@ jobs:
|
|||||||
set -e
|
set -e
|
||||||
cd /opt/drb
|
cd /opt/drb
|
||||||
|
|
||||||
|
# server-26#167: only main's tip deploys. Serialising (the job's
|
||||||
|
# concurrency group) stops two runs overlapping, but not an older
|
||||||
|
# run going AFTER a newer one (its build finished later, or someone
|
||||||
|
# re-ran it), which would silently roll prod back. A superseded run
|
||||||
|
# exits clean before touching anything; the newer run carries this
|
||||||
|
# commit's changes anyway. To deliberately put an older build back,
|
||||||
|
# use TAG=<sha> by hand on the VM, not a re-run.
|
||||||
|
git fetch -q origin main
|
||||||
|
MAIN_TIP=$(git rev-parse origin/main)
|
||||||
|
if [ "$MAIN_TIP" != "${{ gitea.sha }}" ]; then
|
||||||
|
echo "SUPERSEDED=$MAIN_TIP"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
# server-26#129: every deploy pushes 3 freshly SHA-tagged images and
|
# server-26#129: every deploy pushes 3 freshly SHA-tagged images and
|
||||||
# nothing ever removed the old ones except a prune that only ran
|
# nothing ever removed the old ones except a prune that only ran
|
||||||
# AFTER a successful `compose pull` -- so a run that never got that
|
# AFTER a successful `compose pull` -- so a run that never got that
|
||||||
@@ -183,6 +212,12 @@ jobs:
|
|||||||
ENDSSH
|
ENDSSH
|
||||||
)
|
)
|
||||||
echo "$OUTPUT"
|
echo "$OUTPUT"
|
||||||
|
SUPERSEDED=$(printf '%s\n' "$OUTPUT" | grep '^SUPERSEDED=' | tail -n1 | cut -d'=' -f2 || true)
|
||||||
|
if [ -n "$SUPERSEDED" ]; then
|
||||||
|
echo "main has moved on to $SUPERSEDED - this run (${{ gitea.sha }}) is superseded, nothing deployed."
|
||||||
|
echo "superseded=true" >> "$GITHUB_OUTPUT"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
PREV_TAG=$(printf '%s\n' "$OUTPUT" | grep '^PREV_TAG=' | tail -n1 | cut -d'=' -f2)
|
PREV_TAG=$(printf '%s\n' "$OUTPUT" | grep '^PREV_TAG=' | tail -n1 | cut -d'=' -f2)
|
||||||
if [ -z "$PREV_TAG" ]; then
|
if [ -z "$PREV_TAG" ]; then
|
||||||
echo "Could not determine the previous tag from deploy output - rollback target unknown."
|
echo "Could not determine the previous tag from deploy output - rollback target unknown."
|
||||||
@@ -192,6 +227,9 @@ jobs:
|
|||||||
|
|
||||||
- name: Health check
|
- name: Health check
|
||||||
id: health
|
id: health
|
||||||
|
# A superseded run deployed nothing, so there is nothing of its own to
|
||||||
|
# check, and /health legitimately answers with the newer SHA.
|
||||||
|
if: steps.deploy.outputs.superseded != 'true'
|
||||||
run: |
|
run: |
|
||||||
# Poll rather than sleep-once: the container has to finish starting,
|
# Poll rather than sleep-once: the container has to finish starting,
|
||||||
# and a fixed sleep is either too short (flaky red) or wastes time on
|
# and a fixed sleep is either too short (flaky red) or wastes time on
|
||||||
@@ -295,6 +333,12 @@ jobs:
|
|||||||
name: Deploy Firestore rules & indexes
|
name: Deploy Firestore rules & indexes
|
||||||
needs: build
|
needs: build
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
# server-26#167: same race as the app deploy, smaller blast radius: an
|
||||||
|
# older run's `firebase deploy` finishing last would put stale rules back.
|
||||||
|
# Its own group, so it never waits on the VM deploy.
|
||||||
|
concurrency:
|
||||||
|
group: drb-prod-firestore-rules
|
||||||
|
cancel-in-progress: false
|
||||||
# Deliberately independent of the `deploy` job (app containers) and its
|
# Deliberately independent of the `deploy` job (app containers) and its
|
||||||
# health-check/rollback chain above: a rules/indexes deploy failure has
|
# health-check/rollback chain above: a rules/indexes deploy failure has
|
||||||
# nothing to roll back (there is no previous "build" of a ruleset to
|
# nothing to roll back (there is no previous "build" of a ruleset to
|
||||||
@@ -310,6 +354,13 @@ jobs:
|
|||||||
FIREBASE_SA_KEY: ${{ secrets.FIREBASE_SA_KEY }}
|
FIREBASE_SA_KEY: ${{ secrets.FIREBASE_SA_KEY }}
|
||||||
run: |
|
run: |
|
||||||
set -e
|
set -e
|
||||||
|
# server-26#167: only main's tip deploys rules (see the app Deploy
|
||||||
|
# step). An unreadable remote falls through to deploying, as before.
|
||||||
|
MAIN_TIP=$(git ls-remote origin refs/heads/main | cut -f1)
|
||||||
|
if [ -n "$MAIN_TIP" ] && [ "$MAIN_TIP" != "${{ gitea.sha }}" ]; then
|
||||||
|
echo "main has moved on to $MAIN_TIP - this run is superseded, rules not deployed."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
# server-26#51: this used to run over SSH on the deploy VM, gated
|
# server-26#51: this used to run over SSH on the deploy VM, gated
|
||||||
# on the VM having firebase-tools installed. It never did, so it
|
# on the VM having firebase-tools installed. It never did, so it
|
||||||
# silently warned-and-skipped on every single deploy for weeks.
|
# silently warned-and-skipped on every single deploy for weeks.
|
||||||
|
|||||||
Reference in New Issue
Block a user