diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 01f8de9..3341d04 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -68,6 +68,10 @@ jobs: name: Deploy to VM needs: build runs-on: ubuntu-latest + outputs: + prev_sha: ${{ steps.deploy.outputs.prev_sha }} + rollback_status: ${{ steps.rollback.outputs.status }} + rollback_sha: ${{ steps.rollback.outputs.rolled_back_to }} steps: - name: Check runner outbound IP @@ -80,19 +84,33 @@ jobs: ssh-keygen -l -f /tmp/deploy_key - name: Deploy + id: deploy run: | - ssh -o StrictHostKeyChecking=no \ + set -o pipefail + OUTPUT=$(ssh -o StrictHostKeyChecking=no \ -o HostKeyAlgorithms=ssh-ed25519,rsa-sha2-256,rsa-sha2-512 \ -o ConnectTimeout=15 \ -v \ -i /tmp/deploy_key \ - drb@${{ secrets.SERVER_IP }} << 'ENDSSH' + drb@${{ secrets.SERVER_IP }} << 'ENDSSH' | tee /dev/stderr set -e cd /opt/drb # Update compose files + mosquitto config git pull origin main + # server-26#65: capture what is actually live BEFORE switching, so + # a bad deploy has something concrete to fall back to. This reads + # from a state file rather than re-deriving it from git log, + # because a PRIOR deploy could itself have failed and already + # rolled back to something older than HEAD~1 -- the file is only + # ever written by the Health check step below, after that step + # has confirmed the tag it names actually answered /health. A + # fresh VM with no file yet falls back to :latest, same escape + # hatch as a manual `up -d` with no TAG set. + PREV_TAG=$(cat /opt/drb/.last_good_tag 2>/dev/null || echo latest) + echo "PREV_TAG=$PREV_TAG" + # Deploy THIS commit's images, not :latest. Overlapping runs are # normal here, and with :latest whichever finishes last wins for # both -- run 544 asserted its own SHA and found run 545's build @@ -120,8 +138,17 @@ jobs: $COMPOSE up -d --remove-orphans docker image prune -f ENDSSH + ) + echo "$OUTPUT" + PREV_TAG=$(printf '%s\n' "$OUTPUT" | grep '^PREV_TAG=' | tail -n1 | cut -d'=' -f2) + if [ -z "$PREV_TAG" ]; then + echo "Could not determine the previous tag from deploy output - rollback target unknown." + exit 1 + fi + echo "prev_sha=$PREV_TAG" >> "$GITHUB_OUTPUT" - name: Health check + id: health run: | # Poll rather than sleep-once: the container has to finish starting, # and a fixed sleep is either too short (flaky red) or wastes time on @@ -149,6 +176,78 @@ jobs: exit 1 fi + # server-26#65: only now -- confirmed by /health, not by "up -d + # returned 0" -- record this as the rollback target for the NEXT + # deploy. A failure to write this is a bookkeeping problem, not a + # deploy problem, so it warns instead of failing the job (a hard + # failure here would trigger the Rollback step below against a + # perfectly good deploy). + ssh -o StrictHostKeyChecking=no \ + -o HostKeyAlgorithms=ssh-ed25519,rsa-sha2-256,rsa-sha2-512 \ + -o ConnectTimeout=15 \ + -i /tmp/deploy_key \ + drb@${{ secrets.SERVER_IP }} \ + "echo '${{ gitea.sha }}' > /opt/drb/.last_good_tag" \ + || echo "warning: failed to persist .last_good_tag - next deploy's rollback target may be stale" + + - name: Rollback on failed health check + id: rollback + if: failure() + run: | + # server-26#65 decision 3 / board minutes #62: up -d used to be the + # last word -- a build that passes tests, returns 200, and still + # corrupts incidents on live traffic would stay live for 12+ hours + # before a human noticed. This step is what makes that impossible: + # any failure above (pull, restart, or the health/SHA check) lands + # here and puts the previously-verified tag back. + PREV_TAG="${{ steps.deploy.outputs.prev_sha }}" + if [ -z "$PREV_TAG" ]; then + echo "No previous tag was captured (Deploy step itself failed before recording one) - cannot roll back automatically." + echo "status=skipped" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "Rolling back to $PREV_TAG" + + ssh -o StrictHostKeyChecking=no \ + -o HostKeyAlgorithms=ssh-ed25519,rsa-sha2-256,rsa-sha2-512 \ + -o ConnectTimeout=15 \ + -i /tmp/deploy_key \ + drb@${{ secrets.SERVER_IP }} << ENDSSH + set -e + cd /opt/drb + export TAG=$PREV_TAG + COMPOSE="docker compose -f docker-compose.yml -f docker-compose.prod.yml" + if ! \$COMPOSE pull; then + echo "rollback image pull failed - pruning and retrying once" + docker image prune -af + \$COMPOSE pull + fi + \$COMPOSE up -d --remove-orphans + ENDSSH + + # Re-verify exactly like the forward health check does: liveness + # alone doesn't prove the rollback took, the SHA has to match the + # tag we just switched back to. + BODY="" + for _ in $(seq 1 12); do + sleep 5 + BODY=$(curl -fsS https://api.${{ secrets.DRB_DOMAIN }}/health) || continue + case "$BODY" in *"$PREV_TAG"*) break ;; esac + done + + RUNNING=$(printf '%s' "$BODY" | tr ',' '\n' | grep git_sha | cut -d'"' -f4) + if [ "$RUNNING" != "$PREV_TAG" ]; then + echo "ROLLBACK FAILED: expected git_sha '$PREV_TAG', got '$RUNNING'." + echo "Production state is UNKNOWN - check the VM by hand immediately." + echo "status=failed" >> "$GITHUB_OUTPUT" + echo "rolled_back_to=$PREV_TAG" >> "$GITHUB_OUTPUT" + exit 1 + fi + + echo "Rolled back successfully to $PREV_TAG" + echo "status=success" >> "$GITHUB_OUTPUT" + echo "rolled_back_to=$PREV_TAG" >> "$GITHUB_OUTPUT" + notify-failure: name: Report a failed deploy needs: [build, deploy] @@ -166,6 +265,8 @@ jobs: WEBHOOK: ${{ secrets.DEPLOY_ALERT_WEBHOOK }} RUN_URL: ${{ gitea.server_url }}/${{ gitea.repository }}/actions/runs/${{ gitea.run_number }} SHA: ${{ gitea.sha }} + ROLLBACK_STATUS: ${{ needs.deploy.outputs.rollback_status }} + ROLLBACK_SHA: ${{ needs.deploy.outputs.rollback_sha }} run: | if [ -z "$WEBHOOK" ]; then echo "DEPLOY_ALERT_WEBHOOK is not set - skipping notification." @@ -173,9 +274,30 @@ jobs: fi python3 - <<'PY' > /tmp/payload.json import json, os + sha = os.environ["SHA"][:8] + run_url = os.environ["RUN_URL"] + status = os.environ.get("ROLLBACK_STATUS", "") + rollback_sha = os.environ.get("ROLLBACK_SHA", "") + + # server-26#65: the old text here unconditionally claimed + # "production is still running the previous build" -- true only + # when the pull/restart itself failed. It's false the moment a + # build passes the SHA check but has a live logic bug (exactly the + # class of bug the correlator instrumentation exists to catch), or + # once the deploy job's own rollback path has run. Say what + # actually happened instead. + if status == "success": + detail = "Automatic rollback to `%s` succeeded. Production is back on the previous good build." % rollback_sha[:8] + elif status == "failed": + detail = ("Automatic rollback to `%s` FAILED. Production state is UNKNOWN -- " + "check the VM by hand immediately.") % rollback_sha[:8] + elif status == "skipped": + detail = "No rollback was attempted (no previous tag captured, or build/push failed before any deploy). Check the VM by hand." + else: + detail = "Build failed before any deploy was attempted. Production is unchanged." + print(json.dumps({"content": - "**DRB deploy failed** on `%s`\n%s\nProduction is still running the previous build." - % (os.environ["SHA"][:8], os.environ["RUN_URL"])})) + "**DRB deploy failed** on `%s`\n%s\n%s" % (sha, run_url, detail)})) PY curl -sS -X POST -H "Content-Type: application/json" \ --data @/tmp/payload.json "$WEBHOOK" || echo "notification POST failed"