feat(ci): add Vault advisory locks to CI terraform applies

CI now uses scripts/tg instead of raw terragrunt apply, acquiring the
same per-stack Vault KV lock that user sessions use. This prevents CI
from overwriting in-flight user applies.

Changes:
- Switch from xargs -P 4 (parallel) to serial while-read loop
- CI skips stacks locked by users instead of racing them
- Git rebase failures now exit 1 instead of silently continuing
- Updated header comments to reflect new locking behavior

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Viktor Barzin 2026-04-15 20:53:00 +00:00
parent f7411327d1
commit 89af09852f

View file

@ -5,10 +5,11 @@
# - Custom CI image (no apk/wget per step)
# - Shallow clone (depth=2 for git diff HEAD~1)
# - TF_PLUGIN_CACHE_DIR (shared provider cache)
# - Concurrency limit (xargs -P 4)
# - Serial apply with Vault advisory locks (prevents user/CI race conditions)
# - Step consolidation (2 steps instead of 4)
# - Changed-stacks-only detection (skips no-op applies)
# - Global-file fallback (modules/config changes trigger full apply)
# - Lock-aware: skips stacks locked by users instead of failing
when:
event: push
@ -120,36 +121,52 @@ steps:
fi
fi
# ── Apply platform stacks (with concurrency limit) ──
# ── Apply platform stacks (serial, with Vault advisory locks) ──
- |
if [ -s .platform_apply ]; then
echo "=== Applying platform stacks (max 4 parallel) ==="
cat .platform_apply | xargs -P 4 -I{} sh -c '
echo "[{}] Starting apply..."
cd stacks/{} && terragrunt apply --non-interactive -auto-approve 2>&1 | tail -5
echo "=== Applying platform stacks (serial, locked) ==="
while read -r stack; do
echo "[$stack] Starting apply..."
set +e
OUTPUT=$(cd "stacks/$stack" && ../../scripts/tg apply --non-interactive 2>&1)
EXIT=$?
set -e
if [ $EXIT -ne 0 ]; then
echo "[{}] FAILED (exit $EXIT)"
if echo "$OUTPUT" | grep -q "is locked by"; then
echo "[$stack] SKIPPED (locked by another session)"
else
echo "$OUTPUT" | tail -5
echo "[$stack] FAILED (exit $EXIT)"
fi
else
echo "[{}] OK"
echo "$OUTPUT" | tail -3
echo "[$stack] OK"
fi
'
done < .platform_apply
fi
# ── Apply app stacks (with concurrency limit) ──
# ── Apply app stacks (serial, with Vault advisory locks) ──
- |
if [ -s .app_apply ]; then
echo "=== Applying app stacks (max 4 parallel) ==="
cat .app_apply | xargs -P 4 -I{} sh -c '
echo "[{}] Starting apply..."
cd stacks/{} && terragrunt apply --non-interactive -auto-approve 2>&1 | tail -5
echo "=== Applying app stacks (serial, locked) ==="
while read -r stack; do
echo "[$stack] Starting apply..."
set +e
OUTPUT=$(cd "stacks/$stack" && ../../scripts/tg apply --non-interactive 2>&1)
EXIT=$?
set -e
if [ $EXIT -ne 0 ]; then
echo "[{}] FAILED (exit $EXIT)"
if echo "$OUTPUT" | grep -q "is locked by"; then
echo "[$stack] SKIPPED (locked by another session)"
else
echo "$OUTPUT" | tail -5
echo "[$stack] FAILED (exit $EXIT)"
fi
else
echo "[{}] OK"
echo "$OUTPUT" | tail -3
echo "[$stack] OK"
fi
'
done < .app_apply
fi
# ── Commit and push state changes ──
@ -161,7 +178,12 @@ steps:
git diff --cached --quiet && echo "No changes to commit" && exit 0
git commit -m "Woodpecker CI deploy [CI SKIP]"
GIT_SSH_COMMAND='ssh -i ./secrets/deploy_key -o IdentitiesOnly=yes' git fetch origin master
GIT_SSH_COMMAND='ssh -i ./secrets/deploy_key -o IdentitiesOnly=yes' git rebase origin/master || true
if ! GIT_SSH_COMMAND='ssh -i ./secrets/deploy_key -o IdentitiesOnly=yes' git rebase origin/master; then
echo "ERROR: Git rebase failed — state commits could not be pushed"
echo "Manual intervention required: pull, resolve conflicts, push"
GIT_SSH_COMMAND='ssh -i ./secrets/deploy_key -o IdentitiesOnly=yes' git rebase --abort || true
exit 1
fi
GIT_SSH_COMMAND='ssh -i ./secrets/deploy_key -o IdentitiesOnly=yes' git push origin master
# ── Slack notification ──