feat(ci): add Vault advisory locks to CI terraform applies
CI now uses scripts/tg instead of raw terragrunt apply, acquiring the same per-stack Vault KV lock that user sessions use. This prevents CI from overwriting in-flight user applies. Changes: - Switch from xargs -P 4 (parallel) to serial while-read loop - CI skips stacks locked by users instead of racing them - Git rebase failures now exit 1 instead of silently continuing - Updated header comments to reflect new locking behavior Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
f7411327d1
commit
89af09852f
1 changed files with 40 additions and 18 deletions
|
|
@ -5,10 +5,11 @@
|
||||||
# - Custom CI image (no apk/wget per step)
|
# - Custom CI image (no apk/wget per step)
|
||||||
# - Shallow clone (depth=2 for git diff HEAD~1)
|
# - Shallow clone (depth=2 for git diff HEAD~1)
|
||||||
# - TF_PLUGIN_CACHE_DIR (shared provider cache)
|
# - TF_PLUGIN_CACHE_DIR (shared provider cache)
|
||||||
# - Concurrency limit (xargs -P 4)
|
# - Serial apply with Vault advisory locks (prevents user/CI race conditions)
|
||||||
# - Step consolidation (2 steps instead of 4)
|
# - Step consolidation (2 steps instead of 4)
|
||||||
# - Changed-stacks-only detection (skips no-op applies)
|
# - Changed-stacks-only detection (skips no-op applies)
|
||||||
# - Global-file fallback (modules/config changes trigger full apply)
|
# - Global-file fallback (modules/config changes trigger full apply)
|
||||||
|
# - Lock-aware: skips stacks locked by users instead of failing
|
||||||
|
|
||||||
when:
|
when:
|
||||||
event: push
|
event: push
|
||||||
|
|
@ -120,36 +121,52 @@ steps:
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Apply platform stacks (with concurrency limit) ──
|
# ── Apply platform stacks (serial, with Vault advisory locks) ──
|
||||||
- |
|
- |
|
||||||
if [ -s .platform_apply ]; then
|
if [ -s .platform_apply ]; then
|
||||||
echo "=== Applying platform stacks (max 4 parallel) ==="
|
echo "=== Applying platform stacks (serial, locked) ==="
|
||||||
cat .platform_apply | xargs -P 4 -I{} sh -c '
|
while read -r stack; do
|
||||||
echo "[{}] Starting apply..."
|
echo "[$stack] Starting apply..."
|
||||||
cd stacks/{} && terragrunt apply --non-interactive -auto-approve 2>&1 | tail -5
|
set +e
|
||||||
|
OUTPUT=$(cd "stacks/$stack" && ../../scripts/tg apply --non-interactive 2>&1)
|
||||||
EXIT=$?
|
EXIT=$?
|
||||||
|
set -e
|
||||||
if [ $EXIT -ne 0 ]; then
|
if [ $EXIT -ne 0 ]; then
|
||||||
echo "[{}] FAILED (exit $EXIT)"
|
if echo "$OUTPUT" | grep -q "is locked by"; then
|
||||||
|
echo "[$stack] SKIPPED (locked by another session)"
|
||||||
|
else
|
||||||
|
echo "$OUTPUT" | tail -5
|
||||||
|
echo "[$stack] FAILED (exit $EXIT)"
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
echo "[{}] OK"
|
echo "$OUTPUT" | tail -3
|
||||||
|
echo "[$stack] OK"
|
||||||
fi
|
fi
|
||||||
'
|
done < .platform_apply
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Apply app stacks (with concurrency limit) ──
|
# ── Apply app stacks (serial, with Vault advisory locks) ──
|
||||||
- |
|
- |
|
||||||
if [ -s .app_apply ]; then
|
if [ -s .app_apply ]; then
|
||||||
echo "=== Applying app stacks (max 4 parallel) ==="
|
echo "=== Applying app stacks (serial, locked) ==="
|
||||||
cat .app_apply | xargs -P 4 -I{} sh -c '
|
while read -r stack; do
|
||||||
echo "[{}] Starting apply..."
|
echo "[$stack] Starting apply..."
|
||||||
cd stacks/{} && terragrunt apply --non-interactive -auto-approve 2>&1 | tail -5
|
set +e
|
||||||
|
OUTPUT=$(cd "stacks/$stack" && ../../scripts/tg apply --non-interactive 2>&1)
|
||||||
EXIT=$?
|
EXIT=$?
|
||||||
|
set -e
|
||||||
if [ $EXIT -ne 0 ]; then
|
if [ $EXIT -ne 0 ]; then
|
||||||
echo "[{}] FAILED (exit $EXIT)"
|
if echo "$OUTPUT" | grep -q "is locked by"; then
|
||||||
|
echo "[$stack] SKIPPED (locked by another session)"
|
||||||
|
else
|
||||||
|
echo "$OUTPUT" | tail -5
|
||||||
|
echo "[$stack] FAILED (exit $EXIT)"
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
echo "[{}] OK"
|
echo "$OUTPUT" | tail -3
|
||||||
|
echo "[$stack] OK"
|
||||||
fi
|
fi
|
||||||
'
|
done < .app_apply
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Commit and push state changes ──
|
# ── Commit and push state changes ──
|
||||||
|
|
@ -161,7 +178,12 @@ steps:
|
||||||
git diff --cached --quiet && echo "No changes to commit" && exit 0
|
git diff --cached --quiet && echo "No changes to commit" && exit 0
|
||||||
git commit -m "Woodpecker CI deploy [CI SKIP]"
|
git commit -m "Woodpecker CI deploy [CI SKIP]"
|
||||||
GIT_SSH_COMMAND='ssh -i ./secrets/deploy_key -o IdentitiesOnly=yes' git fetch origin master
|
GIT_SSH_COMMAND='ssh -i ./secrets/deploy_key -o IdentitiesOnly=yes' git fetch origin master
|
||||||
GIT_SSH_COMMAND='ssh -i ./secrets/deploy_key -o IdentitiesOnly=yes' git rebase origin/master || true
|
if ! GIT_SSH_COMMAND='ssh -i ./secrets/deploy_key -o IdentitiesOnly=yes' git rebase origin/master; then
|
||||||
|
echo "ERROR: Git rebase failed — state commits could not be pushed"
|
||||||
|
echo "Manual intervention required: pull, resolve conflicts, push"
|
||||||
|
GIT_SSH_COMMAND='ssh -i ./secrets/deploy_key -o IdentitiesOnly=yes' git rebase --abort || true
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
GIT_SSH_COMMAND='ssh -i ./secrets/deploy_key -o IdentitiesOnly=yes' git push origin master
|
GIT_SSH_COMMAND='ssh -i ./secrets/deploy_key -o IdentitiesOnly=yes' git push origin master
|
||||||
|
|
||||||
# ── Slack notification ──
|
# ── Slack notification ──
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue