fix: restore tree dropped by 6d224861; land stem95su gdrive-sync (10m) [ci skip]
6d224861 came from a --no-checkout worktree whose empty index made the
commit drop every file except two. This restores 05b50d2b's full tree and
correctly adds stacks/stem95su/gdrive-sync.tf + the service-catalog stem95su
entry. Forward-only (parent=6d224861, no force-push); [ci skip] since the
live infra was never applied from the broken commit.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>This commit is contained in:
Viktor Barzin
parent
6d224861c4
commit
fd0f4a0365
1166 changed files with 358546 additions and 0 deletions
151
.woodpecker/drift-detection.yml
Normal file
151
.woodpecker/drift-detection.yml
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
# Daily drift detection — runs terraform plan on all stacks and alerts on drift.
|
||||
# Triggered by Woodpecker cron schedule "drift-detection" (must be registered in Woodpecker UI/API).
|
||||
|
||||
when:
|
||||
event: cron
|
||||
cron: drift-detection
|
||||
|
||||
clone:
|
||||
git:
|
||||
image: woodpeckerci/plugin-git
|
||||
settings:
|
||||
depth: 1
|
||||
attempts: 3
|
||||
|
||||
steps:
|
||||
- name: detect-drift
|
||||
image: forgejo.viktorbarzin.me/viktor/infra-ci:latest
|
||||
pull: true
|
||||
backend_options:
|
||||
kubernetes:
|
||||
resources:
|
||||
requests:
|
||||
memory: 2Gi
|
||||
limits:
|
||||
memory: 4Gi
|
||||
environment:
|
||||
SLACK_WEBHOOK:
|
||||
from_secret: slack_webhook
|
||||
commands:
|
||||
# ── git-crypt unlock ──
|
||||
- |
|
||||
SA_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
|
||||
curl -sk "https://10.0.20.100:6443/api/v1/namespaces/woodpecker/configmaps/git-crypt-key" \
|
||||
-H "Authorization:Bearer $SA_TOKEN" | jq -r .data.key | base64 -d > /tmp/key
|
||||
git-crypt unlock /tmp/key && rm /tmp/key
|
||||
|
||||
# ── Vault auth ──
|
||||
- |
|
||||
SA_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
|
||||
export VAULT_ADDR=http://vault-active.vault.svc.cluster.local:8200
|
||||
export VAULT_TOKEN=$(curl -s -X POST "$VAULT_ADDR/v1/auth/kubernetes/login" \
|
||||
-d "{\"role\":\"ci\",\"jwt\":\"$SA_TOKEN\"}" | jq -r .auth.client_token)
|
||||
|
||||
# ── Generate kubeconfig from projected SA token ──
|
||||
# See default.yml for rationale. terragrunt.hcl injects
|
||||
# `-var kube_config_path=<repo>/config` for every terraform invocation,
|
||||
# so we need a kubeconfig file at that path. The woodpecker default SA
|
||||
# is cluster-admin, so the projected token is sufficient.
|
||||
- |
|
||||
cat > config <<'EOF'
|
||||
apiVersion: v1
|
||||
kind: Config
|
||||
clusters:
|
||||
- name: kubernetes
|
||||
cluster:
|
||||
server: https://10.0.20.100:6443
|
||||
certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
contexts:
|
||||
- name: ci
|
||||
context:
|
||||
cluster: kubernetes
|
||||
user: ci
|
||||
current-context: ci
|
||||
users:
|
||||
- name: ci
|
||||
user:
|
||||
tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
EOF
|
||||
chmod 600 config
|
||||
kubectl --kubeconfig=config get ns kube-system -o name >/dev/null
|
||||
|
||||
# ── Run terraform plan on all stacks ──
|
||||
# Emits two timestamps per drifted stack so the Pushgateway/Prometheus
|
||||
# side can compute drift-age-hours via `time() - drift_stack_first_seen`.
|
||||
- |
|
||||
DRIFTED=""
|
||||
CLEAN=0
|
||||
ERRORS=""
|
||||
NOW=$(date +%s)
|
||||
# Metrics accumulator — written once per stack, then pushed as a batch.
|
||||
METRICS=""
|
||||
|
||||
for stack_dir in stacks/*/; do
|
||||
stack=$(basename "$stack_dir")
|
||||
[ -f "$stack_dir/terragrunt.hcl" ] || continue
|
||||
|
||||
echo -n "[$stack] planning... "
|
||||
OUTPUT=$(cd "$stack_dir" && terragrunt plan -detailed-exitcode -input=false 2>&1)
|
||||
EXIT=$?
|
||||
|
||||
case $EXIT in
|
||||
0)
|
||||
echo "OK (no changes)"
|
||||
CLEAN=$((CLEAN + 1))
|
||||
# drift_stack_state=0 means clean; age-hours irrelevant so we
|
||||
# still push 0 so per-stack gauges don't go stale.
|
||||
METRICS="${METRICS}drift_stack_state{stack=\"$stack\"} 0\n"
|
||||
METRICS="${METRICS}drift_stack_age_hours{stack=\"$stack\"} 0\n"
|
||||
;;
|
||||
1)
|
||||
echo "ERROR"
|
||||
ERRORS="$ERRORS $stack"
|
||||
METRICS="${METRICS}drift_stack_state{stack=\"$stack\"} 2\n"
|
||||
;;
|
||||
2)
|
||||
echo "DRIFT DETECTED"
|
||||
DRIFTED="$DRIFTED $stack"
|
||||
# Fetch first-seen timestamp from Pushgateway (preserve across runs).
|
||||
FIRST_SEEN=$(curl -s "http://prometheus-prometheus-pushgateway.monitoring:9091/metrics" \
|
||||
| awk -v s="$stack" '$1 == "drift_stack_first_seen{stack=\""s"\"}" {print $2; exit}')
|
||||
if [ -z "$FIRST_SEEN" ] || [ "$FIRST_SEEN" = "0" ]; then
|
||||
FIRST_SEEN="$NOW"
|
||||
fi
|
||||
AGE_HOURS=$(( (NOW - FIRST_SEEN) / 3600 ))
|
||||
METRICS="${METRICS}drift_stack_state{stack=\"$stack\"} 1\n"
|
||||
METRICS="${METRICS}drift_stack_first_seen{stack=\"$stack\"} $FIRST_SEEN\n"
|
||||
METRICS="${METRICS}drift_stack_age_hours{stack=\"$stack\"} $AGE_HOURS\n"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Summary counters — single gauge per run.
|
||||
DRIFT_COUNT=$(echo "$DRIFTED" | wc -w)
|
||||
ERROR_COUNT=$(echo "$ERRORS" | wc -w)
|
||||
METRICS="${METRICS}drift_stack_count $DRIFT_COUNT\n"
|
||||
METRICS="${METRICS}drift_error_count $ERROR_COUNT\n"
|
||||
METRICS="${METRICS}drift_clean_count $CLEAN\n"
|
||||
METRICS="${METRICS}drift_detection_last_run_timestamp $NOW\n"
|
||||
|
||||
# ── Push to Pushgateway ──
|
||||
# One batched push keeps the run atomic: either all metrics land or none.
|
||||
printf "%b" "$METRICS" | curl -s --data-binary @- \
|
||||
http://prometheus-prometheus-pushgateway.monitoring:9091/metrics/job/drift-detection \
|
||||
|| echo "(pushgateway unavailable, metrics lost for this run)"
|
||||
|
||||
echo ""
|
||||
echo "=== Drift Detection Summary ==="
|
||||
echo "Clean: $CLEAN stacks"
|
||||
echo "Drift: ${DRIFTED:-none}"
|
||||
echo "Errors: ${ERRORS:-none}"
|
||||
|
||||
# ── Slack alert if drift found ──
|
||||
if [ -n "$DRIFTED" ]; then
|
||||
curl -s -X POST -H 'Content-type: application/json' \
|
||||
--data "{\"channel\":\"general\",\"text\":\":warning: Drift detected in:${DRIFTED}\nClean: ${CLEAN} stacks. Errors:${ERRORS:-none}\"}" \
|
||||
"$SLACK_WEBHOOK" || true
|
||||
else
|
||||
curl -s -X POST -H 'Content-type: application/json' \
|
||||
--data "{\"channel\":\"general\",\"text\":\":white_check_mark: Drift detection: all ${CLEAN} stacks clean${ERRORS:+. Errors: $ERRORS}\"}" \
|
||||
"$SLACK_WEBHOOK" || true
|
||||
fi
|
||||
Loading…
Add table
Add a link
Reference in a new issue