ci-pipeline-health: daily sweep of the off-infra CI chain (ADR-0002)
Viktor asked to monitor the pipelines closely as builds move off-infra (PRD infra#10). New aux stack: daily 07:30 UTC CronJob on the claude-agent-service image running a deterministic shell sweep — GitHub Actions failures/stuck runs across owned repos, Woodpecker pipeline failures, GHA free-tier minutes burn. Healthy = one quiet Slack line; issues = Slack alert + comment on infra#10. In-cluster (not a cloud routine) because Vault + the Woodpecker token are LAN-only. Secrets via ExternalSecret (github_pat deliberately, not the ghcr_pull_token alias — a scoped packages-only rotation couldn't read Actions runs). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Viktor Barzin
parent
98f1f7fc24
commit
d02ca4f2db
3 changed files with 280 additions and 0 deletions
112
stacks/ci-pipeline-health/files/sweep.sh
Normal file
112
stacks/ci-pipeline-health/files/sweep.sh
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
#!/bin/sh
|
||||
# ci-pipeline-health — daily sweep of the off-infra CI chain (ADR-0002, PRD infra#10).
|
||||
# Deterministic (no LLM): GitHub Actions runs + Woodpecker pipelines + GHA minutes.
|
||||
# Healthy => one quiet Slack line. Issues => Slack alert + comment on infra#10.
|
||||
# POSIX sh + curl + jq only (runs on the Alpine claude-agent-service image).
|
||||
# Exit 0 = sweep ran (even with findings); exit 2 = the sweep itself errored,
|
||||
# which surfaces through the existing CronJob-failure alerting.
|
||||
|
||||
GH_API="https://api.github.com"
|
||||
WP_API="https://ci.viktorbarzin.me/api"
|
||||
WP_UI="https://ci.viktorbarzin.me"
|
||||
|
||||
NOW_EPOCH=$(date -u +%s)
|
||||
SINCE_EPOCH=$((NOW_EPOCH - 86400))
|
||||
SINCE_ISO=$(date -u -d "@${SINCE_EPOCH}" +%Y-%m-%dT%H:%M:%SZ)
|
||||
PUSH_CUTOFF=$(date -u -d "@$((NOW_EPOCH - 259200))" +%Y-%m-%dT%H:%M:%SZ)
|
||||
|
||||
ISSUES=$(mktemp)
|
||||
NOTES=$(mktemp)
|
||||
trap 'rm -f "$ISSUES" "$NOTES"' EXIT
|
||||
gha_checked=0
|
||||
wp_checked=0
|
||||
sweep_errors=0
|
||||
|
||||
gh_get() { curl -sf --max-time 30 -H "Authorization: Bearer ${GITHUB_PAT}" -H "Accept: application/vnd.github+json" "$1"; }
|
||||
wp_get() { curl -sf --max-time 30 -H "Authorization: Bearer ${WOODPECKER_API_TOKEN}" "$1"; }
|
||||
|
||||
# --- 1) GitHub Actions runs across owned repos with a recent push ---
|
||||
repos=$(gh_get "${GH_API}/user/repos?affiliation=owner&sort=pushed&per_page=60" \
|
||||
| jq -r --arg cutoff "$PUSH_CUTOFF" '.[] | select(.pushed_at >= $cutoff) | .full_name')
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "sweep: failed to list GitHub repos" >>"$ISSUES"; sweep_errors=1; repos=""
|
||||
fi
|
||||
for repo in $repos; do
|
||||
runs=$(gh_get "${GH_API}/repos/${repo}/actions/runs?created=%3E%3D${SINCE_ISO}&per_page=50")
|
||||
if [ $? -ne 0 ]; then echo "sweep: failed to list runs for ${repo}" >>"$ISSUES"; sweep_errors=1; continue; fi
|
||||
n=$(printf '%s' "$runs" | jq '.workflow_runs | length')
|
||||
gha_checked=$((gha_checked + n))
|
||||
printf '%s' "$runs" | jq -r '.workflow_runs[]
|
||||
| select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled" or .conclusion == "action_required")
|
||||
| "GHA: \(.repository.full_name) #\(.run_number) [\(.name)] \(.conclusion) \(.html_url)"' >>"$ISSUES"
|
||||
printf '%s' "$runs" | jq -r --argjson now "$NOW_EPOCH" '.workflow_runs[]
|
||||
| select(.status == "in_progress" or .status == "queued")
|
||||
| select(($now - ((.run_started_at // .created_at) | fromdateiso8601)) > 7200)
|
||||
| "GHA stuck >2h: \(.repository.full_name) #\(.run_number) [\(.name)] \(.status) \(.html_url)"' >>"$ISSUES"
|
||||
done
|
||||
|
||||
# --- 2) Woodpecker pipelines (deploy chain) ---
|
||||
wrepos=$(wp_get "${WP_API}/repos?perPage=100" | jq -r '.[] | select(.active == true) | "\(.id) \(.full_name)"')
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "sweep: failed to list Woodpecker repos" >>"$ISSUES"; sweep_errors=1; wrepos=""
|
||||
fi
|
||||
printf '%s\n' "$wrepos" | while IFS=' ' read -r id name; do
|
||||
[ -z "$id" ] && continue
|
||||
pls=$(wp_get "${WP_API}/repos/${id}/pipelines?perPage=10")
|
||||
if [ $? -ne 0 ]; then echo "sweep: failed pipelines for ${name}" >>"$ISSUES"; continue; fi
|
||||
printf '%s' "$pls" | jq -r --argjson since "$SINCE_EPOCH" --arg name "$name" --arg ui "$WP_UI" --arg id "$id" '
|
||||
[.[] | select(.created >= $since)][]
|
||||
| select(.status == "failure" or .status == "error" or .status == "killed")
|
||||
| "Woodpecker: \($name) #\(.number) \(.status) (\(.event)) \($ui)/repos/\($id)/pipeline/\(.number)"' >>"$ISSUES"
|
||||
printf '%s' "$pls" | jq --argjson since "$SINCE_EPOCH" '[.[] | select(.created >= $since)] | length' >>"$NOTES.wpcount" 2>/dev/null || true
|
||||
done
|
||||
wp_checked=$(awk '{s+=$1} END {print s+0}' "$NOTES.wpcount" 2>/dev/null || echo 0)
|
||||
rm -f "$NOTES.wpcount"
|
||||
|
||||
# --- 3) GHA minutes vs free tier ---
|
||||
billing=$(gh_get "${GH_API}/users/ViktorBarzin/settings/billing/actions")
|
||||
if [ $? -eq 0 ]; then
|
||||
used=$(printf '%s' "$billing" | jq -r '.total_minutes_used')
|
||||
included=$(printf '%s' "$billing" | jq -r '.included_minutes')
|
||||
if [ "${included:-0}" -gt 0 ] 2>/dev/null; then
|
||||
pct=$((used * 100 / included))
|
||||
echo "GHA minutes: ${used}/${included} (${pct}%)" >>"$NOTES"
|
||||
[ "$pct" -ge 75 ] && echo "GHA minutes at ${pct}% of the free tier (${used}/${included}) — check for runaway workflows or consider Pro" >>"$ISSUES"
|
||||
fi
|
||||
else
|
||||
echo "minutes check unavailable" >>"$NOTES"
|
||||
fi
|
||||
|
||||
# v1 scope (deliberate, not silent): Forgejo→GitHub mirror-gap detection (a
|
||||
# Forgejo push that produced no GHA run) is NOT implemented yet — it needs the
|
||||
# per-repo mirror inventory that lands with the offinfra-onboard rollout (#13+).
|
||||
|
||||
# --- Report ---
|
||||
issue_count=$(grep -c . "$ISSUES" || true)
|
||||
summary="ci-pipeline-health: checked ${gha_checked} GHA runs + ${wp_checked} Woodpecker pipelines (24h). $(tr '\n' '; ' <"$NOTES")"
|
||||
|
||||
if [ "$issue_count" -eq 0 ]; then
|
||||
text=":white_check_mark: ${summary}"
|
||||
else
|
||||
text=":rotating_light: ci-pipeline-health: ${issue_count} issue(s)
|
||||
$(sed 's/^/• /' "$ISSUES")
|
||||
${summary}"
|
||||
body="Daily CI sweep found ${issue_count} issue(s):
|
||||
|
||||
$(sed 's/^/- /' "$ISSUES")
|
||||
|
||||
_${summary}_"
|
||||
printf '%s' "$body" | jq -Rs '{body: .}' \
|
||||
| curl -sf --max-time 30 -X POST -H "Authorization: Bearer ${GITHUB_PAT}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-d @- "${GH_API}/repos/ViktorBarzin/infra/issues/10/comments" >/dev/null \
|
||||
|| { echo "sweep: failed to comment on infra#10"; sweep_errors=1; }
|
||||
fi
|
||||
|
||||
printf '%s' "$text" | jq -Rs '{text: .}' \
|
||||
| curl -sf --max-time 30 -X POST -H 'Content-Type: application/json' -d @- "$SLACK_WEBHOOK" >/dev/null \
|
||||
|| { echo "sweep: failed to post to Slack"; sweep_errors=1; }
|
||||
|
||||
echo "$text"
|
||||
[ "$sweep_errors" -ne 0 ] && exit 2
|
||||
exit 0
|
||||
159
stacks/ci-pipeline-health/main.tf
Normal file
159
stacks/ci-pipeline-health/main.tf
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
# ci-pipeline-health — daily sweep of the off-infra CI chain (ADR-0002).
|
||||
#
|
||||
# Viktor's standing instruction (2026-06-12): monitor the pipelines closely
|
||||
# during/after the off-infra builds migration (PRD infra#10). Deterministic
|
||||
# shell sweep (files/sweep.sh) on the claude-agent-service image: GitHub
|
||||
# Actions failures/stuck runs across owned repos, Woodpecker pipeline
|
||||
# failures, GHA free-tier minutes burn. Healthy => one quiet Slack line;
|
||||
# issues => Slack alert + a comment on infra#10.
|
||||
#
|
||||
# Runs IN-CLUSTER (not a claude.ai cloud routine) because Vault and the
|
||||
# Woodpecker token are LAN-only — cloud agents can't reach them.
|
||||
|
||||
variable "schedule" {
|
||||
type = string
|
||||
# 07:30 UTC = 08:30 London in summer (07:30 in winter — acceptable drift,
|
||||
# CronJob schedules are UTC-only).
|
||||
default = "30 7 * * *"
|
||||
}
|
||||
|
||||
# Mirrors stacks/claude-agent-service image tag — the image ships curl + jq.
|
||||
variable "image_tag" {
|
||||
type = string
|
||||
default = "2fd7670d"
|
||||
}
|
||||
|
||||
locals {
|
||||
namespace = "ci-pipeline-health"
|
||||
image = "forgejo.viktorbarzin.me/viktor/claude-agent-service:${var.image_tag}"
|
||||
labels = {
|
||||
app = "ci-pipeline-health"
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_namespace" "ci_pipeline_health" {
|
||||
metadata {
|
||||
name = local.namespace
|
||||
labels = {
|
||||
tier = local.tiers.aux
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# github_pat (NOT the ghcr_pull_token alias): the sweep reads Actions runs +
|
||||
# billing on PRIVATE mirrors, which a future scoped read:packages rotation of
|
||||
# the alias could not do. Blast radius = this single-CronJob namespace.
|
||||
resource "kubernetes_manifest" "external_secret" {
|
||||
manifest = {
|
||||
apiVersion = "external-secrets.io/v1beta1"
|
||||
kind = "ExternalSecret"
|
||||
metadata = {
|
||||
name = "ci-pipeline-health-creds"
|
||||
namespace = kubernetes_namespace.ci_pipeline_health.metadata[0].name
|
||||
}
|
||||
spec = {
|
||||
refreshInterval = "15m"
|
||||
secretStoreRef = {
|
||||
name = "vault-kv"
|
||||
kind = "ClusterSecretStore"
|
||||
}
|
||||
target = {
|
||||
name = "ci-pipeline-health-creds"
|
||||
}
|
||||
data = [
|
||||
{
|
||||
secretKey = "GITHUB_PAT"
|
||||
remoteRef = { key = "viktor", property = "github_pat" }
|
||||
},
|
||||
{
|
||||
secretKey = "WOODPECKER_API_TOKEN"
|
||||
remoteRef = { key = "ci/global", property = "woodpecker_api_token" }
|
||||
},
|
||||
{
|
||||
secretKey = "SLACK_WEBHOOK"
|
||||
remoteRef = { key = "ci/global", property = "slack_webhook" }
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_config_map" "sweep_script" {
|
||||
metadata {
|
||||
name = "ci-pipeline-health-sweep"
|
||||
namespace = kubernetes_namespace.ci_pipeline_health.metadata[0].name
|
||||
}
|
||||
data = {
|
||||
"sweep.sh" = file("${path.module}/files/sweep.sh")
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_cron_job_v1" "sweep" {
|
||||
metadata {
|
||||
name = "ci-pipeline-health"
|
||||
namespace = kubernetes_namespace.ci_pipeline_health.metadata[0].name
|
||||
labels = local.labels
|
||||
}
|
||||
spec {
|
||||
schedule = var.schedule
|
||||
concurrency_policy = "Forbid"
|
||||
successful_jobs_history_limit = 3
|
||||
failed_jobs_history_limit = 3
|
||||
job_template {
|
||||
metadata {
|
||||
labels = local.labels
|
||||
}
|
||||
spec {
|
||||
backoff_limit = 1
|
||||
active_deadline_seconds = 600
|
||||
ttl_seconds_after_finished = 86400
|
||||
template {
|
||||
metadata {
|
||||
labels = local.labels
|
||||
}
|
||||
spec {
|
||||
restart_policy = "Never"
|
||||
image_pull_secrets {
|
||||
name = "registry-credentials"
|
||||
}
|
||||
container {
|
||||
name = "sweep"
|
||||
image = local.image
|
||||
command = ["/bin/sh", "/scripts/sweep.sh"]
|
||||
env_from {
|
||||
secret_ref {
|
||||
name = "ci-pipeline-health-creds"
|
||||
}
|
||||
}
|
||||
volume_mount {
|
||||
name = "sweep-script"
|
||||
mount_path = "/scripts"
|
||||
read_only = true
|
||||
}
|
||||
resources {
|
||||
requests = {
|
||||
cpu = "50m"
|
||||
memory = "64Mi"
|
||||
}
|
||||
limits = {
|
||||
memory = "128Mi"
|
||||
}
|
||||
}
|
||||
}
|
||||
volume {
|
||||
name = "sweep-script"
|
||||
config_map {
|
||||
name = kubernetes_config_map.sweep_script.metadata[0].name
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
lifecycle {
|
||||
# KYVERNO_LIFECYCLE_V1
|
||||
ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config]
|
||||
}
|
||||
depends_on = [kubernetes_manifest.external_secret]
|
||||
}
|
||||
9
stacks/ci-pipeline-health/terragrunt.hcl
Normal file
9
stacks/ci-pipeline-health/terragrunt.hcl
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
include "root" {
|
||||
path = find_in_parent_folders()
|
||||
}
|
||||
|
||||
# ExternalSecret hits ESO which needs to be alive when the manifest applies.
|
||||
dependency "external_secrets" {
|
||||
config_path = "../external-secrets"
|
||||
skip_outputs = true
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue