From 3c3e6bfc958b4e5b737e3e13963f1bf9e1cba20b Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Sat, 13 Jun 2026 10:07:58 +0000 Subject: [PATCH] ci: retire in-cluster infra-ci build; breakglass becomes manual ghcr pull-and-save (ADR-0002 #30) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit infra-ci now builds on GHA → ghcr and the ghcr-based apply is PROVEN (pipeline 165 ran terragrunt apply in the ghcr image). Removing the Woodpecker build-ci-image.yml (clean cut). The breakglass tarball is preserved as a MANUAL Woodpecker job pulling ghcr (public) → registry VM; infra-ci on ghcr is external + node-cached, so the Forgejo-down rationale for the old auto-tarball is moot — this is belt-and-braces DR. Co-Authored-By: Claude Fable 5 --- .woodpecker/breakglass-infra-ci.yml | 31 ++++++++++ .woodpecker/build-ci-image.yml | 88 ----------------------------- 2 files changed, 31 insertions(+), 88 deletions(-) create mode 100644 .woodpecker/breakglass-infra-ci.yml delete mode 100644 .woodpecker/build-ci-image.yml diff --git a/.woodpecker/breakglass-infra-ci.yml b/.woodpecker/breakglass-infra-ci.yml new file mode 100644 index 00000000..bbc43d7d --- /dev/null +++ b/.woodpecker/breakglass-infra-ci.yml @@ -0,0 +1,31 @@ +# Break-glass: save the ghcr infra-ci image to a tarball on the registry VM +# (10.0.20.10) so it can be `docker load`-ed onto a node if ghcr is ever +# unreachable during a recovery. infra-ci now builds on GHA → ghcr (ADR-0002), +# which is external + node-cached, so this is a belt-and-braces DR artifact — +# run MANUALLY after an infra-ci rebuild (or periodically). Pulls from ghcr +# (public, no login). Recovery: docs/runbooks/forgejo-registry-breakglass.md. +when: + - event: manual + +steps: + - name: breakglass-tarball + image: alpine:3.20 + failure: ignore + environment: + REGISTRY_SSH_KEY: + from_secret: registry_ssh_key + commands: + - apk add --no-cache openssh-client + - mkdir -p ~/.ssh && chmod 700 ~/.ssh + - printf '%s\n' "$REGISTRY_SSH_KEY" > ~/.ssh/id_ed25519 + - chmod 600 ~/.ssh/id_ed25519 + - ssh-keyscan -t ed25519 10.0.20.10 >> ~/.ssh/known_hosts 2>/dev/null + - | + ssh -n -o BatchMode=yes root@10.0.20.10 " + set -e + mkdir -p /opt/registry/data/private/_breakglass + IMAGE=ghcr.io/viktorbarzin/infra-ci:latest + docker pull \$IMAGE + docker save \$IMAGE | gzip > /opt/registry/data/private/_breakglass/infra-ci-latest.tar.gz + ls -lh /opt/registry/data/private/_breakglass/infra-ci-latest.tar.gz + " diff --git a/.woodpecker/build-ci-image.yml b/.woodpecker/build-ci-image.yml deleted file mode 100644 index 796426ac..00000000 --- a/.woodpecker/build-ci-image.yml +++ /dev/null @@ -1,88 +0,0 @@ -# Build the CI tools Docker image used by all infra pipelines. -# Triggers on push that touches ci/Dockerfile, or manual (API/UI) so -# rebuilds after a registry incident don't need a cosmetic Dockerfile edit. - -when: - - event: push - branch: master - path: - include: - - 'ci/Dockerfile' - - event: manual - -steps: - - name: build-and-push - image: woodpeckerci/plugin-docker-buildx - settings: - # Phase 4 of forgejo-registry-consolidation 2026-05-07 — - # registry.viktorbarzin.me dropped, Forgejo is the only target. - repo: - - forgejo.viktorbarzin.me/viktor/infra-ci - dockerfile: ci/Dockerfile - context: ci/ - tags: - - latest - - "${CI_COMMIT_SHA:0:8}" - platforms: linux/amd64 - logins: - - registry: forgejo.viktorbarzin.me - username: - from_secret: forgejo_user - password: - from_secret: forgejo_push_token - - # Post-push integrity check is now redundant with the every-15min - # forgejo-integrity-probe in stacks/monitoring/, which walks - # /v2/_catalog + HEADs every blob across the entire Forgejo registry. - # If a corruption pattern emerges that the periodic probe misses, - # restore a verify step similar to the pre-Phase-4 version (see - # commit 49f4956f) but pointed at forgejo.viktorbarzin.me. - - # Break-glass tarball: save the just-pushed infra-ci image to disk on the - # registry VM (10.0.20.10) so we can `docker load` it back into a node - # when Forgejo is unreachable. Pulls from Forgejo (the only registry now). - # Best-effort — failure here doesn't fail the pipeline. - # Recovery procedure: docs/runbooks/forgejo-registry-breakglass.md. - - name: breakglass-tarball - image: alpine:3.20 - failure: ignore - environment: - REGISTRY_SSH_KEY: - from_secret: registry_ssh_key - FORGEJO_USER: - from_secret: forgejo_user - FORGEJO_PASS: - from_secret: forgejo_push_token - commands: - - apk add --no-cache openssh-client - - mkdir -p ~/.ssh && chmod 700 ~/.ssh - - printf '%s\n' "$REGISTRY_SSH_KEY" > ~/.ssh/id_ed25519 - - chmod 600 ~/.ssh/id_ed25519 - - ssh-keyscan -t ed25519 10.0.20.10 >> ~/.ssh/known_hosts 2>/dev/null - - SHA=${CI_COMMIT_SHA:0:8} - - | - ssh -n -o BatchMode=yes root@10.0.20.10 " - set -e - mkdir -p /opt/registry/data/private/_breakglass - IMAGE=forgejo.viktorbarzin.me/viktor/infra-ci:$SHA - echo \$FORGEJO_PASS | docker login forgejo.viktorbarzin.me -u \$FORGEJO_USER --password-stdin - docker pull \$IMAGE - docker save \$IMAGE | gzip > /opt/registry/data/private/_breakglass/infra-ci-$SHA.tar.gz - ln -sfn infra-ci-$SHA.tar.gz /opt/registry/data/private/_breakglass/infra-ci-latest.tar.gz - ls -t /opt/registry/data/private/_breakglass/infra-ci-*.tar.gz \ - | grep -v 'latest' | tail -n +6 | xargs -r rm -v - ls -lh /opt/registry/data/private/_breakglass/ - " - - - name: slack - image: curlimages/curl - commands: - - | - curl -s -X POST -H 'Content-type: application/json' \ - --data "{\"text\":\"CI image built: forgejo.viktorbarzin.me/viktor/infra-ci:${CI_COMMIT_SHA:0:8} (and registry-private mirror)\"}" \ - "$SLACK_WEBHOOK" || true - environment: - SLACK_WEBHOOK: - from_secret: slack_webhook - when: - status: [success]