[forgejo] Phase 0 of registry consolidation: prepare Forgejo OCI registry
Stage 1 of moving private images off the registry:2 container at registry.viktorbarzin.me:5050 (which has hit distribution#3324 corruption 3x in 3 weeks) onto Forgejo's built-in OCI registry. No cutover risk — pods still pull from the existing registry until Phase 3. What changes: * Forgejo deployment: memory 384Mi→1Gi, PVC 5Gi→15Gi (cap 50Gi). Explicit FORGEJO__packages__ENABLED + CHUNKED_UPLOAD_PATH (defensive, v11 default-on). * ingress_factory: max_body_size variable was declared but never wired in after the nginx→Traefik migration. Now creates a per-ingress Buffering middleware when set; default null = no limit (preserves existing behavior). Forgejo ingress sets max_body_size=5g to allow multi-GB layer pushes. * Cluster-wide registry-credentials Secret: 4th auths entry for forgejo.viktorbarzin.me, populated from Vault secret/viktor/ forgejo_pull_token (cluster-puller PAT, read:package). Existing Kyverno ClusterPolicy syncs cluster-wide — no policy edits. * Containerd hosts.toml redirect: forgejo.viktorbarzin.me → in-cluster Traefik LB 10.0.20.200 (avoids hairpin NAT for in-cluster pulls). Cloud-init for new VMs + scripts/setup-forgejo-containerd-mirror.sh for existing nodes. * Forgejo retention CronJob (0 4 * * *): keeps newest 10 versions per package + always :latest. First 7 days dry-run (DRY_RUN=true); flip the local in cleanup.tf after log review. * Forgejo integrity probe CronJob (*/15): same algorithm as the existing registry-integrity-probe. Existing Prometheus alerts (RegistryManifestIntegrityFailure et al) made instance-aware so they cover both registries during the bake. * Docs: design+plan in docs/plans/, setup runbook in docs/runbooks/. Operational note — the apply order is non-trivial because the new Vault keys (forgejo_pull_token, forgejo_cleanup_token, secret/ci/global/forgejo_*) must exist BEFORE terragrunt apply in the kyverno + monitoring + forgejo stacks. The setup runbook documents the bootstrap sequence. Phase 1 (per-project dual-push pipelines) follows in subsequent commits. Bake clock starts when the last project goes dual-push. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Viktor Barzin
parent
b1c21f78b9
commit
5d22b449f9
13 changed files with 1072 additions and 10 deletions
120
stacks/forgejo/cleanup.tf
Normal file
120
stacks/forgejo/cleanup.tf
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
# Forgejo container-package retention CronJob.
|
||||
#
|
||||
# Forgejo's per-package "Cleanup Rules" UI is not exposed via Terraform —
|
||||
# it's per-user runtime state inside the Forgejo DB. Driving retention from
|
||||
# a CronJob hitting the public API keeps the policy versioned in this repo.
|
||||
#
|
||||
# Auth: a write:package PAT belonging to ci-pusher (same user that pushes
|
||||
# from CI). DELETE on packages requires write:package scope. PAT lives in
|
||||
# Vault at secret/viktor/forgejo_cleanup_token.
|
||||
|
||||
data "vault_kv_secret_v2" "forgejo_viktor" {
|
||||
mount = "secret"
|
||||
name = "viktor"
|
||||
}
|
||||
|
||||
locals {
|
||||
# Flip to false after first 7 days of dry-run logs look correct.
|
||||
forgejo_cleanup_dry_run = true
|
||||
}
|
||||
|
||||
resource "kubernetes_config_map" "forgejo_cleanup_script" {
|
||||
metadata {
|
||||
name = "forgejo-cleanup-script"
|
||||
namespace = kubernetes_namespace.forgejo.metadata[0].name
|
||||
}
|
||||
data = {
|
||||
"cleanup.sh" = file("${path.module}/files/cleanup.sh")
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_secret" "forgejo_cleanup_token" {
|
||||
metadata {
|
||||
name = "forgejo-cleanup-token"
|
||||
namespace = kubernetes_namespace.forgejo.metadata[0].name
|
||||
}
|
||||
type = "Opaque"
|
||||
data = {
|
||||
FORGEJO_TOKEN = data.vault_kv_secret_v2.forgejo_viktor.data["forgejo_cleanup_token"]
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_cron_job_v1" "forgejo_cleanup" {
|
||||
metadata {
|
||||
name = "forgejo-cleanup"
|
||||
namespace = kubernetes_namespace.forgejo.metadata[0].name
|
||||
}
|
||||
spec {
|
||||
concurrency_policy = "Forbid"
|
||||
schedule = "0 4 * * *"
|
||||
failed_jobs_history_limit = 3
|
||||
successful_jobs_history_limit = 3
|
||||
job_template {
|
||||
metadata {}
|
||||
spec {
|
||||
backoff_limit = 1
|
||||
ttl_seconds_after_finished = 3600
|
||||
template {
|
||||
metadata {}
|
||||
spec {
|
||||
container {
|
||||
name = "cleanup"
|
||||
image = "docker.io/library/alpine:3.20"
|
||||
command = ["/bin/sh", "/scripts/cleanup.sh"]
|
||||
env {
|
||||
name = "FORGEJO_TOKEN"
|
||||
value_from {
|
||||
secret_key_ref {
|
||||
name = kubernetes_secret.forgejo_cleanup_token.metadata[0].name
|
||||
key = "FORGEJO_TOKEN"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "FORGEJO_HOST"
|
||||
value = "http://forgejo.forgejo.svc.cluster.local"
|
||||
}
|
||||
env {
|
||||
name = "FORGEJO_OWNER"
|
||||
value = "viktor"
|
||||
}
|
||||
env {
|
||||
name = "KEEP_LAST_N"
|
||||
value = "10"
|
||||
}
|
||||
env {
|
||||
name = "DRY_RUN"
|
||||
value = local.forgejo_cleanup_dry_run ? "true" : "false"
|
||||
}
|
||||
volume_mount {
|
||||
name = "scripts"
|
||||
mount_path = "/scripts"
|
||||
}
|
||||
resources {
|
||||
requests = {
|
||||
cpu = "10m"
|
||||
memory = "32Mi"
|
||||
}
|
||||
limits = {
|
||||
memory = "96Mi"
|
||||
}
|
||||
}
|
||||
}
|
||||
volume {
|
||||
name = "scripts"
|
||||
config_map {
|
||||
name = kubernetes_config_map.forgejo_cleanup_script.metadata[0].name
|
||||
default_mode = "0755"
|
||||
}
|
||||
}
|
||||
restart_policy = "OnFailure"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
lifecycle {
|
||||
# KYVERNO_LIFECYCLE_V1: Kyverno admission webhook mutates dns_config with ndots=2
|
||||
ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config]
|
||||
}
|
||||
}
|
||||
109
stacks/forgejo/files/cleanup.sh
Normal file
109
stacks/forgejo/files/cleanup.sh
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
#!/bin/sh
|
||||
# Forgejo container-package retention.
|
||||
#
|
||||
# For each container package owned by ${FORGEJO_OWNER}, keep newest
|
||||
# ${KEEP_LAST_N} versions + always keep tag "latest". Deletes the rest via
|
||||
# DELETE /api/v1/packages/{owner}/container/{name}/{version}.
|
||||
#
|
||||
# DRY_RUN=true logs what would be deleted but issues no DELETE calls.
|
||||
#
|
||||
# Required env:
|
||||
# FORGEJO_HOST e.g. http://forgejo.forgejo.svc.cluster.local
|
||||
# FORGEJO_OWNER e.g. viktor
|
||||
# FORGEJO_USER PAT owner (write:package scope)
|
||||
# FORGEJO_TOKEN PAT
|
||||
# KEEP_LAST_N integer (default 10)
|
||||
# DRY_RUN true|false (default true)
|
||||
|
||||
set -eu
|
||||
|
||||
apk add --no-cache curl jq >/dev/null
|
||||
|
||||
OWNER="${FORGEJO_OWNER}"
|
||||
KEEP="${KEEP_LAST_N:-10}"
|
||||
DRY="${DRY_RUN:-true}"
|
||||
BASE="${FORGEJO_HOST%/}/api/v1"
|
||||
|
||||
AUTH_HEADER="Authorization: token $FORGEJO_TOKEN"
|
||||
|
||||
echo "Forgejo cleanup: owner=$OWNER keep_last=$KEEP dry_run=$DRY"
|
||||
echo "API base: $BASE"
|
||||
|
||||
# Page through ALL container packages.
|
||||
TMPDIR=$(mktemp -d)
|
||||
trap 'rm -rf "$TMPDIR"' EXIT
|
||||
ALL="$TMPDIR/all.json"
|
||||
echo "[]" > "$ALL"
|
||||
|
||||
PAGE=1
|
||||
while :; do
|
||||
RESP=$(curl -sf -H "$AUTH_HEADER" \
|
||||
"$BASE/packages/$OWNER?type=container&limit=50&page=$PAGE")
|
||||
COUNT=$(echo "$RESP" | jq 'length')
|
||||
if [ "$COUNT" = "0" ]; then break; fi
|
||||
jq -s '.[0] + .[1]' "$ALL" <(echo "$RESP") > "$TMPDIR/merged.json"
|
||||
mv "$TMPDIR/merged.json" "$ALL"
|
||||
PAGE=$((PAGE + 1))
|
||||
# Safety: never run away.
|
||||
if [ "$PAGE" -gt 100 ]; then break; fi
|
||||
done
|
||||
|
||||
TOTAL=$(jq 'length' "$ALL")
|
||||
echo "Found $TOTAL package version(s)."
|
||||
|
||||
if [ "$TOTAL" = "0" ]; then
|
||||
echo "Nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Group by name and process each group.
|
||||
NAMES=$(jq -r '.[].name' "$ALL" | sort -u)
|
||||
|
||||
DEL=0
|
||||
KEPT=0
|
||||
|
||||
for NAME in $NAMES; do
|
||||
# All versions of this name, sorted by created_at descending.
|
||||
jq --arg n "$NAME" '
|
||||
[.[] | select(.name == $n)]
|
||||
| sort_by(.created_at) | reverse
|
||||
' "$ALL" > "$TMPDIR/$NAME.json"
|
||||
|
||||
N_VERSIONS=$(jq 'length' "$TMPDIR/$NAME.json")
|
||||
echo "[$NAME] $N_VERSIONS version(s)"
|
||||
|
||||
# Build the keep set: top $KEEP + anything tagged 'latest'.
|
||||
jq -r --argjson keep "$KEEP" '
|
||||
[.[0:$keep][].version] + [.[] | select(.version == "latest") | .version]
|
||||
| unique
|
||||
| .[]
|
||||
' "$TMPDIR/$NAME.json" > "$TMPDIR/$NAME.keep"
|
||||
|
||||
# Build the delete set.
|
||||
jq -r '.[].version' "$TMPDIR/$NAME.json" \
|
||||
| grep -vxFf "$TMPDIR/$NAME.keep" > "$TMPDIR/$NAME.delete" || true
|
||||
|
||||
D_COUNT=$(wc -l < "$TMPDIR/$NAME.delete" | tr -d ' ')
|
||||
K_COUNT=$(wc -l < "$TMPDIR/$NAME.keep" | tr -d ' ')
|
||||
echo " keep=$K_COUNT delete=$D_COUNT"
|
||||
KEPT=$((KEPT + K_COUNT))
|
||||
|
||||
while IFS= read -r VER; do
|
||||
[ -z "$VER" ] && continue
|
||||
URL="$BASE/packages/$OWNER/container/$NAME/$VER"
|
||||
if [ "$DRY" = "true" ]; then
|
||||
echo " DRY_RUN would DELETE $URL"
|
||||
else
|
||||
HTTP=$(curl -s -o /dev/null -w '%{http_code}' \
|
||||
-X DELETE -H "$AUTH_HEADER" "$URL" || echo "000")
|
||||
if [ "$HTTP" = "204" ] || [ "$HTTP" = "200" ]; then
|
||||
echo " deleted $NAME:$VER"
|
||||
else
|
||||
echo " FAIL $NAME:$VER HTTP $HTTP"
|
||||
fi
|
||||
fi
|
||||
DEL=$((DEL + 1))
|
||||
done < "$TMPDIR/$NAME.delete"
|
||||
done
|
||||
|
||||
echo "Summary: kept=$KEPT to_delete=$DEL dry_run=$DRY"
|
||||
|
|
@ -32,7 +32,7 @@ resource "kubernetes_persistent_volume_claim" "data_encrypted" {
|
|||
annotations = {
|
||||
"resize.topolvm.io/threshold" = "80%"
|
||||
"resize.topolvm.io/increase" = "50%"
|
||||
"resize.topolvm.io/storage_limit" = "20Gi"
|
||||
"resize.topolvm.io/storage_limit" = "50Gi"
|
||||
}
|
||||
}
|
||||
spec {
|
||||
|
|
@ -40,7 +40,7 @@ resource "kubernetes_persistent_volume_claim" "data_encrypted" {
|
|||
storage_class_name = "proxmox-lvm-encrypted"
|
||||
resources {
|
||||
requests = {
|
||||
storage = "5Gi"
|
||||
storage = "15Gi"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -106,6 +106,18 @@ resource "kubernetes_deployment" "forgejo" {
|
|||
name = "FORGEJO__webhook__ALLOWED_HOST_LIST"
|
||||
value = "*.svc.cluster.local"
|
||||
}
|
||||
# OCI registry (container packages). Default-on in Forgejo v11 but
|
||||
# explicit so it can't be silently disabled by an upstream config
|
||||
# change. Chunked-upload path needs a directory inside /data so it
|
||||
# survives pod restarts and shares the same PVC as the registry blobs.
|
||||
env {
|
||||
name = "FORGEJO__packages__ENABLED"
|
||||
value = "true"
|
||||
}
|
||||
env {
|
||||
name = "FORGEJO__packages__CHUNKED_UPLOAD_PATH"
|
||||
value = "/data/tmp/package-upload"
|
||||
}
|
||||
volume_mount {
|
||||
name = "data"
|
||||
mount_path = "/data"
|
||||
|
|
@ -113,10 +125,10 @@ resource "kubernetes_deployment" "forgejo" {
|
|||
resources {
|
||||
requests = {
|
||||
cpu = "15m"
|
||||
memory = "384Mi"
|
||||
memory = "1Gi"
|
||||
}
|
||||
limits = {
|
||||
memory = "384Mi"
|
||||
memory = "1Gi"
|
||||
}
|
||||
}
|
||||
port {
|
||||
|
|
@ -165,6 +177,9 @@ module "ingress" {
|
|||
namespace = kubernetes_namespace.forgejo.metadata[0].name
|
||||
name = "forgejo"
|
||||
tls_secret_name = var.tls_secret_name
|
||||
# OCI registry pushes ship full image layer blobs in one request; default
|
||||
# Traefik buffering chokes on anything past a few hundred MB.
|
||||
max_body_size = "5g"
|
||||
extra_annotations = {
|
||||
"gethomepage.dev/enabled" = "true"
|
||||
"gethomepage.dev/name" = "Forgejo"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue