diff --git a/modules/create-vm/main.tf b/modules/create-vm/main.tf index b7e451df..90420b4e 100644 --- a/modules/create-vm/main.tf +++ b/modules/create-vm/main.tf @@ -34,12 +34,16 @@ variable "ipconfig0" { type = string default = "ip=dhcp,ip6=dhcp" } +variable "agent" { + type = number + default = 0 +} resource "proxmox_vm_qemu" "cloudinit-vm" { vmid = var.vmid name = var.vm_name target_node = "pve" - agent = 0 + agent = var.agent memory = var.vm_mem_mb boot = "order=scsi0" # has to be the same as the OS disk of the template clone = var.template_name # The name of the template diff --git a/modules/docker-registry/cleanup-tags.sh b/modules/docker-registry/cleanup-tags.sh index ceaa85b1..b822db5e 100644 --- a/modules/docker-registry/cleanup-tags.sh +++ b/modules/docker-registry/cleanup-tags.sh @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Keeps only the N most recent tags per image in a pull-through cache registry. +"""Keeps only the N most recent tags per image in pull-through cache registries. Deletes old tag links directly from the filesystem since the API doesn't support DELETE on proxy registries. Run garbage-collect after to reclaim blob storage.""" @@ -10,39 +10,40 @@ import sys sys.stdout.reconfigure(line_buffering=True) KEEP = int(sys.argv[1]) if len(sys.argv) > 1 else 10 - -STORAGE = "/var/lib/docker/volumes/57b3f1c5fcc7f39c040e17072e10b4536245357d09340206683c04096d30b942/_data/docker/registry/v2/repositories" +BASE = sys.argv[2] if len(sys.argv) > 2 else "/opt/registry/data" total_deleted = 0 -for root, dirs, _ in os.walk(STORAGE): - # Look for _manifests/tags directories - if not root.endswith("_manifests/tags"): +for registry_name in sorted(os.listdir(BASE)): + storage = os.path.join(BASE, registry_name, "docker/registry/v2/repositories") + if not os.path.isdir(storage): continue - repo = root.replace(STORAGE + "/", "").replace("/_manifests/tags", "") + for root, dirs, _ in os.walk(storage): + if not root.endswith("_manifests/tags"): + continue - # Get tags with modification times - tag_times = [] - for tag in os.listdir(root): - tag_path = os.path.join(root, tag) - if os.path.isdir(tag_path): - mtime = os.path.getmtime(tag_path) - tag_times.append((mtime, tag, tag_path)) + repo = root.replace(storage + "/", "").replace("/_manifests/tags", "") - if len(tag_times) <= KEEP: - continue + tag_times = [] + for tag in os.listdir(root): + tag_path = os.path.join(root, tag) + if os.path.isdir(tag_path): + mtime = os.path.getmtime(tag_path) + tag_times.append((mtime, tag, tag_path)) - # Sort by mtime descending (newest first) - tag_times.sort(reverse=True) - to_delete = tag_times[KEEP:] + if len(tag_times) <= KEEP: + continue - print(f"[{repo}] {len(tag_times)} tags -> keeping {KEEP}, deleting {len(to_delete)}") + tag_times.sort(reverse=True) + to_delete = tag_times[KEEP:] - for _, tag, tag_path in to_delete: - shutil.rmtree(tag_path) - total_deleted += 1 + print(f"[{registry_name}/{repo}] {len(tag_times)} tags -> keeping {KEEP}, deleting {len(to_delete)}") - print(f" done") + for _, tag, tag_path in to_delete: + shutil.rmtree(tag_path) + total_deleted += 1 -print(f"\nDeleted {total_deleted} tags. Restart registry and run garbage-collect to reclaim space.") + print(f" done") + +print(f"\nDeleted {total_deleted} tags. Run garbage-collect to reclaim space.") diff --git a/modules/docker-registry/docker-compose.yml b/modules/docker-registry/docker-compose.yml new file mode 100644 index 00000000..983437b8 --- /dev/null +++ b/modules/docker-registry/docker-compose.yml @@ -0,0 +1,143 @@ +networks: + registry: + driver: bridge + +services: + registry-dockerhub: + image: registry:2 + container_name: registry-dockerhub + restart: always + volumes: + - /opt/registry/data/dockerhub:/var/lib/registry + - /opt/registry/config-dockerhub.yml:/etc/docker/registry/config.yml:ro + networks: + - registry + ports: + - "5001:5001" + healthcheck: + test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + + registry-ghcr: + image: registry:2 + container_name: registry-ghcr + restart: always + volumes: + - /opt/registry/data/ghcr:/var/lib/registry + - /opt/registry/config-ghcr.yml:/etc/docker/registry/config.yml:ro + networks: + - registry + healthcheck: + test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + + registry-quay: + image: registry:2 + container_name: registry-quay + restart: always + volumes: + - /opt/registry/data/quay:/var/lib/registry + - /opt/registry/config-quay.yml:/etc/docker/registry/config.yml:ro + networks: + - registry + healthcheck: + test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + + registry-k8s: + image: registry:2 + container_name: registry-k8s + restart: always + volumes: + - /opt/registry/data/k8s:/var/lib/registry + - /opt/registry/config-k8s.yml:/etc/docker/registry/config.yml:ro + networks: + - registry + healthcheck: + test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + + registry-kyverno: + image: registry:2 + container_name: registry-kyverno + restart: always + volumes: + - /opt/registry/data/kyverno:/var/lib/registry + - /opt/registry/config-kyverno.yml:/etc/docker/registry/config.yml:ro + networks: + - registry + healthcheck: + test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + + nginx: + image: nginx:alpine + container_name: registry-nginx + restart: always + ports: + - "5000:5000" + - "5010:5010" + - "5020:5020" + - "5030:5030" + - "5040:5040" + volumes: + - /opt/registry/nginx.conf:/etc/nginx/nginx.conf:ro + - nginx-cache:/var/cache/nginx + networks: + - registry + depends_on: + registry-dockerhub: + condition: service_healthy + registry-ghcr: + condition: service_healthy + registry-quay: + condition: service_healthy + registry-k8s: + condition: service_healthy + registry-kyverno: + condition: service_healthy + healthcheck: + test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 15s + + registry-ui: + image: joxit/docker-registry-ui:latest + container_name: registry-ui + restart: always + ports: + - "8080:80" + environment: + - NGINX_PROXY_PASS_URL=http://registry-dockerhub:5000 + - DELETE_IMAGES=true + - SINGLE_REGISTRY=true + - SHOW_CONTENT_DIGEST=true + - SHOW_CATALOG_NB_TAGS=true + - CATALOG_ELEMENTS_LIMIT=1000 + - TAGLIST_PAGE_SIZE=100 + - REGISTRY_TITLE=viktorbarzin.me + networks: + - registry + depends_on: + registry-dockerhub: + condition: service_healthy + +volumes: + nginx-cache: diff --git a/modules/docker-registry/nginx_registry.conf b/modules/docker-registry/nginx_registry.conf index 3391079e..15e43994 100644 --- a/modules/docker-registry/nginx_registry.conf +++ b/modules/docker-registry/nginx_registry.conf @@ -1,58 +1,220 @@ -proxy_cache_path /var/cache/nginx/registry - levels=1:2 - keys_zone=registry:500m - max_size=50g - inactive=24h - use_temp_path=off; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /tmp/nginx.pid; -upstream docker_registry { - server 127.0.0.1:5000; - keepalive 32; +events { + worker_connections 1024; } -server { - listen 5002; - server_name _; +http { + proxy_cache_path /var/cache/nginx/registry + levels=1:2 + keys_zone=registry:500m + max_size=50g + inactive=24h + use_temp_path=off; - # Access log - access_log /var/log/nginx/registry.access.log combined; + log_format registry '$remote_addr [$time_local] "$request" ' + '$status $body_bytes_sent ' + 'upstream=$upstream_addr time=$upstream_response_time ' + 'cache=$upstream_cache_status'; - # Error log - error_log /var/log/nginx/registry.error.log warn; + access_log /var/log/nginx/access.log registry; - # Required for large blobs - client_max_body_size 0; + # --- Upstreams --- - # Disable buffering to clients, keep it between nginx<->registry - proxy_request_buffering off; - proxy_buffering on; + upstream dockerhub { + server registry-dockerhub:5000; + keepalive 32; + } - location /v2/ { - proxy_pass http://docker_registry; + upstream ghcr { + server registry-ghcr:5000; + keepalive 32; + } - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header Connection ""; + upstream quay { + server registry-quay:5000; + keepalive 32; + } - # --- CRITICAL PART --- - proxy_cache registry; - proxy_cache_lock on; - proxy_cache_lock_timeout 15m; - proxy_cache_lock_age 15m; - proxy_cache_use_stale updating; + upstream k8s { + server registry-k8s:5000; + keepalive 32; + } - # Cache only successful pulls - proxy_cache_valid 200 206 24h; + upstream kyverno { + server registry-kyverno:5000; + keepalive 32; + } - # HEAD requests must not poison cache - proxy_cache_methods GET; + # --- Docker Hub (port 5000) --- - # Do not cache pushes - proxy_no_cache $http_authorization; - proxy_cache_bypass $http_authorization; + server { + listen 5000; + server_name _; - # Prevent partial responses - proxy_read_timeout 900; - proxy_send_timeout 900; + client_max_body_size 0; + proxy_request_buffering off; + proxy_buffering on; + + location /v2/ { + proxy_pass http://dockerhub; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Connection ""; + + proxy_cache registry; + proxy_cache_lock on; + proxy_cache_lock_timeout 15m; + proxy_cache_lock_age 15m; + proxy_cache_use_stale updating; + proxy_cache_valid 200 206 24h; + proxy_cache_methods GET; + + proxy_read_timeout 900; + proxy_send_timeout 900; + } + + location / { + return 200 'ok'; + add_header Content-Type text/plain; + } + } + + # --- GHCR (port 5010) --- + + server { + listen 5010; + server_name _; + + client_max_body_size 0; + proxy_request_buffering off; + proxy_buffering on; + + location /v2/ { + proxy_pass http://ghcr; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Connection ""; + + proxy_cache registry; + proxy_cache_lock on; + proxy_cache_lock_timeout 15m; + proxy_cache_lock_age 15m; + proxy_cache_use_stale updating; + proxy_cache_valid 200 206 24h; + proxy_cache_methods GET; + + proxy_read_timeout 900; + proxy_send_timeout 900; + } + + location / { + return 200 'ok'; + add_header Content-Type text/plain; + } + } + + # --- Quay (port 5020) --- + + server { + listen 5020; + server_name _; + + client_max_body_size 0; + proxy_request_buffering off; + proxy_buffering on; + + location /v2/ { + proxy_pass http://quay; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Connection ""; + + proxy_cache registry; + proxy_cache_lock on; + proxy_cache_lock_timeout 15m; + proxy_cache_lock_age 15m; + proxy_cache_use_stale updating; + proxy_cache_valid 200 206 24h; + proxy_cache_methods GET; + + proxy_read_timeout 900; + proxy_send_timeout 900; + } + + location / { + return 200 'ok'; + add_header Content-Type text/plain; + } + } + + # --- registry.k8s.io (port 5030) --- + + server { + listen 5030; + server_name _; + + client_max_body_size 0; + proxy_request_buffering off; + proxy_buffering on; + + location /v2/ { + proxy_pass http://k8s; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Connection ""; + + proxy_cache registry; + proxy_cache_lock on; + proxy_cache_lock_timeout 15m; + proxy_cache_lock_age 15m; + proxy_cache_use_stale updating; + proxy_cache_valid 200 206 24h; + proxy_cache_methods GET; + + proxy_read_timeout 900; + proxy_send_timeout 900; + } + + location / { + return 200 'ok'; + add_header Content-Type text/plain; + } + } + + # --- reg.kyverno.io (port 5040) --- + + server { + listen 5040; + server_name _; + + client_max_body_size 0; + proxy_request_buffering off; + proxy_buffering on; + + location /v2/ { + proxy_pass http://kyverno; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Connection ""; + + proxy_cache registry; + proxy_cache_lock on; + proxy_cache_lock_timeout 15m; + proxy_cache_lock_age 15m; + proxy_cache_use_stale updating; + proxy_cache_valid 200 206 24h; + proxy_cache_methods GET; + + proxy_read_timeout 900; + proxy_send_timeout 900; + } + + location / { + return 200 'ok'; + add_header Content-Type text/plain; + } } } diff --git a/stacks/infra/main.tf b/stacks/infra/main.tf index 6fd4374f..15b073a5 100644 --- a/stacks/infra/main.tf +++ b/stacks/infra/main.tf @@ -141,22 +141,33 @@ module "docker-registry-template" { # Setup registry config and start container provision_cmds = [ - "mkdir -p /etc/docker-registry", - format("echo %s | base64 -d > /etc/docker-registry/config.yml", + # Install and enable QEMU guest agent for remote management + "apt-get install -y qemu-guest-agent", + "systemctl enable qemu-guest-agent", + "systemctl start qemu-guest-agent", + # Stop host nginx — we run nginx inside Docker instead + "systemctl stop nginx || true", + "systemctl disable nginx || true", + # Create directory structure + "mkdir -p /opt/registry/data/dockerhub /opt/registry/data/ghcr /opt/registry/data/quay /opt/registry/data/k8s /opt/registry/data/kyverno", + # Write Docker Compose file + format("echo %s | base64 -d > /opt/registry/docker-compose.yml", + base64encode(file("${path.root}/../../modules/docker-registry/docker-compose.yml")) + ), + # Write nginx config + format("echo %s | base64 -d > /opt/registry/nginx.conf", + base64encode(file("${path.root}/../../modules/docker-registry/nginx_registry.conf")) + ), + # Write Docker Hub registry config (with auth) + format("echo %s | base64 -d > /opt/registry/config-dockerhub.yml", base64encode( templatefile("../../modules/docker-registry/config.yaml", { password = var.dockerhub_registry_password - } - ) + }) ) ), - "( crontab -l 2>/dev/null; echo '0 3 * * 0 /usr/bin/docker exec registry registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -", - # Hourly restart cron removed - it wiped the in-memory blobdescriptor cache every hour, - # causing low cache hit rates on the pull-through proxy. Docker containers use --restart always. - "docker run -p 5000:5000 -p 5001:5001 -d --restart always --name registry -v /etc/docker-registry/config.yml:/etc/docker/registry/config.yml registry:2", - # ghcr.io proxy - "mkdir -p /etc/docker-registry/ghcr", - format("echo %s | base64 -d > /etc/docker-registry/ghcr/config.yml", + # Write GHCR registry config + format("echo %s | base64 -d > /opt/registry/config-ghcr.yml", base64encode( templatefile("../../modules/docker-registry/config-proxy.yaml.tpl", { name = "ghcr" @@ -164,11 +175,8 @@ module "docker-registry-template" { }) ) ), - "docker run -p 5010:5000 -d --restart always --name registry-ghcr -v /etc/docker-registry/ghcr/config.yml:/etc/docker/registry/config.yml registry:2", - "( crontab -l 2>/dev/null; echo '5 3 * * 0 /usr/bin/docker exec registry-ghcr registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -", - # quay.io proxy - "mkdir -p /etc/docker-registry/quay", - format("echo %s | base64 -d > /etc/docker-registry/quay/config.yml", + # Write Quay registry config + format("echo %s | base64 -d > /opt/registry/config-quay.yml", base64encode( templatefile("../../modules/docker-registry/config-proxy.yaml.tpl", { name = "quay" @@ -176,11 +184,8 @@ module "docker-registry-template" { }) ) ), - "docker run -p 5020:5000 -d --restart always --name registry-quay -v /etc/docker-registry/quay/config.yml:/etc/docker/registry/config.yml registry:2", - "( crontab -l 2>/dev/null; echo '10 3 * * 0 /usr/bin/docker exec registry-quay registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -", - # registry.k8s.io proxy - "mkdir -p /etc/docker-registry/k8s", - format("echo %s | base64 -d > /etc/docker-registry/k8s/config.yml", + # Write registry.k8s.io registry config + format("echo %s | base64 -d > /opt/registry/config-k8s.yml", base64encode( templatefile("../../modules/docker-registry/config-proxy.yaml.tpl", { name = "k8s" @@ -188,11 +193,8 @@ module "docker-registry-template" { }) ) ), - "docker run -p 5030:5000 -d --restart always --name registry-k8s -v /etc/docker-registry/k8s/config.yml:/etc/docker/registry/config.yml registry:2", - "( crontab -l 2>/dev/null; echo '15 3 * * 0 /usr/bin/docker exec registry-k8s registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -", - # reg.kyverno.io proxy - "mkdir -p /etc/docker-registry/kyverno", - format("echo %s | base64 -d > /etc/docker-registry/kyverno/config.yml", + # Write reg.kyverno.io registry config + format("echo %s | base64 -d > /opt/registry/config-kyverno.yml", base64encode( templatefile("../../modules/docker-registry/config-proxy.yaml.tpl", { name = "kyverno" @@ -200,22 +202,43 @@ module "docker-registry-template" { }) ) ), - "docker run -p 5040:5000 -d --restart always --name registry-kyverno -v /etc/docker-registry/kyverno/config.yml:/etc/docker/registry/config.yml registry:2", - "( crontab -l 2>/dev/null; echo '20 3 * * 0 /usr/bin/docker exec registry-kyverno registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -", - # Setup the registry nginx config; We want clients to connect via the nginx to serialize requests for the same blobs - # Otherwise race conditions lead to corrupt blobs - "mkdir -p /var/cache/nginx/registry", - format("echo %s | base64 -d > /etc/nginx/conf.d/registry.conf", - base64encode( - templatefile("${path.root}/../../modules/docker-registry/nginx_registry.conf", {}) - ) - ), - "docker run -d --restart always --net host --name registry-ui -e NGINX_LISTEN_PORT=8080 -e NGINX_PROXY_PASS_URL=http://127.0.0.1:5000 -e DELETE_IMAGES=true -e SINGLE_REGISTRY=true -e SHOW_CONTENT_DIGEST=true -e SHOW_CATALOG_NB_TAGS=true -e CATALOG_ELEMENTS_LIMIT=1000 -e TAGLIST_PAGE_SIZE=100 -e REGISTRY_TITLE=viktorbarzin.me joxit/docker-registry-ui:latest", - # Deploy tag cleanup script (keep last 10 tags per image) and schedule daily at 2am before weekly GC - format("echo %s | base64 -d > /etc/docker-registry/cleanup-tags.sh && chmod +x /etc/docker-registry/cleanup-tags.sh", + # Write tag cleanup script + format("echo %s | base64 -d > /opt/registry/cleanup-tags.sh && chmod +x /opt/registry/cleanup-tags.sh", base64encode(file("${path.root}/../../modules/docker-registry/cleanup-tags.sh")) ), - "( crontab -l 2>/dev/null; echo '0 2 * * * python3 /etc/docker-registry/cleanup-tags.sh 10 >> /var/log/registry-cleanup.log 2>&1' ) | crontab -", + # Create systemd unit for docker compose + format("echo %s | base64 -d > /etc/systemd/system/docker-compose-registry.service", + base64encode(<<-UNIT +[Unit] +Description=Docker Compose Registry Stack +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory=/opt/registry +ExecStart=/usr/bin/docker compose up -d --remove-orphans +ExecStop=/usr/bin/docker compose down +TimeoutStartSec=300 + +[Install] +WantedBy=multi-user.target +UNIT + ) + ), + # Enable and start the registry stack + "systemctl daemon-reload", + "systemctl enable docker-compose-registry.service", + "systemctl start docker-compose-registry.service", + # Cron: garbage collection (weekly, Sunday 3am, staggered per registry) + "( crontab -l 2>/dev/null; echo '0 3 * * 0 /usr/bin/docker exec registry-dockerhub registry garbage-collect -m /etc/docker/registry/config.yml >> /var/log/registry-gc.log 2>&1' ) | crontab -", + "( crontab -l 2>/dev/null; echo '5 3 * * 0 /usr/bin/docker exec registry-ghcr registry garbage-collect -m /etc/docker/registry/config.yml >> /var/log/registry-gc.log 2>&1' ) | crontab -", + "( crontab -l 2>/dev/null; echo '10 3 * * 0 /usr/bin/docker exec registry-quay registry garbage-collect -m /etc/docker/registry/config.yml >> /var/log/registry-gc.log 2>&1' ) | crontab -", + "( crontab -l 2>/dev/null; echo '15 3 * * 0 /usr/bin/docker exec registry-k8s registry garbage-collect -m /etc/docker/registry/config.yml >> /var/log/registry-gc.log 2>&1' ) | crontab -", + "( crontab -l 2>/dev/null; echo '20 3 * * 0 /usr/bin/docker exec registry-kyverno registry garbage-collect -m /etc/docker/registry/config.yml >> /var/log/registry-gc.log 2>&1' ) | crontab -", + # Cron: tag cleanup (daily 2am, keep last 10 tags per image) + "( crontab -l 2>/dev/null; echo '0 2 * * * python3 /opt/registry/cleanup-tags.sh 10 >> /var/log/registry-cleanup.log 2>&1' ) | crontab -", ] } @@ -234,18 +257,18 @@ module "docker-registry-vm" { template_name = "docker-registry-template" vm_name = "docker-registry" cisnippet_name = "docker-registry.yaml" + agent = 1 vm_mac_address = "DE:AD:BE:EF:22:22" # mapped to 10.0.20.10 in dhcp bridge = "vmbr1" vlan_tag = "20" ipconfig0 = "ip=10.0.20.10/24,gw=10.0.20.1" - # ports: - # 5000 -> registry (docker.io proxy) - # 5001 -> metrics - # 5002 -> nginx proxy <-- use this to prevent races on the same blobs - # 5010 -> registry-ghcr (ghcr.io proxy) - # 5020 -> registry-quay (quay.io proxy) - # 5030 -> registry-k8s (registry.k8s.io proxy) - # 5040 -> registry-kyverno (reg.kyverno.io proxy) + # All ports go through nginx for request serialization (proxy_cache_lock): + # 5000 -> nginx -> registry-dockerhub (docker.io proxy) + # 5001 -> registry-dockerhub direct (Prometheus metrics) + # 5010 -> nginx -> registry-ghcr (ghcr.io proxy) + # 5020 -> nginx -> registry-quay (quay.io proxy) + # 5030 -> nginx -> registry-k8s (registry.k8s.io proxy) + # 5040 -> nginx -> registry-kyverno (reg.kyverno.io proxy) # 8080 -> registry-ui (joxit/docker-registry-ui) }