[ci skip] Rebuild docker-registry with nginx serialization on all ports

Replace individual `docker run` commands with Docker Compose stack managed
by systemd. Nginx now fronts all 5 registry ports (5000/5010/5020/5030/5040)
with proxy_cache_lock to serialize concurrent blob pulls and prevent
corrupt partial responses. Adds QEMU guest agent for remote management.

modules/create-vm/main.tf

@@ -34,12 +34,16 @@ variable "ipconfig0" {
   type    = string
   default = "ip=dhcp,ip6=dhcp"
 }
+variable "agent" {
+  type    = number
+  default = 0
+}
 
 resource "proxmox_vm_qemu" "cloudinit-vm" {
   vmid             = var.vmid
   name             = var.vm_name
   target_node      = "pve"
-  agent            = 0
+  agent            = var.agent
   memory           = var.vm_mem_mb
   boot             = "order=scsi0"     # has to be the same as the OS disk of the template
   clone            = var.template_name # The name of the template

modules/docker-registry/cleanup-tags.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Keeps only the N most recent tags per image in a pull-through cache registry.
+"""Keeps only the N most recent tags per image in pull-through cache registries.
 Deletes old tag links directly from the filesystem since the API doesn't support
 DELETE on proxy registries. Run garbage-collect after to reclaim blob storage."""
 
@@ -10,39 +10,40 @@ import sys
 sys.stdout.reconfigure(line_buffering=True)
 
 KEEP = int(sys.argv[1]) if len(sys.argv) > 1 else 10
-
+BASE = sys.argv[2] if len(sys.argv) > 2 else "/opt/registry/data"
-STORAGE = "/var/lib/docker/volumes/57b3f1c5fcc7f39c040e17072e10b4536245357d09340206683c04096d30b942/_data/docker/registry/v2/repositories"
 
 total_deleted = 0
 
-for root, dirs, _ in os.walk(STORAGE):
+for registry_name in sorted(os.listdir(BASE)):
-    # Look for _manifests/tags directories
+    storage = os.path.join(BASE, registry_name, "docker/registry/v2/repositories")
-    if not root.endswith("_manifests/tags"):
+    if not os.path.isdir(storage):
         continue
 
-    repo = root.replace(STORAGE + "/", "").replace("/_manifests/tags", "")
+    for root, dirs, _ in os.walk(storage):
+        if not root.endswith("_manifests/tags"):
+            continue
 
-    # Get tags with modification times
+        repo = root.replace(storage + "/", "").replace("/_manifests/tags", "")
-    tag_times = []
-    for tag in os.listdir(root):
-        tag_path = os.path.join(root, tag)
-        if os.path.isdir(tag_path):
-            mtime = os.path.getmtime(tag_path)
-            tag_times.append((mtime, tag, tag_path))
 
-    if len(tag_times) <= KEEP:
+        tag_times = []
-        continue
+        for tag in os.listdir(root):
+            tag_path = os.path.join(root, tag)
+            if os.path.isdir(tag_path):
+                mtime = os.path.getmtime(tag_path)
+                tag_times.append((mtime, tag, tag_path))
 
-    # Sort by mtime descending (newest first)
+        if len(tag_times) <= KEEP:
-    tag_times.sort(reverse=True)
+            continue
-    to_delete = tag_times[KEEP:]
 
-    print(f"[{repo}] {len(tag_times)} tags -> keeping {KEEP}, deleting {len(to_delete)}")
+        tag_times.sort(reverse=True)
+        to_delete = tag_times[KEEP:]
 
-    for _, tag, tag_path in to_delete:
+        print(f"[{registry_name}/{repo}] {len(tag_times)} tags -> keeping {KEEP}, deleting {len(to_delete)}")
-        shutil.rmtree(tag_path)
-        total_deleted += 1
 
-    print(f"  done")
+        for _, tag, tag_path in to_delete:
+            shutil.rmtree(tag_path)
+            total_deleted += 1
 
-print(f"\nDeleted {total_deleted} tags. Restart registry and run garbage-collect to reclaim space.")
+        print(f"  done")
+
+print(f"\nDeleted {total_deleted} tags. Run garbage-collect to reclaim space.")

modules/docker-registry/docker-compose.yml

@@ -0,0 +1,143 @@
+networks:
+  registry:
+    driver: bridge
+
+services:
+  registry-dockerhub:
+    image: registry:2
+    container_name: registry-dockerhub
+    restart: always
+    volumes:
+      - /opt/registry/data/dockerhub:/var/lib/registry
+      - /opt/registry/config-dockerhub.yml:/etc/docker/registry/config.yml:ro
+    networks:
+      - registry
+    ports:
+      - "5001:5001"
+    healthcheck:
+      test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+
+  registry-ghcr:
+    image: registry:2
+    container_name: registry-ghcr
+    restart: always
+    volumes:
+      - /opt/registry/data/ghcr:/var/lib/registry
+      - /opt/registry/config-ghcr.yml:/etc/docker/registry/config.yml:ro
+    networks:
+      - registry
+    healthcheck:
+      test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+
+  registry-quay:
+    image: registry:2
+    container_name: registry-quay
+    restart: always
+    volumes:
+      - /opt/registry/data/quay:/var/lib/registry
+      - /opt/registry/config-quay.yml:/etc/docker/registry/config.yml:ro
+    networks:
+      - registry
+    healthcheck:
+      test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+
+  registry-k8s:
+    image: registry:2
+    container_name: registry-k8s
+    restart: always
+    volumes:
+      - /opt/registry/data/k8s:/var/lib/registry
+      - /opt/registry/config-k8s.yml:/etc/docker/registry/config.yml:ro
+    networks:
+      - registry
+    healthcheck:
+      test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+
+  registry-kyverno:
+    image: registry:2
+    container_name: registry-kyverno
+    restart: always
+    volumes:
+      - /opt/registry/data/kyverno:/var/lib/registry
+      - /opt/registry/config-kyverno.yml:/etc/docker/registry/config.yml:ro
+    networks:
+      - registry
+    healthcheck:
+      test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+
+  nginx:
+    image: nginx:alpine
+    container_name: registry-nginx
+    restart: always
+    ports:
+      - "5000:5000"
+      - "5010:5010"
+      - "5020:5020"
+      - "5030:5030"
+      - "5040:5040"
+    volumes:
+      - /opt/registry/nginx.conf:/etc/nginx/nginx.conf:ro
+      - nginx-cache:/var/cache/nginx
+    networks:
+      - registry
+    depends_on:
+      registry-dockerhub:
+        condition: service_healthy
+      registry-ghcr:
+        condition: service_healthy
+      registry-quay:
+        condition: service_healthy
+      registry-k8s:
+        condition: service_healthy
+      registry-kyverno:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 15s
+
+  registry-ui:
+    image: joxit/docker-registry-ui:latest
+    container_name: registry-ui
+    restart: always
+    ports:
+      - "8080:80"
+    environment:
+      - NGINX_PROXY_PASS_URL=http://registry-dockerhub:5000
+      - DELETE_IMAGES=true
+      - SINGLE_REGISTRY=true
+      - SHOW_CONTENT_DIGEST=true
+      - SHOW_CATALOG_NB_TAGS=true
+      - CATALOG_ELEMENTS_LIMIT=1000
+      - TAGLIST_PAGE_SIZE=100
+      - REGISTRY_TITLE=viktorbarzin.me
+    networks:
+      - registry
+    depends_on:
+      registry-dockerhub:
+        condition: service_healthy
+
+volumes:
+  nginx-cache:

modules/docker-registry/nginx_registry.conf

@@ -1,58 +1,220 @@
-proxy_cache_path /var/cache/nginx/registry
+worker_processes auto;
-    levels=1:2
+error_log /var/log/nginx/error.log warn;
-    keys_zone=registry:500m
+pid /tmp/nginx.pid;
-    max_size=50g
-    inactive=24h
-    use_temp_path=off;
 
-upstream docker_registry {
+events {
-    server 127.0.0.1:5000;
+    worker_connections 1024;
-    keepalive 32;
 }
 
-server {
+http {
-    listen 5002;
+    proxy_cache_path /var/cache/nginx/registry
-    server_name _;
+        levels=1:2
+        keys_zone=registry:500m
+        max_size=50g
+        inactive=24h
+        use_temp_path=off;
 
-    # Access log
+    log_format registry '$remote_addr [$time_local] "$request" '
-    access_log /var/log/nginx/registry.access.log combined;
+                        '$status $body_bytes_sent '
+                        'upstream=$upstream_addr time=$upstream_response_time '
+                        'cache=$upstream_cache_status';
 
-    # Error log
+    access_log /var/log/nginx/access.log registry;
-    error_log /var/log/nginx/registry.error.log warn;
 
-    # Required for large blobs
+    # --- Upstreams ---
-    client_max_body_size 0;
 
-    # Disable buffering to clients, keep it between nginx<->registry
+    upstream dockerhub {
-    proxy_request_buffering off;
+        server registry-dockerhub:5000;
-    proxy_buffering on;
+        keepalive 32;
+    }
 
-    location /v2/ {
+    upstream ghcr {
-        proxy_pass http://docker_registry;
+        server registry-ghcr:5000;
+        keepalive 32;
+    }
 
-        proxy_http_version 1.1;
+    upstream quay {
-        proxy_set_header Host $host;
+        server registry-quay:5000;
-        proxy_set_header Connection "";
+        keepalive 32;
+    }
 
-        # --- CRITICAL PART ---
+    upstream k8s {
-        proxy_cache registry;
+        server registry-k8s:5000;
-        proxy_cache_lock on;
+        keepalive 32;
-        proxy_cache_lock_timeout 15m;
+    }
-        proxy_cache_lock_age 15m;
-        proxy_cache_use_stale updating;
 
-        # Cache only successful pulls
+    upstream kyverno {
-        proxy_cache_valid 200 206 24h;
+        server registry-kyverno:5000;
+        keepalive 32;
+    }
 
-        # HEAD requests must not poison cache
+    # --- Docker Hub (port 5000) ---
-        proxy_cache_methods GET;
 
-        # Do not cache pushes
+    server {
-        proxy_no_cache $http_authorization;
+        listen 5000;
-        proxy_cache_bypass $http_authorization;
+        server_name _;
 
-        # Prevent partial responses
+        client_max_body_size 0;
-        proxy_read_timeout 900;
+        proxy_request_buffering off;
-        proxy_send_timeout 900;
+        proxy_buffering on;
+
+        location /v2/ {
+            proxy_pass http://dockerhub;
+            proxy_http_version 1.1;
+            proxy_set_header Host $host;
+            proxy_set_header Connection "";
+
+            proxy_cache registry;
+            proxy_cache_lock on;
+            proxy_cache_lock_timeout 15m;
+            proxy_cache_lock_age 15m;
+            proxy_cache_use_stale updating;
+            proxy_cache_valid 200 206 24h;
+            proxy_cache_methods GET;
+
+            proxy_read_timeout 900;
+            proxy_send_timeout 900;
+        }
+
+        location / {
+            return 200 'ok';
+            add_header Content-Type text/plain;
+        }
+    }
+
+    # --- GHCR (port 5010) ---
+
+    server {
+        listen 5010;
+        server_name _;
+
+        client_max_body_size 0;
+        proxy_request_buffering off;
+        proxy_buffering on;
+
+        location /v2/ {
+            proxy_pass http://ghcr;
+            proxy_http_version 1.1;
+            proxy_set_header Host $host;
+            proxy_set_header Connection "";
+
+            proxy_cache registry;
+            proxy_cache_lock on;
+            proxy_cache_lock_timeout 15m;
+            proxy_cache_lock_age 15m;
+            proxy_cache_use_stale updating;
+            proxy_cache_valid 200 206 24h;
+            proxy_cache_methods GET;
+
+            proxy_read_timeout 900;
+            proxy_send_timeout 900;
+        }
+
+        location / {
+            return 200 'ok';
+            add_header Content-Type text/plain;
+        }
+    }
+
+    # --- Quay (port 5020) ---
+
+    server {
+        listen 5020;
+        server_name _;
+
+        client_max_body_size 0;
+        proxy_request_buffering off;
+        proxy_buffering on;
+
+        location /v2/ {
+            proxy_pass http://quay;
+            proxy_http_version 1.1;
+            proxy_set_header Host $host;
+            proxy_set_header Connection "";
+
+            proxy_cache registry;
+            proxy_cache_lock on;
+            proxy_cache_lock_timeout 15m;
+            proxy_cache_lock_age 15m;
+            proxy_cache_use_stale updating;
+            proxy_cache_valid 200 206 24h;
+            proxy_cache_methods GET;
+
+            proxy_read_timeout 900;
+            proxy_send_timeout 900;
+        }
+
+        location / {
+            return 200 'ok';
+            add_header Content-Type text/plain;
+        }
+    }
+
+    # --- registry.k8s.io (port 5030) ---
+
+    server {
+        listen 5030;
+        server_name _;
+
+        client_max_body_size 0;
+        proxy_request_buffering off;
+        proxy_buffering on;
+
+        location /v2/ {
+            proxy_pass http://k8s;
+            proxy_http_version 1.1;
+            proxy_set_header Host $host;
+            proxy_set_header Connection "";
+
+            proxy_cache registry;
+            proxy_cache_lock on;
+            proxy_cache_lock_timeout 15m;
+            proxy_cache_lock_age 15m;
+            proxy_cache_use_stale updating;
+            proxy_cache_valid 200 206 24h;
+            proxy_cache_methods GET;
+
+            proxy_read_timeout 900;
+            proxy_send_timeout 900;
+        }
+
+        location / {
+            return 200 'ok';
+            add_header Content-Type text/plain;
+        }
+    }
+
+    # --- reg.kyverno.io (port 5040) ---
+
+    server {
+        listen 5040;
+        server_name _;
+
+        client_max_body_size 0;
+        proxy_request_buffering off;
+        proxy_buffering on;
+
+        location /v2/ {
+            proxy_pass http://kyverno;
+            proxy_http_version 1.1;
+            proxy_set_header Host $host;
+            proxy_set_header Connection "";
+
+            proxy_cache registry;
+            proxy_cache_lock on;
+            proxy_cache_lock_timeout 15m;
+            proxy_cache_lock_age 15m;
+            proxy_cache_use_stale updating;
+            proxy_cache_valid 200 206 24h;
+            proxy_cache_methods GET;
+
+            proxy_read_timeout 900;
+            proxy_send_timeout 900;
+        }
+
+        location / {
+            return 200 'ok';
+            add_header Content-Type text/plain;
+        }
     }
 }

stacks/infra/main.tf

@@ -141,22 +141,33 @@ module "docker-registry-template" {
 
   # Setup registry config and start container
   provision_cmds = [
-    "mkdir -p /etc/docker-registry",
+    # Install and enable QEMU guest agent for remote management
-    format("echo %s | base64 -d > /etc/docker-registry/config.yml",
+    "apt-get install -y qemu-guest-agent",
+    "systemctl enable qemu-guest-agent",
+    "systemctl start qemu-guest-agent",
+    # Stop host nginx — we run nginx inside Docker instead
+    "systemctl stop nginx || true",
+    "systemctl disable nginx || true",
+    # Create directory structure
+    "mkdir -p /opt/registry/data/dockerhub /opt/registry/data/ghcr /opt/registry/data/quay /opt/registry/data/k8s /opt/registry/data/kyverno",
+    # Write Docker Compose file
+    format("echo %s | base64 -d > /opt/registry/docker-compose.yml",
+      base64encode(file("${path.root}/../../modules/docker-registry/docker-compose.yml"))
+    ),
+    # Write nginx config
+    format("echo %s | base64 -d > /opt/registry/nginx.conf",
+      base64encode(file("${path.root}/../../modules/docker-registry/nginx_registry.conf"))
+    ),
+    # Write Docker Hub registry config (with auth)
+    format("echo %s | base64 -d > /opt/registry/config-dockerhub.yml",
       base64encode(
         templatefile("../../modules/docker-registry/config.yaml", {
           password = var.dockerhub_registry_password
-          }
+        })
-        )
       )
     ),
-    "( crontab -l 2>/dev/null; echo '0 3 * * 0 /usr/bin/docker exec registry registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -",
+    # Write GHCR registry config
-    # Hourly restart cron removed - it wiped the in-memory blobdescriptor cache every hour,
+    format("echo %s | base64 -d > /opt/registry/config-ghcr.yml",
-    # causing low cache hit rates on the pull-through proxy. Docker containers use --restart always.
-    "docker run -p 5000:5000 -p 5001:5001 -d --restart always --name registry -v /etc/docker-registry/config.yml:/etc/docker/registry/config.yml registry:2",
-    # ghcr.io proxy
-    "mkdir -p /etc/docker-registry/ghcr",
-    format("echo %s | base64 -d > /etc/docker-registry/ghcr/config.yml",
       base64encode(
         templatefile("../../modules/docker-registry/config-proxy.yaml.tpl", {
           name       = "ghcr"
@@ -164,11 +175,8 @@ module "docker-registry-template" {
         })
       )
     ),
-    "docker run -p 5010:5000 -d --restart always --name registry-ghcr -v /etc/docker-registry/ghcr/config.yml:/etc/docker/registry/config.yml registry:2",
+    # Write Quay registry config
-    "( crontab -l 2>/dev/null; echo '5 3 * * 0 /usr/bin/docker exec registry-ghcr registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -",
+    format("echo %s | base64 -d > /opt/registry/config-quay.yml",
-    # quay.io proxy
-    "mkdir -p /etc/docker-registry/quay",
-    format("echo %s | base64 -d > /etc/docker-registry/quay/config.yml",
       base64encode(
         templatefile("../../modules/docker-registry/config-proxy.yaml.tpl", {
           name       = "quay"
@@ -176,11 +184,8 @@ module "docker-registry-template" {
         })
       )
     ),
-    "docker run -p 5020:5000 -d --restart always --name registry-quay -v /etc/docker-registry/quay/config.yml:/etc/docker/registry/config.yml registry:2",
+    # Write registry.k8s.io registry config
-    "( crontab -l 2>/dev/null; echo '10 3 * * 0 /usr/bin/docker exec registry-quay registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -",
+    format("echo %s | base64 -d > /opt/registry/config-k8s.yml",
-    # registry.k8s.io proxy
-    "mkdir -p /etc/docker-registry/k8s",
-    format("echo %s | base64 -d > /etc/docker-registry/k8s/config.yml",
       base64encode(
         templatefile("../../modules/docker-registry/config-proxy.yaml.tpl", {
           name       = "k8s"
@@ -188,11 +193,8 @@ module "docker-registry-template" {
         })
       )
     ),
-    "docker run -p 5030:5000 -d --restart always --name registry-k8s -v /etc/docker-registry/k8s/config.yml:/etc/docker/registry/config.yml registry:2",
+    # Write reg.kyverno.io registry config
-    "( crontab -l 2>/dev/null; echo '15 3 * * 0 /usr/bin/docker exec registry-k8s registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -",
+    format("echo %s | base64 -d > /opt/registry/config-kyverno.yml",
-    # reg.kyverno.io proxy
-    "mkdir -p /etc/docker-registry/kyverno",
-    format("echo %s | base64 -d > /etc/docker-registry/kyverno/config.yml",
       base64encode(
         templatefile("../../modules/docker-registry/config-proxy.yaml.tpl", {
           name       = "kyverno"
@@ -200,22 +202,43 @@ module "docker-registry-template" {
         })
       )
     ),
-    "docker run -p 5040:5000 -d --restart always --name registry-kyverno -v /etc/docker-registry/kyverno/config.yml:/etc/docker/registry/config.yml registry:2",
+    # Write tag cleanup script
-    "( crontab -l 2>/dev/null; echo '20 3 * * 0 /usr/bin/docker exec registry-kyverno registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -",
+    format("echo %s | base64 -d > /opt/registry/cleanup-tags.sh && chmod +x /opt/registry/cleanup-tags.sh",
-    # Setup the registry nginx config; We want clients to connect via the nginx to serialize requests for the same blobs
-    # Otherwise race conditions lead to corrupt blobs
-    "mkdir -p /var/cache/nginx/registry",
-    format("echo %s | base64 -d > /etc/nginx/conf.d/registry.conf",
-      base64encode(
-        templatefile("${path.root}/../../modules/docker-registry/nginx_registry.conf", {})
-      )
-    ),
-    "docker run -d --restart always --net host --name registry-ui -e NGINX_LISTEN_PORT=8080 -e NGINX_PROXY_PASS_URL=http://127.0.0.1:5000 -e DELETE_IMAGES=true -e SINGLE_REGISTRY=true -e SHOW_CONTENT_DIGEST=true -e SHOW_CATALOG_NB_TAGS=true -e CATALOG_ELEMENTS_LIMIT=1000 -e TAGLIST_PAGE_SIZE=100 -e REGISTRY_TITLE=viktorbarzin.me joxit/docker-registry-ui:latest",
-    # Deploy tag cleanup script (keep last 10 tags per image) and schedule daily at 2am before weekly GC
-    format("echo %s | base64 -d > /etc/docker-registry/cleanup-tags.sh && chmod +x /etc/docker-registry/cleanup-tags.sh",
       base64encode(file("${path.root}/../../modules/docker-registry/cleanup-tags.sh"))
     ),
-    "( crontab -l 2>/dev/null; echo '0 2 * * * python3 /etc/docker-registry/cleanup-tags.sh 10 >> /var/log/registry-cleanup.log 2>&1' ) | crontab -",
+    # Create systemd unit for docker compose
+    format("echo %s | base64 -d > /etc/systemd/system/docker-compose-registry.service",
+      base64encode(<<-UNIT
+[Unit]
+Description=Docker Compose Registry Stack
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+WorkingDirectory=/opt/registry
+ExecStart=/usr/bin/docker compose up -d --remove-orphans
+ExecStop=/usr/bin/docker compose down
+TimeoutStartSec=300
+
+[Install]
+WantedBy=multi-user.target
+UNIT
+      )
+    ),
+    # Enable and start the registry stack
+    "systemctl daemon-reload",
+    "systemctl enable docker-compose-registry.service",
+    "systemctl start docker-compose-registry.service",
+    # Cron: garbage collection (weekly, Sunday 3am, staggered per registry)
+    "( crontab -l 2>/dev/null; echo '0 3 * * 0 /usr/bin/docker exec registry-dockerhub registry garbage-collect -m /etc/docker/registry/config.yml >> /var/log/registry-gc.log 2>&1' ) | crontab -",
+    "( crontab -l 2>/dev/null; echo '5 3 * * 0 /usr/bin/docker exec registry-ghcr registry garbage-collect -m /etc/docker/registry/config.yml >> /var/log/registry-gc.log 2>&1' ) | crontab -",
+    "( crontab -l 2>/dev/null; echo '10 3 * * 0 /usr/bin/docker exec registry-quay registry garbage-collect -m /etc/docker/registry/config.yml >> /var/log/registry-gc.log 2>&1' ) | crontab -",
+    "( crontab -l 2>/dev/null; echo '15 3 * * 0 /usr/bin/docker exec registry-k8s registry garbage-collect -m /etc/docker/registry/config.yml >> /var/log/registry-gc.log 2>&1' ) | crontab -",
+    "( crontab -l 2>/dev/null; echo '20 3 * * 0 /usr/bin/docker exec registry-kyverno registry garbage-collect -m /etc/docker/registry/config.yml >> /var/log/registry-gc.log 2>&1' ) | crontab -",
+    # Cron: tag cleanup (daily 2am, keep last 10 tags per image)
+    "( crontab -l 2>/dev/null; echo '0 2 * * * python3 /opt/registry/cleanup-tags.sh 10 >> /var/log/registry-cleanup.log 2>&1' ) | crontab -",
   ]
 }
 
@@ -234,18 +257,18 @@ module "docker-registry-vm" {
   template_name  = "docker-registry-template"
   vm_name        = "docker-registry"
   cisnippet_name = "docker-registry.yaml"
+  agent          = 1
 
   vm_mac_address = "DE:AD:BE:EF:22:22" # mapped to 10.0.20.10 in dhcp
   bridge         = "vmbr1"
   vlan_tag       = "20"
   ipconfig0      = "ip=10.0.20.10/24,gw=10.0.20.1"
-  # ports:
+  # All ports go through nginx for request serialization (proxy_cache_lock):
-  # 5000 -> registry (docker.io proxy)
+  # 5000 -> nginx -> registry-dockerhub (docker.io proxy)
-  # 5001 -> metrics
+  # 5001 -> registry-dockerhub direct (Prometheus metrics)
-  # 5002 -> nginx proxy <-- use this to prevent races on the same blobs
+  # 5010 -> nginx -> registry-ghcr (ghcr.io proxy)
-  # 5010 -> registry-ghcr (ghcr.io proxy)
+  # 5020 -> nginx -> registry-quay (quay.io proxy)
-  # 5020 -> registry-quay (quay.io proxy)
+  # 5030 -> nginx -> registry-k8s (registry.k8s.io proxy)
-  # 5030 -> registry-k8s (registry.k8s.io proxy)
+  # 5040 -> nginx -> registry-kyverno (reg.kyverno.io proxy)
-  # 5040 -> registry-kyverno (reg.kyverno.io proxy)
   # 8080 -> registry-ui (joxit/docker-registry-ui)
 }