[ci skip] Rebuild docker-registry with nginx serialization on all ports

Replace individual `docker run` commands with Docker Compose stack managed by systemd. Nginx now fronts all 5 registry ports (5000/5010/5020/5030/5040) with proxy_cache_lock to serialize concurrent blob pulls and prevent corrupt partial responses. Adds QEMU guest agent for remote management.
2026-02-22 21:45:53 +00:00 · 2026-02-22 21:45:53 +00:00 · 88960ba3a4
commit 88960ba3a4
parent 9488af2397
5 changed files with 449 additions and 116 deletions
--- a/modules/create-vm/main.tf
+++ b/modules/create-vm/main.tf
@ -34,12 +34,16 @@ variable "ipconfig0" {
  type    = string
  default = "ip=dhcp,ip6=dhcp"
 }
+variable "agent" {
+  type    = number
+  default = 0
+}

 resource "proxmox_vm_qemu" "cloudinit-vm" {
  vmid             = var.vmid
  name             = var.vm_name
  target_node      = "pve"
-  agent            = 0
+  agent            = var.agent
  memory           = var.vm_mem_mb
  boot             = "order=scsi0"     # has to be the same as the OS disk of the template
  clone            = var.template_name # The name of the template
--- a/modules/docker-registry/cleanup-tags.sh
+++ b/modules/docker-registry/cleanup-tags.sh
@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Keeps only the N most recent tags per image in a pull-through cache registry.
+"""Keeps only the N most recent tags per image in pull-through cache registries.
 Deletes old tag links directly from the filesystem since the API doesn't support
 DELETE on proxy registries. Run garbage-collect after to reclaim blob storage."""

@ -10,39 +10,40 @@ import sys
 sys.stdout.reconfigure(line_buffering=True)

 KEEP = int(sys.argv[1]) if len(sys.argv) > 1 else 10
-
-STORAGE = "/var/lib/docker/volumes/57b3f1c5fcc7f39c040e17072e10b4536245357d09340206683c04096d30b942/_data/docker/registry/v2/repositories"
+BASE = sys.argv[2] if len(sys.argv) > 2 else "/opt/registry/data"

 total_deleted = 0

-for root, dirs, _ in os.walk(STORAGE):
-    # Look for _manifests/tags directories
-    if not root.endswith("_manifests/tags"):
+for registry_name in sorted(os.listdir(BASE)):
+    storage = os.path.join(BASE, registry_name, "docker/registry/v2/repositories")
+    if not os.path.isdir(storage):
        continue

-    repo = root.replace(STORAGE + "/", "").replace("/_manifests/tags", "")
+    for root, dirs, _ in os.walk(storage):
+        if not root.endswith("_manifests/tags"):
+            continue

-    # Get tags with modification times
-    tag_times = []
-    for tag in os.listdir(root):
-        tag_path = os.path.join(root, tag)
-        if os.path.isdir(tag_path):
-            mtime = os.path.getmtime(tag_path)
-            tag_times.append((mtime, tag, tag_path))
+        repo = root.replace(storage + "/", "").replace("/_manifests/tags", "")

-    if len(tag_times) <= KEEP:
-        continue
+        tag_times = []
+        for tag in os.listdir(root):
+            tag_path = os.path.join(root, tag)
+            if os.path.isdir(tag_path):
+                mtime = os.path.getmtime(tag_path)
+                tag_times.append((mtime, tag, tag_path))

-    # Sort by mtime descending (newest first)
-    tag_times.sort(reverse=True)
-    to_delete = tag_times[KEEP:]
+        if len(tag_times) <= KEEP:
+            continue

-    print(f"[{repo}] {len(tag_times)} tags -> keeping {KEEP}, deleting {len(to_delete)}")
+        tag_times.sort(reverse=True)
+        to_delete = tag_times[KEEP:]

-    for _, tag, tag_path in to_delete:
-        shutil.rmtree(tag_path)
-        total_deleted += 1
+        print(f"[{registry_name}/{repo}] {len(tag_times)} tags -> keeping {KEEP}, deleting {len(to_delete)}")

-    print(f"  done")
+        for _, tag, tag_path in to_delete:
+            shutil.rmtree(tag_path)
+            total_deleted += 1

-print(f"\nDeleted {total_deleted} tags. Restart registry and run garbage-collect to reclaim space.")
+        print(f"  done")
+
+print(f"\nDeleted {total_deleted} tags. Run garbage-collect to reclaim space.")
--- a/modules/docker-registry/docker-compose.yml
+++ b/modules/docker-registry/docker-compose.yml
@ -0,0 +1,143 @@
+networks:
+  registry:
+    driver: bridge
+
+services:
+  registry-dockerhub:
+    image: registry:2
+    container_name: registry-dockerhub
+    restart: always
+    volumes:
+      - /opt/registry/data/dockerhub:/var/lib/registry
+      - /opt/registry/config-dockerhub.yml:/etc/docker/registry/config.yml:ro
+    networks:
+      - registry
+    ports:
+      - "5001:5001"
+    healthcheck:
+      test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+
+  registry-ghcr:
+    image: registry:2
+    container_name: registry-ghcr
+    restart: always
+    volumes:
+      - /opt/registry/data/ghcr:/var/lib/registry
+      - /opt/registry/config-ghcr.yml:/etc/docker/registry/config.yml:ro
+    networks:
+      - registry
+    healthcheck:
+      test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+
+  registry-quay:
+    image: registry:2
+    container_name: registry-quay
+    restart: always
+    volumes:
+      - /opt/registry/data/quay:/var/lib/registry
+      - /opt/registry/config-quay.yml:/etc/docker/registry/config.yml:ro
+    networks:
+      - registry
+    healthcheck:
+      test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+
+  registry-k8s:
+    image: registry:2
+    container_name: registry-k8s
+    restart: always
+    volumes:
+      - /opt/registry/data/k8s:/var/lib/registry
+      - /opt/registry/config-k8s.yml:/etc/docker/registry/config.yml:ro
+    networks:
+      - registry
+    healthcheck:
+      test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+
+  registry-kyverno:
+    image: registry:2
+    container_name: registry-kyverno
+    restart: always
+    volumes:
+      - /opt/registry/data/kyverno:/var/lib/registry
+      - /opt/registry/config-kyverno.yml:/etc/docker/registry/config.yml:ro
+    networks:
+      - registry
+    healthcheck:
+      test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+
+  nginx:
+    image: nginx:alpine
+    container_name: registry-nginx
+    restart: always
+    ports:
+      - "5000:5000"
+      - "5010:5010"
+      - "5020:5020"
+      - "5030:5030"
+      - "5040:5040"
+    volumes:
+      - /opt/registry/nginx.conf:/etc/nginx/nginx.conf:ro
+      - nginx-cache:/var/cache/nginx
+    networks:
+      - registry
+    depends_on:
+      registry-dockerhub:
+        condition: service_healthy
+      registry-ghcr:
+        condition: service_healthy
+      registry-quay:
+        condition: service_healthy
+      registry-k8s:
+        condition: service_healthy
+      registry-kyverno:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 15s
+
+  registry-ui:
+    image: joxit/docker-registry-ui:latest
+    container_name: registry-ui
+    restart: always
+    ports:
+      - "8080:80"
+    environment:
+      - NGINX_PROXY_PASS_URL=http://registry-dockerhub:5000
+      - DELETE_IMAGES=true
+      - SINGLE_REGISTRY=true
+      - SHOW_CONTENT_DIGEST=true
+      - SHOW_CATALOG_NB_TAGS=true
+      - CATALOG_ELEMENTS_LIMIT=1000
+      - TAGLIST_PAGE_SIZE=100
+      - REGISTRY_TITLE=viktorbarzin.me
+    networks:
+      - registry
+    depends_on:
+      registry-dockerhub:
+        condition: service_healthy
+
+volumes:
+  nginx-cache:
--- a/modules/docker-registry/nginx_registry.conf
+++ b/modules/docker-registry/nginx_registry.conf
@ -1,58 +1,220 @@
-proxy_cache_path /var/cache/nginx/registry
-    levels=1:2
-    keys_zone=registry:500m
-    max_size=50g
-    inactive=24h
-    use_temp_path=off;
+worker_processes auto;
+error_log /var/log/nginx/error.log warn;
+pid /tmp/nginx.pid;

-upstream docker_registry {
-    server 127.0.0.1:5000;
-    keepalive 32;
+events {
+    worker_connections 1024;
 }

-server {
-    listen 5002;
-    server_name _;
+http {
+    proxy_cache_path /var/cache/nginx/registry
+        levels=1:2
+        keys_zone=registry:500m
+        max_size=50g
+        inactive=24h
+        use_temp_path=off;

-    # Access log
-    access_log /var/log/nginx/registry.access.log combined;
+    log_format registry '$remote_addr [$time_local] "$request" '
+                        '$status $body_bytes_sent '
+                        'upstream=$upstream_addr time=$upstream_response_time '
+                        'cache=$upstream_cache_status';

-    # Error log
-    error_log /var/log/nginx/registry.error.log warn;
+    access_log /var/log/nginx/access.log registry;

-    # Required for large blobs
-    client_max_body_size 0;
+    # --- Upstreams ---

-    # Disable buffering to clients, keep it between nginx<->registry
-    proxy_request_buffering off;
-    proxy_buffering on;
+    upstream dockerhub {
+        server registry-dockerhub:5000;
+        keepalive 32;
+    }

-    location /v2/ {
-        proxy_pass http://docker_registry;
+    upstream ghcr {
+        server registry-ghcr:5000;
+        keepalive 32;
+    }

-        proxy_http_version 1.1;
-        proxy_set_header Host $host;
-        proxy_set_header Connection "";
+    upstream quay {
+        server registry-quay:5000;
+        keepalive 32;
+    }

-        # --- CRITICAL PART ---
-        proxy_cache registry;
-        proxy_cache_lock on;
-        proxy_cache_lock_timeout 15m;
-        proxy_cache_lock_age 15m;
-        proxy_cache_use_stale updating;
+    upstream k8s {
+        server registry-k8s:5000;
+        keepalive 32;
+    }

-        # Cache only successful pulls
-        proxy_cache_valid 200 206 24h;
+    upstream kyverno {
+        server registry-kyverno:5000;
+        keepalive 32;
+    }

-        # HEAD requests must not poison cache
-        proxy_cache_methods GET;
+    # --- Docker Hub (port 5000) ---

-        # Do not cache pushes
-        proxy_no_cache $http_authorization;
-        proxy_cache_bypass $http_authorization;
+    server {
+        listen 5000;
+        server_name _;

-        # Prevent partial responses
-        proxy_read_timeout 900;
-        proxy_send_timeout 900;
+        client_max_body_size 0;
+        proxy_request_buffering off;
+        proxy_buffering on;
+
+        location /v2/ {
+            proxy_pass http://dockerhub;
+            proxy_http_version 1.1;
+            proxy_set_header Host $host;
+            proxy_set_header Connection "";
+
+            proxy_cache registry;
+            proxy_cache_lock on;
+            proxy_cache_lock_timeout 15m;
+            proxy_cache_lock_age 15m;
+            proxy_cache_use_stale updating;
+            proxy_cache_valid 200 206 24h;
+            proxy_cache_methods GET;
+
+            proxy_read_timeout 900;
+            proxy_send_timeout 900;
+        }
+
+        location / {
+            return 200 'ok';
+            add_header Content-Type text/plain;
+        }
+    }
+
+    # --- GHCR (port 5010) ---
+
+    server {
+        listen 5010;
+        server_name _;
+
+        client_max_body_size 0;
+        proxy_request_buffering off;
+        proxy_buffering on;
+
+        location /v2/ {
+            proxy_pass http://ghcr;
+            proxy_http_version 1.1;
+            proxy_set_header Host $host;
+            proxy_set_header Connection "";
+
+            proxy_cache registry;
+            proxy_cache_lock on;
+            proxy_cache_lock_timeout 15m;
+            proxy_cache_lock_age 15m;
+            proxy_cache_use_stale updating;
+            proxy_cache_valid 200 206 24h;
+            proxy_cache_methods GET;
+
+            proxy_read_timeout 900;
+            proxy_send_timeout 900;
+        }
+
+        location / {
+            return 200 'ok';
+            add_header Content-Type text/plain;
+        }
+    }
+
+    # --- Quay (port 5020) ---
+
+    server {
+        listen 5020;
+        server_name _;
+
+        client_max_body_size 0;
+        proxy_request_buffering off;
+        proxy_buffering on;
+
+        location /v2/ {
+            proxy_pass http://quay;
+            proxy_http_version 1.1;
+            proxy_set_header Host $host;
+            proxy_set_header Connection "";
+
+            proxy_cache registry;
+            proxy_cache_lock on;
+            proxy_cache_lock_timeout 15m;
+            proxy_cache_lock_age 15m;
+            proxy_cache_use_stale updating;
+            proxy_cache_valid 200 206 24h;
+            proxy_cache_methods GET;
+
+            proxy_read_timeout 900;
+            proxy_send_timeout 900;
+        }
+
+        location / {
+            return 200 'ok';
+            add_header Content-Type text/plain;
+        }
+    }
+
+    # --- registry.k8s.io (port 5030) ---
+
+    server {
+        listen 5030;
+        server_name _;
+
+        client_max_body_size 0;
+        proxy_request_buffering off;
+        proxy_buffering on;
+
+        location /v2/ {
+            proxy_pass http://k8s;
+            proxy_http_version 1.1;
+            proxy_set_header Host $host;
+            proxy_set_header Connection "";
+
+            proxy_cache registry;
+            proxy_cache_lock on;
+            proxy_cache_lock_timeout 15m;
+            proxy_cache_lock_age 15m;
+            proxy_cache_use_stale updating;
+            proxy_cache_valid 200 206 24h;
+            proxy_cache_methods GET;
+
+            proxy_read_timeout 900;
+            proxy_send_timeout 900;
+        }
+
+        location / {
+            return 200 'ok';
+            add_header Content-Type text/plain;
+        }
+    }
+
+    # --- reg.kyverno.io (port 5040) ---
+
+    server {
+        listen 5040;
+        server_name _;
+
+        client_max_body_size 0;
+        proxy_request_buffering off;
+        proxy_buffering on;
+
+        location /v2/ {
+            proxy_pass http://kyverno;
+            proxy_http_version 1.1;
+            proxy_set_header Host $host;
+            proxy_set_header Connection "";
+
+            proxy_cache registry;
+            proxy_cache_lock on;
+            proxy_cache_lock_timeout 15m;
+            proxy_cache_lock_age 15m;
+            proxy_cache_use_stale updating;
+            proxy_cache_valid 200 206 24h;
+            proxy_cache_methods GET;
+
+            proxy_read_timeout 900;
+            proxy_send_timeout 900;
+        }
+
+        location / {
+            return 200 'ok';
+            add_header Content-Type text/plain;
+        }
    }
 }