viktor/infra
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[ci skip] Rebuild docker-registry with nginx serialization on all ports

Browse source 
Replace individual `docker run` commands with Docker Compose stack managed
by systemd. Nginx now fronts all 5 registry ports (5000/5010/5020/5030/5040)
with proxy_cache_lock to serialize concurrent blob pulls and prevent
corrupt partial responses. Adds QEMU guest agent for remote management.
This commit is contained in:
Viktor Barzin 2026-02-22 21:45:53 +00:00
parent 9488af2397
commit 88960ba3a4
No known key found for this signature in database
GPG key ID: 0EB088298288D958
5 changed files with 449 additions and 116 deletions
Download patch file Download diff file Expand all files Collapse all files

6
modules/create-vm/main.tf
View file

@ -34,12 +34,16 @@ variable "ipconfig0" {
type = string
default = "ip=dhcp,ip6=dhcp"
}
variable "agent" {
type = number
default = 0
}
resource "proxmox_vm_qemu" "cloudinit-vm" {
vmid = var.vmid
name = var.vm_name
target_node = "pve"
agent = 0
agent = var.agent
memory = var.vm_mem_mb
boot = "order=scsi0" # has to be the same as the OS disk of the template
clone = var.template_name # The name of the template

51
modules/docker-registry/cleanup-tags.sh
View file

@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Keeps only the N most recent tags per image in a pull-through cache registry.
"""Keeps only the N most recent tags per image in pull-through cache registries.
Deletes old tag links directly from the filesystem since the API doesn't support
DELETE on proxy registries. Run garbage-collect after to reclaim blob storage."""
@ -10,39 +10,40 @@ import sys
sys.stdout.reconfigure(line_buffering=True)
KEEP = int(sys.argv[1]) if len(sys.argv) > 1 else 10
STORAGE = "/var/lib/docker/volumes/57b3f1c5fcc7f39c040e17072e10b4536245357d09340206683c04096d30b942/_data/docker/registry/v2/repositories"
BASE = sys.argv[2] if len(sys.argv) > 2 else "/opt/registry/data"
total_deleted = 0
for root, dirs, _ in os.walk(STORAGE):
# Look for _manifests/tags directories
if not root.endswith("_manifests/tags"):
for registry_name in sorted(os.listdir(BASE)):
storage = os.path.join(BASE, registry_name, "docker/registry/v2/repositories")
if not os.path.isdir(storage):
continue
repo = root.replace(STORAGE + "/", "").replace("/_manifests/tags", "")
for root, dirs, _ in os.walk(storage):
if not root.endswith("_manifests/tags"):
continue
# Get tags with modification times
tag_times = []
for tag in os.listdir(root):
tag_path = os.path.join(root, tag)
if os.path.isdir(tag_path):
mtime = os.path.getmtime(tag_path)
tag_times.append((mtime, tag, tag_path))
repo = root.replace(storage + "/", "").replace("/_manifests/tags", "")
if len(tag_times) <= KEEP:
continue
tag_times = []
for tag in os.listdir(root):
tag_path = os.path.join(root, tag)
if os.path.isdir(tag_path):
mtime = os.path.getmtime(tag_path)
tag_times.append((mtime, tag, tag_path))
# Sort by mtime descending (newest first)
tag_times.sort(reverse=True)
to_delete = tag_times[KEEP:]
if len(tag_times) <= KEEP:
continue
print(f"[{repo}] {len(tag_times)} tags -> keeping {KEEP}, deleting {len(to_delete)}")
tag_times.sort(reverse=True)
to_delete = tag_times[KEEP:]
for _, tag, tag_path in to_delete:
shutil.rmtree(tag_path)
total_deleted += 1
print(f"[{registry_name}/{repo}] {len(tag_times)} tags -> keeping {KEEP}, deleting {len(to_delete)}")
print(f" done")
for _, tag, tag_path in to_delete:
shutil.rmtree(tag_path)
total_deleted += 1
print(f"\nDeleted {total_deleted} tags. Restart registry and run garbage-collect to reclaim space.")
print(f" done")
print(f"\nDeleted {total_deleted} tags. Run garbage-collect to reclaim space.")

143
modules/docker-registry/docker-compose.yml Normal file
View file

@ -0,0 +1,143 @@
networks:
registry:
driver: bridge
services:
registry-dockerhub:
image: registry:2
container_name: registry-dockerhub
restart: always
volumes:
- /opt/registry/data/dockerhub:/var/lib/registry
- /opt/registry/config-dockerhub.yml:/etc/docker/registry/config.yml:ro
networks:
- registry
ports:
- "5001:5001"
healthcheck:
test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
interval: 30s
timeout: 10s
retries: 3
start_period: 10s
registry-ghcr:
image: registry:2
container_name: registry-ghcr
restart: always
volumes:
- /opt/registry/data/ghcr:/var/lib/registry
- /opt/registry/config-ghcr.yml:/etc/docker/registry/config.yml:ro
networks:
- registry
healthcheck:
test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
interval: 30s
timeout: 10s
retries: 3
start_period: 10s
registry-quay:
image: registry:2
container_name: registry-quay
restart: always
volumes:
- /opt/registry/data/quay:/var/lib/registry
- /opt/registry/config-quay.yml:/etc/docker/registry/config.yml:ro
networks:
- registry
healthcheck:
test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
interval: 30s
timeout: 10s
retries: 3
start_period: 10s
registry-k8s:
image: registry:2
container_name: registry-k8s
restart: always
volumes:
- /opt/registry/data/k8s:/var/lib/registry
- /opt/registry/config-k8s.yml:/etc/docker/registry/config.yml:ro
networks:
- registry
healthcheck:
test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
interval: 30s
timeout: 10s
retries: 3
start_period: 10s
registry-kyverno:
image: registry:2
container_name: registry-kyverno
restart: always
volumes:
- /opt/registry/data/kyverno:/var/lib/registry
- /opt/registry/config-kyverno.yml:/etc/docker/registry/config.yml:ro
networks:
- registry
healthcheck:
test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
interval: 30s
timeout: 10s
retries: 3
start_period: 10s
nginx:
image: nginx:alpine
container_name: registry-nginx
restart: always
ports:
- "5000:5000"
- "5010:5010"
- "5020:5020"
- "5030:5030"
- "5040:5040"
volumes:
- /opt/registry/nginx.conf:/etc/nginx/nginx.conf:ro
- nginx-cache:/var/cache/nginx
networks:
- registry
depends_on:
registry-dockerhub:
condition: service_healthy
registry-ghcr:
condition: service_healthy
registry-quay:
condition: service_healthy
registry-k8s:
condition: service_healthy
registry-kyverno:
condition: service_healthy
healthcheck:
test: ["CMD", "sh", "-c", "wget -qO- http://localhost:5000/v2/ >/dev/null 2>&1"]
interval: 30s
timeout: 10s
retries: 3
start_period: 15s
registry-ui:
image: joxit/docker-registry-ui:latest
container_name: registry-ui
restart: always
ports:
- "8080:80"
environment:
- NGINX_PROXY_PASS_URL=http://registry-dockerhub:5000
- DELETE_IMAGES=true
- SINGLE_REGISTRY=true
- SHOW_CONTENT_DIGEST=true
- SHOW_CATALOG_NB_TAGS=true
- CATALOG_ELEMENTS_LIMIT=1000
- TAGLIST_PAGE_SIZE=100
- REGISTRY_TITLE=viktorbarzin.me
networks:
- registry
depends_on:
registry-dockerhub:
condition: service_healthy
volumes:
nginx-cache:

246
modules/docker-registry/nginx_registry.conf
View file

@ -1,58 +1,220 @@
proxy_cache_path /var/cache/nginx/registry
levels=1:2
keys_zone=registry:500m
max_size=50g
inactive=24h
use_temp_path=off;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /tmp/nginx.pid;
upstream docker_registry {
server 127.0.0.1:5000;
keepalive 32;
events {
worker_connections 1024;
}
server {
listen 5002;
server_name _;
http {
proxy_cache_path /var/cache/nginx/registry
levels=1:2
keys_zone=registry:500m
max_size=50g
inactive=24h
use_temp_path=off;
# Access log
access_log /var/log/nginx/registry.access.log combined;
log_format registry '$remote_addr [$time_local] "$request" '
'$status $body_bytes_sent '
'upstream=$upstream_addr time=$upstream_response_time '
'cache=$upstream_cache_status';
# Error log
error_log /var/log/nginx/registry.error.log warn;
access_log /var/log/nginx/access.log registry;
# Required for large blobs
client_max_body_size 0;
# --- Upstreams ---
# Disable buffering to clients, keep it between nginx<->registry
proxy_request_buffering off;
proxy_buffering on;
upstream dockerhub {
server registry-dockerhub:5000;
keepalive 32;
}
location /v2/ {
proxy_pass http://docker_registry;
upstream ghcr {
server registry-ghcr:5000;
keepalive 32;
}
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Connection "";
upstream quay {
server registry-quay:5000;
keepalive 32;
}
# --- CRITICAL PART ---
proxy_cache registry;
proxy_cache_lock on;
proxy_cache_lock_timeout 15m;
proxy_cache_lock_age 15m;
proxy_cache_use_stale updating;
upstream k8s {
server registry-k8s:5000;
keepalive 32;
}
# Cache only successful pulls
proxy_cache_valid 200 206 24h;
upstream kyverno {
server registry-kyverno:5000;
keepalive 32;
}
# HEAD requests must not poison cache
proxy_cache_methods GET;
# --- Docker Hub (port 5000) ---
# Do not cache pushes
proxy_no_cache $http_authorization;
proxy_cache_bypass $http_authorization;
server {
listen 5000;
server_name _;
# Prevent partial responses
proxy_read_timeout 900;
proxy_send_timeout 900;
client_max_body_size 0;
proxy_request_buffering off;
proxy_buffering on;
location /v2/ {
proxy_pass http://dockerhub;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Connection "";
proxy_cache registry;
proxy_cache_lock on;
proxy_cache_lock_timeout 15m;
proxy_cache_lock_age 15m;
proxy_cache_use_stale updating;
proxy_cache_valid 200 206 24h;
proxy_cache_methods GET;
proxy_read_timeout 900;
proxy_send_timeout 900;
}
location / {
return 200 'ok';
add_header Content-Type text/plain;
}
}
# --- GHCR (port 5010) ---
server {
listen 5010;
server_name _;
client_max_body_size 0;
proxy_request_buffering off;
proxy_buffering on;
location /v2/ {
proxy_pass http://ghcr;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Connection "";
proxy_cache registry;
proxy_cache_lock on;
proxy_cache_lock_timeout 15m;
proxy_cache_lock_age 15m;
proxy_cache_use_stale updating;
proxy_cache_valid 200 206 24h;
proxy_cache_methods GET;
proxy_read_timeout 900;
proxy_send_timeout 900;
}
location / {
return 200 'ok';
add_header Content-Type text/plain;
}
}
# --- Quay (port 5020) ---
server {
listen 5020;
server_name _;
client_max_body_size 0;
proxy_request_buffering off;
proxy_buffering on;
location /v2/ {
proxy_pass http://quay;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Connection "";
proxy_cache registry;
proxy_cache_lock on;
proxy_cache_lock_timeout 15m;
proxy_cache_lock_age 15m;
proxy_cache_use_stale updating;
proxy_cache_valid 200 206 24h;
proxy_cache_methods GET;
proxy_read_timeout 900;
proxy_send_timeout 900;
}
location / {
return 200 'ok';
add_header Content-Type text/plain;
}
}
# --- registry.k8s.io (port 5030) ---
server {
listen 5030;
server_name _;
client_max_body_size 0;
proxy_request_buffering off;
proxy_buffering on;
location /v2/ {
proxy_pass http://k8s;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Connection "";
proxy_cache registry;
proxy_cache_lock on;
proxy_cache_lock_timeout 15m;
proxy_cache_lock_age 15m;
proxy_cache_use_stale updating;
proxy_cache_valid 200 206 24h;
proxy_cache_methods GET;
proxy_read_timeout 900;
proxy_send_timeout 900;
}
location / {
return 200 'ok';
add_header Content-Type text/plain;
}
}
# --- reg.kyverno.io (port 5040) ---
server {
listen 5040;
server_name _;
client_max_body_size 0;
proxy_request_buffering off;
proxy_buffering on;
location /v2/ {
proxy_pass http://kyverno;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Connection "";
proxy_cache registry;
proxy_cache_lock on;
proxy_cache_lock_timeout 15m;
proxy_cache_lock_age 15m;
proxy_cache_use_stale updating;
proxy_cache_valid 200 206 24h;
proxy_cache_methods GET;
proxy_read_timeout 900;
proxy_send_timeout 900;
}
location / {
return 200 'ok';
add_header Content-Type text/plain;
}
}
}

119
stacks/infra/main.tf
View file

@ -141,22 +141,33 @@ module "docker-registry-template" {
# Setup registry config and start container
provision_cmds = [
"mkdir -p /etc/docker-registry",
format("echo %s | base64 -d > /etc/docker-registry/config.yml",
# Install and enable QEMU guest agent for remote management
"apt-get install -y qemu-guest-agent",
"systemctl enable qemu-guest-agent",
"systemctl start qemu-guest-agent",
# Stop host nginx — we run nginx inside Docker instead
"systemctl stop nginx || true",
"systemctl disable nginx || true",
# Create directory structure
"mkdir -p /opt/registry/data/dockerhub /opt/registry/data/ghcr /opt/registry/data/quay /opt/registry/data/k8s /opt/registry/data/kyverno",
# Write Docker Compose file
format("echo %s | base64 -d > /opt/registry/docker-compose.yml",
base64encode(file("${path.root}/../../modules/docker-registry/docker-compose.yml"))
),
# Write nginx config
format("echo %s | base64 -d > /opt/registry/nginx.conf",
base64encode(file("${path.root}/../../modules/docker-registry/nginx_registry.conf"))
),
# Write Docker Hub registry config (with auth)
format("echo %s | base64 -d > /opt/registry/config-dockerhub.yml",
base64encode(
templatefile("../../modules/docker-registry/config.yaml", {
password = var.dockerhub_registry_password
}
)
})
)
),
"( crontab -l 2>/dev/null; echo '0 3 * * 0 /usr/bin/docker exec registry registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -",
# Hourly restart cron removed - it wiped the in-memory blobdescriptor cache every hour,
# causing low cache hit rates on the pull-through proxy. Docker containers use --restart always.
"docker run -p 5000:5000 -p 5001:5001 -d --restart always --name registry -v /etc/docker-registry/config.yml:/etc/docker/registry/config.yml registry:2",
# ghcr.io proxy
"mkdir -p /etc/docker-registry/ghcr",
format("echo %s | base64 -d > /etc/docker-registry/ghcr/config.yml",
# Write GHCR registry config
format("echo %s | base64 -d > /opt/registry/config-ghcr.yml",
base64encode(
templatefile("../../modules/docker-registry/config-proxy.yaml.tpl", {
name = "ghcr"
@ -164,11 +175,8 @@ module "docker-registry-template" {
})
)
),
"docker run -p 5010:5000 -d --restart always --name registry-ghcr -v /etc/docker-registry/ghcr/config.yml:/etc/docker/registry/config.yml registry:2",
"( crontab -l 2>/dev/null; echo '5 3 * * 0 /usr/bin/docker exec registry-ghcr registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -",
# quay.io proxy
"mkdir -p /etc/docker-registry/quay",
format("echo %s | base64 -d > /etc/docker-registry/quay/config.yml",
# Write Quay registry config
format("echo %s | base64 -d > /opt/registry/config-quay.yml",
base64encode(
templatefile("../../modules/docker-registry/config-proxy.yaml.tpl", {
name = "quay"
@ -176,11 +184,8 @@ module "docker-registry-template" {
})
)
),
"docker run -p 5020:5000 -d --restart always --name registry-quay -v /etc/docker-registry/quay/config.yml:/etc/docker/registry/config.yml registry:2",
"( crontab -l 2>/dev/null; echo '10 3 * * 0 /usr/bin/docker exec registry-quay registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -",
# registry.k8s.io proxy
"mkdir -p /etc/docker-registry/k8s",
format("echo %s | base64 -d > /etc/docker-registry/k8s/config.yml",
# Write registry.k8s.io registry config
format("echo %s | base64 -d > /opt/registry/config-k8s.yml",
base64encode(
templatefile("../../modules/docker-registry/config-proxy.yaml.tpl", {
name = "k8s"
@ -188,11 +193,8 @@ module "docker-registry-template" {
})
)
),
"docker run -p 5030:5000 -d --restart always --name registry-k8s -v /etc/docker-registry/k8s/config.yml:/etc/docker/registry/config.yml registry:2",
"( crontab -l 2>/dev/null; echo '15 3 * * 0 /usr/bin/docker exec registry-k8s registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -",
# reg.kyverno.io proxy
"mkdir -p /etc/docker-registry/kyverno",
format("echo %s | base64 -d > /etc/docker-registry/kyverno/config.yml",
# Write reg.kyverno.io registry config
format("echo %s | base64 -d > /opt/registry/config-kyverno.yml",
base64encode(
templatefile("../../modules/docker-registry/config-proxy.yaml.tpl", {
name = "kyverno"
@ -200,22 +202,43 @@ module "docker-registry-template" {
})
)
),
"docker run -p 5040:5000 -d --restart always --name registry-kyverno -v /etc/docker-registry/kyverno/config.yml:/etc/docker/registry/config.yml registry:2",
"( crontab -l 2>/dev/null; echo '20 3 * * 0 /usr/bin/docker exec registry-kyverno registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -",
# Setup the registry nginx config; We want clients to connect via the nginx to serialize requests for the same blobs
# Otherwise race conditions lead to corrupt blobs
"mkdir -p /var/cache/nginx/registry",
format("echo %s | base64 -d > /etc/nginx/conf.d/registry.conf",
base64encode(
templatefile("${path.root}/../../modules/docker-registry/nginx_registry.conf", {})
)
),
"docker run -d --restart always --net host --name registry-ui -e NGINX_LISTEN_PORT=8080 -e NGINX_PROXY_PASS_URL=http://127.0.0.1:5000 -e DELETE_IMAGES=true -e SINGLE_REGISTRY=true -e SHOW_CONTENT_DIGEST=true -e SHOW_CATALOG_NB_TAGS=true -e CATALOG_ELEMENTS_LIMIT=1000 -e TAGLIST_PAGE_SIZE=100 -e REGISTRY_TITLE=viktorbarzin.me joxit/docker-registry-ui:latest",
# Deploy tag cleanup script (keep last 10 tags per image) and schedule daily at 2am before weekly GC
format("echo %s | base64 -d > /etc/docker-registry/cleanup-tags.sh && chmod +x /etc/docker-registry/cleanup-tags.sh",
# Write tag cleanup script
format("echo %s | base64 -d > /opt/registry/cleanup-tags.sh && chmod +x /opt/registry/cleanup-tags.sh",
base64encode(file("${path.root}/../../modules/docker-registry/cleanup-tags.sh"))
),
"( crontab -l 2>/dev/null; echo '0 2 * * * python3 /etc/docker-registry/cleanup-tags.sh 10 >> /var/log/registry-cleanup.log 2>&1' ) | crontab -",
# Create systemd unit for docker compose
format("echo %s | base64 -d > /etc/systemd/system/docker-compose-registry.service",
base64encode(<<-UNIT
[Unit]
Description=Docker Compose Registry Stack
After=docker.service
Requires=docker.service
[Service]
Type=oneshot
RemainAfterExit=yes
WorkingDirectory=/opt/registry
ExecStart=/usr/bin/docker compose up -d --remove-orphans
ExecStop=/usr/bin/docker compose down
TimeoutStartSec=300
[Install]
WantedBy=multi-user.target
UNIT
)
),
# Enable and start the registry stack
"systemctl daemon-reload",
"systemctl enable docker-compose-registry.service",
"systemctl start docker-compose-registry.service",
# Cron: garbage collection (weekly, Sunday 3am, staggered per registry)
"( crontab -l 2>/dev/null; echo '0 3 * * 0 /usr/bin/docker exec registry-dockerhub registry garbage-collect -m /etc/docker/registry/config.yml >> /var/log/registry-gc.log 2>&1' ) | crontab -",
"( crontab -l 2>/dev/null; echo '5 3 * * 0 /usr/bin/docker exec registry-ghcr registry garbage-collect -m /etc/docker/registry/config.yml >> /var/log/registry-gc.log 2>&1' ) | crontab -",
"( crontab -l 2>/dev/null; echo '10 3 * * 0 /usr/bin/docker exec registry-quay registry garbage-collect -m /etc/docker/registry/config.yml >> /var/log/registry-gc.log 2>&1' ) | crontab -",
"( crontab -l 2>/dev/null; echo '15 3 * * 0 /usr/bin/docker exec registry-k8s registry garbage-collect -m /etc/docker/registry/config.yml >> /var/log/registry-gc.log 2>&1' ) | crontab -",
"( crontab -l 2>/dev/null; echo '20 3 * * 0 /usr/bin/docker exec registry-kyverno registry garbage-collect -m /etc/docker/registry/config.yml >> /var/log/registry-gc.log 2>&1' ) | crontab -",
# Cron: tag cleanup (daily 2am, keep last 10 tags per image)
"( crontab -l 2>/dev/null; echo '0 2 * * * python3 /opt/registry/cleanup-tags.sh 10 >> /var/log/registry-cleanup.log 2>&1' ) | crontab -",
]
}
@ -234,18 +257,18 @@ module "docker-registry-vm" {
template_name = "docker-registry-template"
vm_name = "docker-registry"
cisnippet_name = "docker-registry.yaml"
agent = 1
vm_mac_address = "DE:AD:BE:EF:22:22" # mapped to 10.0.20.10 in dhcp
bridge = "vmbr1"
vlan_tag = "20"
ipconfig0 = "ip=10.0.20.10/24,gw=10.0.20.1"
# ports:
# 5000 -> registry (docker.io proxy)
# 5001 -> metrics
# 5002 -> nginx proxy <-- use this to prevent races on the same blobs
# 5010 -> registry-ghcr (ghcr.io proxy)
# 5020 -> registry-quay (quay.io proxy)
# 5030 -> registry-k8s (registry.k8s.io proxy)
# 5040 -> registry-kyverno (reg.kyverno.io proxy)
# All ports go through nginx for request serialization (proxy_cache_lock):
# 5000 -> nginx -> registry-dockerhub (docker.io proxy)
# 5001 -> registry-dockerhub direct (Prometheus metrics)
# 5010 -> nginx -> registry-ghcr (ghcr.io proxy)
# 5020 -> nginx -> registry-quay (quay.io proxy)
# 5030 -> nginx -> registry-k8s (registry.k8s.io proxy)
# 5040 -> nginx -> registry-kyverno (reg.kyverno.io proxy)
# 8080 -> registry-ui (joxit/docker-registry-ui)
}