add nginx reverse proxy to serialize registyr requests for the same path to avoid race conditions [ci skip]

main.tf

@@ -227,7 +227,15 @@ module "docker-registry-template" {
     ),
     "( crontab -l 2>/dev/null; echo '0 3 * * 0 /usr/bin/docker exec registry registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -",
     "( crontab -l 2>/dev/null; echo '0 * * * * /usr/bin/docker restart registry' ) | crontab -",
-    "docker run -p 5000:5000 -p 5001:5001 -d --restart always --name registry -v /etc/docker-registry/config.yml:/etc/docker/registry/config.yml registry:2"
+    "docker run -p 5000:5000 -p 5001:5001 -d --restart always --name registry -v /etc/docker-registry/config.yml:/etc/docker/registry/config.yml registry:2",
+    # Setup the registry nginx config; We want clients to connect via the nginx to serialize requests for the same blobs
+    # Otherwise race conditions lead to corrupt blobs
+    "mkdir -p /var/cache/nginx/registry",
+    format("echo %s | base64 -d > /etc/nginx/conf.d/registry.conf",
+      base64encode(
+        templatefile("${path.root}/modules/docker-registry/nginx_registry.conf", {})
+      )
+    ),
   ]
 }
 
@@ -247,6 +255,10 @@ module "docker-registry-vm" {
   vm_mac_address = "DE:AD:BE:EF:22:22" # mapped to 10.0.20.10 in dhcp
   bridge         = "vmbr1"
   vlan_tag       = "20"
+  # ports:
+  # 5000 -> registry
+  # 5001 -> metrics
+  # 5002 -> ngin proxy <-- use this to prevent races on the same blobs
 }
 
 # module that provisions the proxmox host?

modules/create-template-vm/cloud_init.yaml

@@ -26,6 +26,7 @@ packages:
   - gpg
   - isc-dhcp-client
   - cloud-guest-utils # to enable resizing of disk via growpart
+  - nginx
   # docker
   - docker-ce
   - docker-ce-cli

modules/create-template-vm/main.tf

@@ -81,7 +81,13 @@ resource "null_resource" "upload_cloud_init" {
     )
   }
 
+  # Force recreate when the below changes
   triggers = {
-    file_hash = filesha256("${path.module}/cloud_init.yaml")
+    file_hash                        = filesha256("${path.module}/cloud_init.yaml")
+    provision_cmds                   = join(", ", var.provision_cmds)
+    is_k8s_template                  = var.is_k8s_template,
+    passwd                           = var.user_passwd,
+    k8s_join_command                 = var.k8s_join_command,
+    containerd_config_update_command = var.containerd_config_update_command
   }
 }

modules/docker-registry/nginx_registry.conf

@@ -0,0 +1,58 @@
+proxy_cache_path /var/cache/nginx/registry
+    levels=1:2
+    keys_zone=registry:500m
+    max_size=50g
+    inactive=24h
+    use_temp_path=off;
+
+upstream docker_registry {
+    server 127.0.0.1:5000;
+    keepalive 32;
+}
+
+server {
+    listen 5002;
+    server_name _;
+
+    # Access log
+    access_log /var/log/nginx/registry.access.log combined;
+
+    # Error log
+    error_log /var/log/nginx/registry.error.log warn;
+
+    # Required for large blobs
+    client_max_body_size 0;
+
+    # Disable buffering to clients, keep it between nginx<->registry
+    proxy_request_buffering off;
+    proxy_buffering on;
+
+    location /v2/ {
+        proxy_pass http://docker_registry;
+
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header Connection "";
+
+        # --- CRITICAL PART ---
+        proxy_cache registry;
+        proxy_cache_lock on;
+        proxy_cache_lock_timeout 15m;
+        proxy_cache_lock_age 15m;
+        proxy_cache_use_stale updating;
+
+        # Cache only successful pulls
+        proxy_cache_valid 200 206 24h;
+
+        # HEAD requests must not poison cache
+        proxy_cache_methods GET;
+
+        # Do not cache pushes
+        proxy_no_cache $http_authorization;
+        proxy_cache_bypass $http_authorization;
+
+        # Prevent partial responses
+        proxy_read_timeout 900;
+        proxy_send_timeout 900;
+    }
+}