diff --git a/main.tf b/main.tf index b4ec926c..9dcb32eb 100644 --- a/main.tf +++ b/main.tf @@ -227,7 +227,15 @@ module "docker-registry-template" { ), "( crontab -l 2>/dev/null; echo '0 3 * * 0 /usr/bin/docker exec registry registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -", "( crontab -l 2>/dev/null; echo '0 * * * * /usr/bin/docker restart registry' ) | crontab -", - "docker run -p 5000:5000 -p 5001:5001 -d --restart always --name registry -v /etc/docker-registry/config.yml:/etc/docker/registry/config.yml registry:2" + "docker run -p 5000:5000 -p 5001:5001 -d --restart always --name registry -v /etc/docker-registry/config.yml:/etc/docker/registry/config.yml registry:2", + # Setup the registry nginx config; We want clients to connect via the nginx to serialize requests for the same blobs + # Otherwise race conditions lead to corrupt blobs + "mkdir -p /var/cache/nginx/registry", + format("echo %s | base64 -d > /etc/nginx/conf.d/registry.conf", + base64encode( + templatefile("${path.root}/modules/docker-registry/nginx_registry.conf", {}) + ) + ), ] } @@ -247,6 +255,10 @@ module "docker-registry-vm" { vm_mac_address = "DE:AD:BE:EF:22:22" # mapped to 10.0.20.10 in dhcp bridge = "vmbr1" vlan_tag = "20" + # ports: + # 5000 -> registry + # 5001 -> metrics + # 5002 -> ngin proxy <-- use this to prevent races on the same blobs } # module that provisions the proxmox host? diff --git a/modules/create-template-vm/cloud_init.yaml b/modules/create-template-vm/cloud_init.yaml index 74e69011..3792d2f5 100644 --- a/modules/create-template-vm/cloud_init.yaml +++ b/modules/create-template-vm/cloud_init.yaml @@ -26,6 +26,7 @@ packages: - gpg - isc-dhcp-client - cloud-guest-utils # to enable resizing of disk via growpart + - nginx # docker - docker-ce - docker-ce-cli diff --git a/modules/create-template-vm/main.tf b/modules/create-template-vm/main.tf index 0b78b0a7..ad82d57e 100644 --- a/modules/create-template-vm/main.tf +++ b/modules/create-template-vm/main.tf @@ -81,7 +81,13 @@ resource "null_resource" "upload_cloud_init" { ) } + # Force recreate when the below changes triggers = { - file_hash = filesha256("${path.module}/cloud_init.yaml") + file_hash = filesha256("${path.module}/cloud_init.yaml") + provision_cmds = join(", ", var.provision_cmds) + is_k8s_template = var.is_k8s_template, + passwd = var.user_passwd, + k8s_join_command = var.k8s_join_command, + containerd_config_update_command = var.containerd_config_update_command } } diff --git a/modules/docker-registry/nginx_registry.conf b/modules/docker-registry/nginx_registry.conf new file mode 100644 index 00000000..3391079e --- /dev/null +++ b/modules/docker-registry/nginx_registry.conf @@ -0,0 +1,58 @@ +proxy_cache_path /var/cache/nginx/registry + levels=1:2 + keys_zone=registry:500m + max_size=50g + inactive=24h + use_temp_path=off; + +upstream docker_registry { + server 127.0.0.1:5000; + keepalive 32; +} + +server { + listen 5002; + server_name _; + + # Access log + access_log /var/log/nginx/registry.access.log combined; + + # Error log + error_log /var/log/nginx/registry.error.log warn; + + # Required for large blobs + client_max_body_size 0; + + # Disable buffering to clients, keep it between nginx<->registry + proxy_request_buffering off; + proxy_buffering on; + + location /v2/ { + proxy_pass http://docker_registry; + + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Connection ""; + + # --- CRITICAL PART --- + proxy_cache registry; + proxy_cache_lock on; + proxy_cache_lock_timeout 15m; + proxy_cache_lock_age 15m; + proxy_cache_use_stale updating; + + # Cache only successful pulls + proxy_cache_valid 200 206 24h; + + # HEAD requests must not poison cache + proxy_cache_methods GET; + + # Do not cache pushes + proxy_no_cache $http_authorization; + proxy_cache_bypass $http_authorization; + + # Prevent partial responses + proxy_read_timeout 900; + proxy_send_timeout 900; + } +}