diff --git a/stacks/infra/.gitkeep b/stacks/infra/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/stacks/infra/.terraform.lock.hcl b/stacks/infra/.terraform.lock.hcl new file mode 100644 index 00000000..28fba95b --- /dev/null +++ b/stacks/infra/.terraform.lock.hcl @@ -0,0 +1,43 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.4" + hashes = [ + "h1:L5V05xwp/Gto1leRryuesxjMfgZwjb7oool4WS1UEFQ=", + "zh:59f6b52ab4ff35739647f9509ee6d93d7c032985d9f8c6237d1f8a59471bbbe2", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:795c897119ff082133150121d39ff26cb5f89a730a2c8c26f3a9c1abf81a9c43", + "zh:7b9c7b16f118fbc2b05a983817b8ce2f86df125857966ad356353baf4bff5c0a", + "zh:85e33ab43e0e1726e5f97a874b8e24820b6565ff8076523cc2922ba671492991", + "zh:9d32ac3619cfc93eb3c4f423492a8e0f79db05fec58e449dee9b2d5873d5f69f", + "zh:9e15c3c9dd8e0d1e3731841d44c34571b6c97f5b95e8296a45318b94e5287a6e", + "zh:b4c2ab35d1b7696c30b64bf2c0f3a62329107bd1a9121ce70683dec58af19615", + "zh:c43723e8cc65bcdf5e0c92581dcbbdcbdcf18b8d2037406a5f2033b1e22de442", + "zh:ceb5495d9c31bfb299d246ab333f08c7fb0d67a4f82681fbf47f2a21c3e11ab5", + "zh:e171026b3659305c558d9804062762d168f50ba02b88b231d20ec99578a6233f", + "zh:ed0fe2acdb61330b01841fa790be00ec6beaac91d41f311fb8254f74eb6a711f", + ] +} + +provider "registry.terraform.io/telmate/proxmox" { + version = "3.0.2-rc07" + constraints = "3.0.2-rc07" + hashes = [ + "h1:0UpRJ8PFsu9lhD3p2KUdUNVsDPbjZLPR46wYRpt1dxc=", + "zh:2ee860cd0a368b3eaa53f4a9ea46f16dab8a97929e813ea6ef55183f8112c2ca", + "zh:415965fd915bae2040d7f79e45f64d6e3ae61149c10114efeac1b34687d7296c", + "zh:6584b2055df0e32062561c615e3b6b2c291ca8c959440adda09ef3ec1e1436bd", + "zh:65dcfad71928e0a8dd9befc22524ed686be5020b0024dc5cca5184c7420eeb6b", + "zh:7253dc29bd265d33f2791ac4f779c5413f16720bb717de8e6c5fcb2c858648ea", + "zh:7ec8993da10a47606670f9f67cfd10719a7580641d11c7aa761121c4a2bd66fb", + "zh:999a3f7a9dcf517967fc537e6ec930a8172203642fb01b8e1f78f908373db210", + "zh:a50e6df7280eb6584a5fd2456e3f5b6df13b2ec8a7fa4605511e438e1863be42", + "zh:b25b329a1e42681c509d027fee0365414f0cc5062b65690cfc3386aab16132ae", + "zh:c028877fdb438ece48f7bc02b65bbae9ca7b7befbd260e519ccab6c0cbb39f26", + "zh:cf0eaa3ea9fcc6d62793637947f1b8d7c885b6ad74695ab47e134e4ff132190f", + "zh:d5ade3fae031cc629b7c512a7b60e46570f4c41665e88a595d7efd943dde5ab2", + "zh:f388c15ad1ecfc09e7361e3b98bae9b627a3a85f7b908c9f40650969c949901c", + "zh:f415cc6f735a3971faae6ac24034afdb9ee83373ef8de19a9631c187d5adc7db", + ] +} diff --git a/stacks/infra/main.tf b/stacks/infra/main.tf new file mode 100644 index 00000000..6fd4374f --- /dev/null +++ b/stacks/infra/main.tf @@ -0,0 +1,251 @@ +# Infra stack — Proxmox VM templates and docker-registry VM +# +# Wraps the existing create-template-vm and create-vm modules with +# source paths adjusted for the stacks/infra/ working directory. + +# --------------------------------------------------------------------------- +# Variables +# --------------------------------------------------------------------------- + +variable "proxmox_host" { type = string } + +variable "ssh_private_key" { + type = string + default = "" +} + +variable "ssh_public_key" { + type = string + default = "" +} + +variable "vm_wizard_password" { type = string } + +variable "k8s_join_command" { type = string } + +variable "dockerhub_registry_password" {} + +# --------------------------------------------------------------------------- +# Locals +# --------------------------------------------------------------------------- + +locals { + k8s_vm_template = "ubuntu-2404-cloudinit-k8s-template" + k8s_cloud_init_snippet_name = "k8s_cloud_init.yaml" + k8s_cloud_init_image_path = "/var/lib/vz/template/iso/noble-server-cloudimg-amd64-k8s.img" + + non_k8s_vm_template = "ubuntu-2404-cloudinit-non-k8s-template" + non_k8s_cloud_init_snippet_name = "non_k8s_cloud_init.yaml" + non_k8s_cloud_init_image_path = "/var/lib/vz/template/iso/noble-server-cloudimg-amd64-non-k8s.img" + + cloud_init_image_url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" +} + +# --------------------------------------------------------------------------- +# K8s node template +# --------------------------------------------------------------------------- + +module "k8s-node-template" { + source = "../../modules/create-template-vm" + proxmox_host = var.proxmox_host + proxmox_user = "root" # SSH user on Proxmox host + + ssh_private_key = var.ssh_private_key + ssh_public_key = var.ssh_public_key + + cloud_image_url = local.cloud_init_image_url + image_path = local.k8s_cloud_init_image_path + template_id = 2000 + template_name = local.k8s_vm_template + user_passwd = var.vm_wizard_password + + is_k8s_template = true # provision cloud init file with k8s deps + snippet_name = local.k8s_cloud_init_snippet_name + # Add mirror registry + containerd_config_update_command = <<-EOF + # Set up config_path for per-registry mirror configuration + sed -i 's|config_path = ""|config_path = "/etc/containerd/certs.d"|' /etc/containerd/config.toml + + # Create hosts.toml for docker.io (Docker Hub) + mkdir -p /etc/containerd/certs.d/docker.io + printf 'server = "https://registry-1.docker.io"\n\n[host."http://10.0.20.10:5000"]\n capabilities = ["pull", "resolve"]\n' > /etc/containerd/certs.d/docker.io/hosts.toml + + # Create hosts.toml for ghcr.io + mkdir -p /etc/containerd/certs.d/ghcr.io + printf 'server = "https://ghcr.io"\n\n[host."http://10.0.20.10:5010"]\n capabilities = ["pull", "resolve"]\n' > /etc/containerd/certs.d/ghcr.io/hosts.toml + + # Create hosts.toml for quay.io + mkdir -p /etc/containerd/certs.d/quay.io + printf 'server = "https://quay.io"\n\n[host."http://10.0.20.10:5020"]\n capabilities = ["pull", "resolve"]\n' > /etc/containerd/certs.d/quay.io/hosts.toml + + # Create hosts.toml for registry.k8s.io + mkdir -p /etc/containerd/certs.d/registry.k8s.io + printf 'server = "https://registry.k8s.io"\n\n[host."http://10.0.20.10:5030"]\n capabilities = ["pull", "resolve"]\n' > /etc/containerd/certs.d/registry.k8s.io/hosts.toml + + # Create hosts.toml for reg.kyverno.io + mkdir -p /etc/containerd/certs.d/reg.kyverno.io + printf 'server = "https://reg.kyverno.io"\n\n[host."http://10.0.20.10:5040"]\n capabilities = ["pull", "resolve"]\n' > /etc/containerd/certs.d/reg.kyverno.io/hosts.toml + + sed -i 's/.*max_concurrent_downloads = 3/max_concurrent_downloads = 20/g' /etc/containerd/config.toml # Enable multiple concurrent downloads + sudo sed -i '/serializeImagePulls:/d' /var/lib/kubelet/config.yaml && \ + sudo sed -i '/maxParallelImagePulls:/d' /var/lib/kubelet/config.yaml && \ + echo -e 'serializeImagePulls: false\nmaxParallelImagePulls: 50' | sudo tee -a /var/lib/kubelet/config.yaml + EOF + k8s_join_command = var.k8s_join_command +} + +# --------------------------------------------------------------------------- +# Non-K8s node template +# --------------------------------------------------------------------------- + +module "non-k8s-node-template" { + source = "../../modules/create-template-vm" + proxmox_host = var.proxmox_host + proxmox_user = "root" # SSH user on Proxmox host + + ssh_private_key = var.ssh_private_key + ssh_public_key = var.ssh_public_key + + cloud_image_url = local.cloud_init_image_url + image_path = local.non_k8s_cloud_init_image_path + template_id = 1000 + template_name = local.non_k8s_vm_template + user_passwd = var.vm_wizard_password + + is_k8s_template = false # provision cloud init file without k8s deps + snippet_name = local.non_k8s_cloud_init_snippet_name +} + +# --------------------------------------------------------------------------- +# Docker registry template +# --------------------------------------------------------------------------- + +module "docker-registry-template" { + source = "../../modules/create-template-vm" + + proxmox_host = var.proxmox_host + proxmox_user = "root" # SSH user on Proxmox host + + ssh_private_key = var.ssh_private_key + ssh_public_key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDHLhYDfyx237eJgOGVoJRECpUS95+7rEBS9vacsIxtx devvm" + + cloud_image_url = local.cloud_init_image_url + image_path = local.non_k8s_cloud_init_image_path # keke + template_id = 1001 + template_name = "docker-registry-template" + + user_passwd = var.vm_wizard_password + + is_k8s_template = false # provision cloud init file without k8s deps + snippet_name = "docker-registry.yaml" + + # Setup registry config and start container + provision_cmds = [ + "mkdir -p /etc/docker-registry", + format("echo %s | base64 -d > /etc/docker-registry/config.yml", + base64encode( + templatefile("../../modules/docker-registry/config.yaml", { + password = var.dockerhub_registry_password + } + ) + ) + ), + "( crontab -l 2>/dev/null; echo '0 3 * * 0 /usr/bin/docker exec registry registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -", + # Hourly restart cron removed - it wiped the in-memory blobdescriptor cache every hour, + # causing low cache hit rates on the pull-through proxy. Docker containers use --restart always. + "docker run -p 5000:5000 -p 5001:5001 -d --restart always --name registry -v /etc/docker-registry/config.yml:/etc/docker/registry/config.yml registry:2", + # ghcr.io proxy + "mkdir -p /etc/docker-registry/ghcr", + format("echo %s | base64 -d > /etc/docker-registry/ghcr/config.yml", + base64encode( + templatefile("../../modules/docker-registry/config-proxy.yaml.tpl", { + name = "ghcr" + remote_url = "https://ghcr.io" + }) + ) + ), + "docker run -p 5010:5000 -d --restart always --name registry-ghcr -v /etc/docker-registry/ghcr/config.yml:/etc/docker/registry/config.yml registry:2", + "( crontab -l 2>/dev/null; echo '5 3 * * 0 /usr/bin/docker exec registry-ghcr registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -", + # quay.io proxy + "mkdir -p /etc/docker-registry/quay", + format("echo %s | base64 -d > /etc/docker-registry/quay/config.yml", + base64encode( + templatefile("../../modules/docker-registry/config-proxy.yaml.tpl", { + name = "quay" + remote_url = "https://quay.io" + }) + ) + ), + "docker run -p 5020:5000 -d --restart always --name registry-quay -v /etc/docker-registry/quay/config.yml:/etc/docker/registry/config.yml registry:2", + "( crontab -l 2>/dev/null; echo '10 3 * * 0 /usr/bin/docker exec registry-quay registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -", + # registry.k8s.io proxy + "mkdir -p /etc/docker-registry/k8s", + format("echo %s | base64 -d > /etc/docker-registry/k8s/config.yml", + base64encode( + templatefile("../../modules/docker-registry/config-proxy.yaml.tpl", { + name = "k8s" + remote_url = "https://registry.k8s.io" + }) + ) + ), + "docker run -p 5030:5000 -d --restart always --name registry-k8s -v /etc/docker-registry/k8s/config.yml:/etc/docker/registry/config.yml registry:2", + "( crontab -l 2>/dev/null; echo '15 3 * * 0 /usr/bin/docker exec registry-k8s registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -", + # reg.kyverno.io proxy + "mkdir -p /etc/docker-registry/kyverno", + format("echo %s | base64 -d > /etc/docker-registry/kyverno/config.yml", + base64encode( + templatefile("../../modules/docker-registry/config-proxy.yaml.tpl", { + name = "kyverno" + remote_url = "https://reg.kyverno.io" + }) + ) + ), + "docker run -p 5040:5000 -d --restart always --name registry-kyverno -v /etc/docker-registry/kyverno/config.yml:/etc/docker/registry/config.yml registry:2", + "( crontab -l 2>/dev/null; echo '20 3 * * 0 /usr/bin/docker exec registry-kyverno registry garbage-collect -m /etc/docker/registry/config.yml' ) | crontab -", + # Setup the registry nginx config; We want clients to connect via the nginx to serialize requests for the same blobs + # Otherwise race conditions lead to corrupt blobs + "mkdir -p /var/cache/nginx/registry", + format("echo %s | base64 -d > /etc/nginx/conf.d/registry.conf", + base64encode( + templatefile("${path.root}/../../modules/docker-registry/nginx_registry.conf", {}) + ) + ), + "docker run -d --restart always --net host --name registry-ui -e NGINX_LISTEN_PORT=8080 -e NGINX_PROXY_PASS_URL=http://127.0.0.1:5000 -e DELETE_IMAGES=true -e SINGLE_REGISTRY=true -e SHOW_CONTENT_DIGEST=true -e SHOW_CATALOG_NB_TAGS=true -e CATALOG_ELEMENTS_LIMIT=1000 -e TAGLIST_PAGE_SIZE=100 -e REGISTRY_TITLE=viktorbarzin.me joxit/docker-registry-ui:latest", + # Deploy tag cleanup script (keep last 10 tags per image) and schedule daily at 2am before weekly GC + format("echo %s | base64 -d > /etc/docker-registry/cleanup-tags.sh && chmod +x /etc/docker-registry/cleanup-tags.sh", + base64encode(file("${path.root}/../../modules/docker-registry/cleanup-tags.sh")) + ), + "( crontab -l 2>/dev/null; echo '0 2 * * * python3 /etc/docker-registry/cleanup-tags.sh 10 >> /var/log/registry-cleanup.log 2>&1' ) | crontab -", + ] +} + +# --------------------------------------------------------------------------- +# Docker registry VM +# --------------------------------------------------------------------------- + +module "docker-registry-vm" { + source = "../../modules/create-vm" + vmid = 220 + + vm_cpus = 4 + vm_mem_mb = 4196 + vm_disk_size = "64G" + + template_name = "docker-registry-template" + vm_name = "docker-registry" + cisnippet_name = "docker-registry.yaml" + + vm_mac_address = "DE:AD:BE:EF:22:22" # mapped to 10.0.20.10 in dhcp + bridge = "vmbr1" + vlan_tag = "20" + ipconfig0 = "ip=10.0.20.10/24,gw=10.0.20.1" + # ports: + # 5000 -> registry (docker.io proxy) + # 5001 -> metrics + # 5002 -> nginx proxy <-- use this to prevent races on the same blobs + # 5010 -> registry-ghcr (ghcr.io proxy) + # 5020 -> registry-quay (quay.io proxy) + # 5030 -> registry-k8s (registry.k8s.io proxy) + # 5040 -> registry-kyverno (reg.kyverno.io proxy) + # 8080 -> registry-ui (joxit/docker-registry-ui) +} diff --git a/stacks/infra/terragrunt.hcl b/stacks/infra/terragrunt.hcl new file mode 100644 index 00000000..f01b00bf --- /dev/null +++ b/stacks/infra/terragrunt.hcl @@ -0,0 +1,36 @@ +# stacks/infra/terragrunt.hcl +include "root" { + path = find_in_parent_folders() +} + +# Override provider generation to include proxmox (instead of k8s providers) +generate "providers" { + path = "providers.tf" + if_exists = "overwrite_terragrunt" + contents = <