From b15246a2cb59610c381b11e0962e60989cebe67e Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Sun, 12 Oct 2025 18:54:22 +0000 Subject: [PATCH] add docker registry vm and allow multiple provisioning cmds in templates [ci skip] --- main.tf | 128 +++++++++++++++------ modules/create-template-vm/cloud_init.yaml | 13 ++- modules/create-template-vm/main.tf | 20 +++- modules/create-vm/main.tf | 14 +-- modules/docker-registry/config.yaml | 40 +++++++ 5 files changed, 171 insertions(+), 44 deletions(-) create mode 100644 modules/docker-registry/config.yaml diff --git a/main.tf b/main.tf index f1c8fb91..d8975c11 100644 --- a/main.tf +++ b/main.tf @@ -36,7 +36,7 @@ variable "dbaas_pgadmin_password" {} variable "drone_github_client_id" {} variable "drone_github_client_secret" {} variable "drone_rpc_secret" {} -# variable "dockerhub_password" {} +variable "dockerhub_registry_password" {} variable "oauth2_proxy_client_id" {} variable "oauth2_proxy_client_secret" {} variable "oauth2_proxy_authenticated_emails" {} @@ -119,18 +119,6 @@ variable "xray_reality_private_key" { type = string } variable "xray_reality_short_ids" { type = list(string) } -# data "terraform_remote_state" "foo" { -# backend = "kubernetes" -# config = { -# secret_suffix = "state" -# namespace = "drone" -# in_cluster_config = var.prod -# host = "https://kubernetes:6443" -# // load_config_file = true -# } - -# depends_on = [module.kubernetes_cluster] -# } provider "kubernetes" { config_path = var.prod ? "" : "~/.kube/config" } @@ -149,33 +137,107 @@ provider "proxmox" { } # TODO: add DEFCON levels -# resource "proxmox_virtual_environment_network_linux_vlan" "vlan1" { -# node_name = "pve" -# name = "ens160.99" - -# comment = "VLAN 99" -# } +# Main module to init infra from locals { - vm_template_name = "ubuntu-2404-cloudinit-template" - vm_cloud_init_snippet_name = "cloud_init.yaml" + k8s_vm_template = "ubuntu-2404-cloudinit-k8s-template" + k8s_cloud_init_snippet_name = "k8s_cloud_init.yaml" + k8s_cloud_init_image_path = "/var/lib/vz/template/iso/noble-server-cloudimg-amd64-k8s.img" + + non_k8s_vm_template = "ubuntu-2404-cloudinit-non-k8s-template" + non_k8s_cloud_init_snippet_name = "non_k8s_cloud_init.yaml" + non_k8s_cloud_init_image_path = "/var/lib/vz/template/iso/noble-server-cloudimg-amd64-non-k8s.img" + + cloud_init_image_url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" } -# Main module to init infra from -module "template-vm" { - source = "./modules/create-template-vm" - proxmox_host = var.proxmox_host - proxmox_user = "root" # SSH user on Proxmox host - cloud_image_url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" - image_path = "/var/lib/vz/template/iso/noble-server-cloudimg-amd64.img" - template_id = 8000 - template_name = local.vm_template_name +module "k8s-node-template" { + source = "./modules/create-template-vm" + proxmox_host = var.proxmox_host + proxmox_user = "root" # SSH user on Proxmox host - snippet_name = local.vm_cloud_init_snippet_name - user_passwd = var.vm_wizard_password - k8s_join_command = var.k8s_join_command + cloud_image_url = local.cloud_init_image_url + image_path = local.k8s_cloud_init_image_path + template_id = 2000 + template_name = local.k8s_vm_template + user_passwd = var.vm_wizard_password + + is_k8s_template = true # provision cloud init file with k8s deps + snippet_name = local.k8s_cloud_init_snippet_name + # Add mirror registry + containerd_config_update_command = "echo '[plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"docker.io\"]' >> /etc/containerd/config.toml && echo ' endpoint = [\"http://10.0.20.10:5000\"]' >> /etc/containerd/config.toml" # docker registry vm + k8s_join_command = var.k8s_join_command } +module "non-k8s-node-template" { + source = "./modules/create-template-vm" + proxmox_host = var.proxmox_host + proxmox_user = "root" # SSH user on Proxmox host + + cloud_image_url = local.cloud_init_image_url + image_path = local.non_k8s_cloud_init_image_path + template_id = 1000 + template_name = local.non_k8s_vm_template + user_passwd = var.vm_wizard_password + + is_k8s_template = false # provision cloud init file without k8s deps + snippet_name = local.non_k8s_cloud_init_snippet_name +} + +module "docker-registry-template" { + source = "./modules/create-template-vm" + + proxmox_host = var.proxmox_host + proxmox_user = "root" # SSH user on Proxmox host + + cloud_image_url = local.cloud_init_image_url + image_path = local.non_k8s_cloud_init_image_path # keke + template_id = 1001 + template_name = "docker-registry-template" + + user_passwd = var.vm_wizard_password + + is_k8s_template = false # provision cloud init file without k8s deps + snippet_name = "docker-registry.yaml" + + # Setup registry config and start container + provision_cmds = [ + "mkdir -p /etc/docker-registry", + format("echo %s | base64 -d > /etc/docker-registry/config.yml", + base64encode( + templatefile("./modules/docker-registry/config.yaml", { + password = var.dockerhub_registry_password + } + ) + ) + ), + "docker run -p 5000:5000 -d --restart always --name registry -v /etc/docker-registry/config.yml:/etc/docker/registry/config.yml registry:2" + ] +} + + +module "docker-registry-vm" { + source = "./modules/create-vm" + vmid = 220 + + vm_cpus = 4 + vm_mem_mb = 4196 + vm_disk_size = "32G" + + template_name = "docker-registry-template" + vm_name = "docker-registry" + cisnippet_name = "docker-registry.yaml" + + vm_mac_address = "DE:AD:BE:EF:22:22" # mapped to 10.0.20.10 in dhcp + bridge = "vmbr1" + vlan_tag = "20" +} + +# module that provisions the proxmox host? +# make dns stateless? +# pfsense/truenas configs in code +# etcd db backup in code + # module "k8s_node5" { # template_name = local.vm_template_name # source = "./modules/create-vm" diff --git a/modules/create-template-vm/cloud_init.yaml b/modules/create-template-vm/cloud_init.yaml index 4cf82c70..519f5b19 100644 --- a/modules/create-template-vm/cloud_init.yaml +++ b/modules/create-template-vm/cloud_init.yaml @@ -24,32 +24,43 @@ packages: - apt-transport-https - ca-certificates - gpg + - isc-dhcp-client # docker - docker-ce - docker-ce-cli - containerd.io - docker-buildx-plugin - docker-compose-plugin + %{if is_k8s_template} # kubernetes - kubeadm - kubelet + %{endif} apt: sources: + %{if is_k8s_template} kubernetes: source: "deb https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /" keyid: "DE15B14486CD377B9E876E1A234654DA9A296436" filename: kubernetes.list - docket: + %{endif} + docker: source: "deb https://download.docker.com/linux/ubuntu noble stable" keyid: "9DC858229FC7DD38854AE2D88D81803C0EBFCD88" filename: docker.list runcmd: + %{if is_k8s_template} - apt-mark hold kubelet kubeadm kubectl - systemctl stop kubelet - containerd config default | sudo tee /etc/containerd/config.toml + - ${containerd_config_update_command} - systemctl restart containerd - ${k8s_join_command} - systemctl enable kubelet - systemctl start kubelet + %{ endif } + %{ for provision_cmd in provision_cmds ~} + - ${provision_cmd} + %{ endfor ~} diff --git a/modules/create-template-vm/main.tf b/modules/create-template-vm/main.tf index 94854e09..0b78b0a7 100644 --- a/modules/create-template-vm/main.tf +++ b/modules/create-template-vm/main.tf @@ -13,6 +13,16 @@ variable "k8s_join_command" { type = string default = "" } +variable "containerd_config_update_command" { + type = string + default = "" + description = "Command to execute to update containerd config.toml; e.g add mirror" +} +variable "is_k8s_template" { type = bool } +variable "provision_cmds" { + type = list(string) + default = [] +} # SSH connection to Proxmox resource "null_resource" "create_template_remote" { @@ -60,7 +70,15 @@ resource "null_resource" "upload_cloud_init" { provisioner "file" { destination = "/var/lib/vz/snippets/${var.snippet_name}" - content = templatefile("${path.module}/cloud_init.yaml", { authorized_ssh_key = file("~/.ssh/id_ed25519.pub"), passwd = var.user_passwd, k8s_join_command = var.k8s_join_command }) + content = templatefile("${path.module}/cloud_init.yaml", { + is_k8s_template = var.is_k8s_template, + authorized_ssh_key = file("~/.ssh/id_ed25519.pub"), + passwd = var.user_passwd, + provision_cmds = var.provision_cmds, + k8s_join_command = var.k8s_join_command, + containerd_config_update_command = var.containerd_config_update_command + } + ) } triggers = { diff --git a/modules/create-vm/main.tf b/modules/create-vm/main.tf index f39a62e6..705b0e11 100644 --- a/modules/create-vm/main.tf +++ b/modules/create-vm/main.tf @@ -1,20 +1,21 @@ +variable "vm_name" { type = string } +variable "vmid" { + type = number + default = 0 +} variable "template_name" { type = string } -variable "vm_name" { default = "terraform-test" } variable "vm_cpus" { type = number default = 4 } - variable "vm_mem_mb" { type = number default = 8192 } - variable "vm_disk_size" { type = string default = "64G" } - variable "vm_mac_address" { type = string default = null @@ -29,11 +30,6 @@ variable "vlan_tag" { type = string default = null } -variable "vmid" { - type = number - default = 0 -} - resource "proxmox_vm_qemu" "cloudinit-vm" { vmid = var.vmid diff --git a/modules/docker-registry/config.yaml b/modules/docker-registry/config.yaml new file mode 100644 index 00000000..876dc6f6 --- /dev/null +++ b/modules/docker-registry/config.yaml @@ -0,0 +1,40 @@ +version: 0.1 +log: + fields: + service: registry +storage: + cache: + blobdescriptor: inmemory + filesystem: + rootdirectory: /var/lib/registry + maxsize: 5GiB + delete: + enabled: true + maintenance: + uploadpurging: + enabled: true + age: 24h + interval: 4h + dryrun: false + readonly: + enabled: false +http: + addr: :5000 + headers: + X-Content-Type-Options: [nosniff] + debug: + addr: ":5001" + # Enable proxy on nodes - https://github.com/containerd/containerd/blob/main/docs/cri/registry.md + # https://ops.tips/gists/retrieving-docker-registry-metrics-using-prometheus/ + prometheus: + enabled: true + path: "/metrics" +health: + storagedriver: + enabled: true + interval: 10s + threshold: 3 +proxy: + remoteurl: https://registry-1.docker.io + username: vbarzin@gmail.com + password: ${password}