diff --git a/modules/create-template-vm/cloud_init.yaml b/modules/create-template-vm/cloud_init.yaml index 2dc68bf0..fcd634e3 100644 --- a/modules/create-template-vm/cloud_init.yaml +++ b/modules/create-template-vm/cloud_init.yaml @@ -1,5 +1,8 @@ -#cloud-config -hostname: terraform-vm +#cloud-config +# Hostname intentionally NOT set here — cloud-init reads it from +# Proxmox's auto-generated meta-data (which uses `qm set --name `), +# so a single shared snippet works for every node. +manage_etc_hosts: true users: - name: wizard sudo: ALL=(ALL) NOPASSWD:ALL @@ -46,7 +49,7 @@ apt: sources: %{if is_k8s_template} kubernetes: - source: "deb https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /" + source: "deb https://pkgs.k8s.io/core:/stable:/v1.34/deb/ /" keyid: "DE15B14486CD377B9E876E1A234654DA9A296436" filename: kubernetes.list %{endif} @@ -56,23 +59,23 @@ apt: filename: docker.list %{if is_k8s_template} -# write_files delivers the multi-line containerd/kubelet setup script to a -# file BEFORE runcmd executes. This pattern avoids the YAML interpolation bug -# where multi-line $${containerd_config_update_command} (from -# stacks/infra/main.tf — has mixed-indent inner shell heredocs) inserted into -# a single `runcmd: - $${var}` item produces invalid YAML and silently fails -# cloud-init parsing (observed 2026-05-26 during node4 rebuild). With write_files, -# the multi-line content lives in a YAML literal block where ANY indent >= the -# block's content indent is valid — so col-0 heredoc lines like -# `[plugins."io.containerd.gc.v1.scheduler"]` survive cleanly. +# Setup script is base64-encoded by the module so YAML whitespace +# handling never touches the heredoc bodies inside it. Replaces an +# earlier `indent(6, …)` approach that put `[plugins.*]` TOML +# sections at col 6 inside `cat >> /etc/containerd/config.toml` +# heredocs — containerd refused to parse the result and the node5 v1 +# boot failed there (2026-05-26). Source: modules/create-template-vm/k8s-node-containerd-setup.sh write_files: - path: /usr/local/bin/k8s-node-containerd-setup.sh permissions: '0755' owner: root:root - content: | - #!/usr/bin/env bash - set -euo pipefail - ${indent(6, containerd_config_update_command)} + encoding: b64 + content: ${k8s_node_setup_script_b64} + - path: /usr/local/bin/k8s-node-post-join-tune.sh + permissions: '0755' + owner: root:root + encoding: b64 + content: ${k8s_node_post_join_script_b64} %{endif} runcmd: @@ -87,6 +90,20 @@ runcmd: - sed -i 's/#Compress=yes/Compress=yes/' /etc/systemd/journald.conf - systemctl restart systemd-journald %{if is_k8s_template} + # systemd-resolved global DNS fallback. Without this, only the + # link-level DNS from Proxmox's `qm set --nameserver` (Technitium, + # 10.0.20.201) is consulted — and Technitium returns NXDOMAIN for + # forgejo.viktorbarzin.me, so kubelet image pulls from the Forgejo + # registry break. Public DNS upstream + Technitium fallback matches + # the pre-existing manual setup on k8s-node1..4. + - mkdir -p /etc/systemd/resolved.conf.d + - | + cat > /etc/systemd/resolved.conf.d/global-dns.conf <<'EOF' + [Resolve] + DNS=8.8.8.8 1.1.1.1 + FallbackDNS=10.0.20.201 + EOF + - systemctl restart systemd-resolved # Re-enabled 2026-05-10: unattended-upgrades is back on, but with a tight # Allowed-Origins list, a Package-Blacklist for k8s/containerd/runc/calico, # and Automatic-Reboot disabled (kured + sentinel-gate handles reboots in a @@ -149,17 +166,19 @@ runcmd: - systemctl restart iscsid # Create /sentinel directory for kured reboot gating (sentinel gate DaemonSet) - mkdir -p /sentinel - # Create 4Gi swap file for worker node memory pressure relief (NOT for master — etcd is latency-critical) - - fallocate -l 4G /swapfile - - chmod 600 /swapfile - - mkswap /swapfile - - swapon /swapfile - - echo '/swapfile none swap sw 0 0' >> /etc/fstab - - sysctl -w vm.swappiness=10 - - echo 'vm.swappiness=10' >> /etc/sysctl.d/99-swap.conf + # Disable swap — kubelet defaults to failSwapOn=true and won't start otherwise. + # (Previously this snippet created a 4G swapfile for "memory pressure relief" + # but never set failSwapOn=false / memorySwap.swapBehavior together, so the + # join consistently bricked kubelet — observed on node6 boot v3 2026-05-26.) + - swapoff -a + - sed -i '/ swap / s/^/#/' /etc/fstab - ${k8s_join_command} - systemctl enable kubelet - systemctl start kubelet + # Kubelet tuning runs AFTER kubeadm join — that's when + # /var/lib/kubelet/config.yaml gets written. Restarts kubelet at the + # end to pick up the patched config. + - bash /usr/local/bin/k8s-node-post-join-tune.sh %{ endif } %{ for provision_cmd in provision_cmds ~} - ${provision_cmd} diff --git a/modules/create-template-vm/k8s-node-containerd-setup.sh b/modules/create-template-vm/k8s-node-containerd-setup.sh new file mode 100755 index 00000000..999bf4be --- /dev/null +++ b/modules/create-template-vm/k8s-node-containerd-setup.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# +# K8s node containerd + kubelet bootstrap. Runs once via cloud-init runcmd. +# Embedded into the cloud-init snippet base64-encoded by main.tf so YAML +# whitespace handling never touches the heredoc bodies — TOML / Python +# blocks below land in /etc/containerd/config.toml etc. with their leading +# whitespace intact. +# +# Layout: +# 1. /etc/containerd/config.toml — config_path + mirror dirs + GC tuning +# 2. /etc/containerd/certs.d/*/hosts.toml — per-registry mirror configs +# 3. /var/lib/kubelet/config.yaml — eviction + shutdown grace + log rotation +# 4. /etc/systemd/logind.conf.d + kubelet.service.d — graceful shutdown +# 5. (master-only) /etc/kubernetes/manifests — apiserver + controller flags +set -euo pipefail + +# 1. config_path — match BOTH quote styles. containerd v1 writes `""`, +# containerd v2.x writes `''`. Without the v2 match, hosts.toml mirror +# config is silently ignored — observed 2026-05-26 on k8s-node4 +# (containerd v2.2.4) and reproduced on k8s-node5 v1 boot. +sed -i "s|config_path = \"\"|config_path = \"/etc/containerd/certs.d\"|g" /etc/containerd/config.toml +sed -i "s|config_path = ''|config_path = \"/etc/containerd/certs.d\"|g" /etc/containerd/config.toml + +# 2. Per-registry hosts.toml — pull-through caches on docker-registry VM +# (10.0.20.10) for high-traffic registries, Traefik LB (10.0.20.200) for +# forgejo. Low-traffic registries (registry.k8s.io, reg.kyverno.io) skip +# the cache and pull direct because past pull-through cache attempts +# truncated downloads and broke VPA certgen + Kyverno image pulls. + +mkdir -p /etc/containerd/certs.d/docker.io +cat > /etc/containerd/certs.d/docker.io/hosts.toml <<'DOCKERIO' +server = "https://registry-1.docker.io" + +[host."http://10.0.20.10:5000"] + capabilities = ["pull", "resolve"] + +[host."https://registry-1.docker.io"] + capabilities = ["pull", "resolve"] +DOCKERIO + +mkdir -p /etc/containerd/certs.d/ghcr.io +cat > /etc/containerd/certs.d/ghcr.io/hosts.toml <<'GHCR' +server = "https://ghcr.io" + +[host."http://10.0.20.10:5010"] + capabilities = ["pull", "resolve"] + +[host."https://ghcr.io"] + capabilities = ["pull", "resolve"] +GHCR + +# Forgejo OCI registry: prefer in-cluster Traefik LB (10.0.20.200) to +# avoid hairpin NAT. Traefik serves the *.viktorbarzin.me wildcard so +# SNI verification succeeds. If the mirror is unreachable, fall back to +# public DNS resolution (needs the global DNS fallback set up below). +mkdir -p /etc/containerd/certs.d/forgejo.viktorbarzin.me +cat > /etc/containerd/certs.d/forgejo.viktorbarzin.me/hosts.toml <<'FORGEJO' +server = "https://forgejo.viktorbarzin.me" + +[host."https://10.0.20.200"] + capabilities = ["pull", "resolve"] +FORGEJO + +# quay.io + registry.k8s.io: include mirror configs that match node4's +# layout (no real pull-through cache today, server line is the direct +# upstream). Keeping these present makes the per-node config uniform and +# lets us flip a cache on later by editing only the [host."..."] block. +mkdir -p /etc/containerd/certs.d/quay.io +cat > /etc/containerd/certs.d/quay.io/hosts.toml <<'QUAY' +server = "https://quay.io" + +[host."http://10.0.20.10:5020"] + capabilities = ["pull", "resolve"] +QUAY + +mkdir -p /etc/containerd/certs.d/registry.k8s.io +cat > /etc/containerd/certs.d/registry.k8s.io/hosts.toml <<'K8SREG' +server = "https://registry.k8s.io" + +[host."http://10.0.20.10:5030"] + capabilities = ["pull", "resolve"] +K8SREG + +# 3. containerd tuning: parallel pulls + selective GC overrides. +# containerd v2's `config default` ALREADY emits `[plugins.'io.containerd.gc.v1.scheduler']`, +# `[plugins.'io.containerd.runtime.v2.task']`, and `[plugins.'io.containerd.metadata.v1.bolt']` +# sections — declaring them again fails with `toml: table … already exists` +# (observed on node6 boot 2026-05-26). Patch values in place instead. +sed -i 's/.*max_concurrent_downloads = 3/max_concurrent_downloads = 20/g' /etc/containerd/config.toml +# pause_threshold: 0.5 → 0.02 (run GC more aggressively when images dirty %) +sed -i "s/^[[:space:]]*pause_threshold = .*/ pause_threshold = 0.02/" /etc/containerd/config.toml +# schedule_delay: 0s/1ms → 30 min (longer cool-down between GC runs) +sed -i "s/^[[:space:]]*schedule_delay = .*/ schedule_delay = '1800s'/" /etc/containerd/config.toml +# exit_timeout: 0s → 5m (more aggressive container cleanup) +sed -i "s/^[[:space:]]*exit_timeout = .*/ exit_timeout = '5m'/" /etc/containerd/config.toml + +# 4. (kubelet tuning intentionally NOT here — /var/lib/kubelet/config.yaml +# only exists AFTER kubeadm join. That work runs in +# k8s-node-post-join-tune.sh, invoked as a separate cloud-init runcmd +# step after the join completes.) + +# 5. logind + kubelet systemd unit — total kubelet shutdown 310s, so +# logind InhibitDelay > that and kubelet TimeoutStopSec > that. +mkdir -p /etc/systemd/logind.conf.d +cat > /etc/systemd/logind.conf.d/kubelet-shutdown.conf <<'LOGIND_CONF' +[Login] +InhibitDelayMaxSec=480 +LOGIND_CONF +systemctl restart systemd-logind + +mkdir -p /etc/systemd/system/kubelet.service.d +cat > /etc/systemd/system/kubelet.service.d/20-shutdown.conf <<'KUBELET_SHUTDOWN' +[Service] +TimeoutStopSec=420s +KUBELET_SHUTDOWN +systemctl daemon-reload + +# 6. (master-only) faster pod eviction + attach-detach reconcile. +if [ -f /etc/kubernetes/manifests/kube-controller-manager.yaml ]; then + python3 - <<'CM_PATCH' +import yaml +with open('/etc/kubernetes/manifests/kube-controller-manager.yaml') as f: + m = yaml.safe_load(f) +args = m['spec']['containers'][0]['command'] +for flag in ['--attach-detach-reconcile-sync-period=15s']: + key = flag.split('=')[0] + args = [a for a in args if not a.startswith(key)] + args.append(flag) +m['spec']['containers'][0]['command'] = args +with open('/etc/kubernetes/manifests/kube-controller-manager.yaml', 'w') as f: + yaml.dump(m, f, default_flow_style=False) +CM_PATCH + python3 - <<'AS_PATCH' +import yaml +with open('/etc/kubernetes/manifests/kube-apiserver.yaml') as f: + m = yaml.safe_load(f) +args = m['spec']['containers'][0]['command'] +for flag in ['--default-unreachable-toleration-seconds=60', '--default-not-ready-toleration-seconds=60']: + key = flag.split('=')[0] + args = [a for a in args if not a.startswith(key)] + args.append(flag) +m['spec']['containers'][0]['command'] = args +with open('/etc/kubernetes/manifests/kube-apiserver.yaml', 'w') as f: + yaml.dump(m, f, default_flow_style=False) +AS_PATCH +fi diff --git a/modules/create-template-vm/k8s-node-post-join-tune.sh b/modules/create-template-vm/k8s-node-post-join-tune.sh new file mode 100755 index 00000000..9b474677 --- /dev/null +++ b/modules/create-template-vm/k8s-node-post-join-tune.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# +# Runs AFTER `kubeadm join` has written /var/lib/kubelet/config.yaml. +# Patches kubelet config in place (parallel image pulls, eviction +# thresholds, priority-based shutdown grace, container log rotation) +# and (on master) tightens controller-manager / apiserver flags. +# +# Embedded into the cloud-init snippet base64-encoded by main.tf so +# YAML whitespace doesn't touch the heredoc bodies inside. +set -euo pipefail + +if [ ! -f /var/lib/kubelet/config.yaml ]; then + echo "post-join-tune: /var/lib/kubelet/config.yaml not found — was kubeadm join run?" >&2 + exit 1 +fi + +# Parallel image pulls. +sed -i '/serializeImagePulls:/d' /var/lib/kubelet/config.yaml +sed -i '/maxParallelImagePulls:/d' /var/lib/kubelet/config.yaml +printf 'serializeImagePulls: false\nmaxParallelImagePulls: 50\n' >> /var/lib/kubelet/config.yaml + +# Memory / disk eviction. Aggressive disk thresholds (15%/20%) +# prevent the 2026-03-13 containerd image-store corruption that took +# down k8s-node2. +sed -i '/systemReserved:/d; /kubeReserved:/d; /evictionHard:/,/^[^ ]/{ /evictionHard:/d; /^ /d }; /evictionSoft:/,/^[^ ]/{ /evictionSoft:/d; /^ /d }; /evictionSoftGracePeriod:/,/^[^ ]/{ /evictionSoftGracePeriod:/d; /^ /d }' /var/lib/kubelet/config.yaml + +cat >> /var/lib/kubelet/config.yaml <<'KUBELET_PATCH' +systemReserved: + memory: "512Mi" + cpu: "200m" +kubeReserved: + memory: "512Mi" + cpu: "200m" +evictionHard: + memory.available: "500Mi" + nodefs.available: "15%" + imagefs.available: "20%" +evictionSoft: + memory.available: "1Gi" + nodefs.available: "20%" + imagefs.available: "25%" +evictionSoftGracePeriod: + memory.available: "30s" + nodefs.available: "60s" + imagefs.available: "30s" +memorySwap: + swapBehavior: "LimitedSwap" +KUBELET_PATCH + +# Container log rotation + priority-based shutdown grace. +sed -i '/^shutdownGracePeriod:/d; /^shutdownGracePeriodCriticalPods:/d' /var/lib/kubelet/config.yaml +python3 - <<'KUBELET_FINAL' +import yaml +with open('/var/lib/kubelet/config.yaml') as f: + cfg = yaml.safe_load(f) +cfg.pop('shutdownGracePeriod', None) +cfg.pop('shutdownGracePeriodCriticalPods', None) +cfg.pop('shutdownGracePeriodByPodPriority', None) +cfg['containerLogMaxSize'] = '10Mi' +cfg['containerLogMaxFiles'] = 3 +cfg['shutdownGracePeriodByPodPriority'] = [ + {'priority': 0, 'shutdownGracePeriodSeconds': 20}, + {'priority': 200000, 'shutdownGracePeriodSeconds': 20}, + {'priority': 400000, 'shutdownGracePeriodSeconds': 30}, + {'priority': 600000, 'shutdownGracePeriodSeconds': 30}, + {'priority': 800000, 'shutdownGracePeriodSeconds': 90}, + {'priority': 1000000, 'shutdownGracePeriodSeconds': 30}, + {'priority': 1200000, 'shutdownGracePeriodSeconds': 30}, + {'priority': 2000000000, 'shutdownGracePeriodSeconds': 30}, + {'priority': 2000001000, 'shutdownGracePeriodSeconds': 30}, +] +with open('/var/lib/kubelet/config.yaml', 'w') as f: + yaml.dump(cfg, f, default_flow_style=False) +KUBELET_FINAL + +# Reload kubelet to pick up new config (it's already started by the +# preceding cloud-init runcmd line — restart, not start). +systemctl restart kubelet diff --git a/modules/create-template-vm/main.tf b/modules/create-template-vm/main.tf index 032b4b9b..e7861d45 100644 --- a/modules/create-template-vm/main.tf +++ b/modules/create-template-vm/main.tf @@ -16,7 +16,7 @@ variable "k8s_join_command" { variable "containerd_config_update_command" { type = string default = "" - description = "Command to execute to update containerd config.toml; e.g add mirror" + description = "DEPRECATED: was inlined into write_files via indent(); the heredoc-TOML interaction broke containerd config parsing on node5 v1 boot 2026-05-26. The k8s setup script is now bundled inside the module at k8s-node-containerd-setup.sh — pass nothing here. Kept to avoid breaking stacks that still reference it; ignored when is_k8s_template=true." } variable "is_k8s_template" { type = bool } variable "ssh_private_key" { @@ -79,23 +79,26 @@ resource "null_resource" "upload_cloud_init" { provisioner "file" { destination = "/var/lib/vz/snippets/${var.snippet_name}" content = templatefile("${path.module}/cloud_init.yaml", { - is_k8s_template = var.is_k8s_template, - authorized_ssh_key = var.ssh_public_key, - passwd = var.user_passwd, - provision_cmds = var.provision_cmds, - k8s_join_command = var.k8s_join_command, - containerd_config_update_command = var.containerd_config_update_command + is_k8s_template = var.is_k8s_template, + authorized_ssh_key = var.ssh_public_key, + passwd = var.user_passwd, + provision_cmds = var.provision_cmds, + k8s_join_command = var.k8s_join_command, + k8s_node_setup_script_b64 = var.is_k8s_template ? base64encode(file("${path.module}/k8s-node-containerd-setup.sh")) : "" + k8s_node_post_join_script_b64 = var.is_k8s_template ? base64encode(file("${path.module}/k8s-node-post-join-tune.sh")) : "" } ) } # Force recreate when the below changes triggers = { - file_hash = filesha256("${path.module}/cloud_init.yaml") - provision_cmds = join(", ", var.provision_cmds) - is_k8s_template = var.is_k8s_template, - passwd = var.user_passwd, - k8s_join_command = var.k8s_join_command, - containerd_config_update_command = var.containerd_config_update_command + file_hash = filesha256("${path.module}/cloud_init.yaml") + setup_script_hash = var.is_k8s_template ? filesha256("${path.module}/k8s-node-containerd-setup.sh") : "" + post_join_script_hash = var.is_k8s_template ? filesha256("${path.module}/k8s-node-post-join-tune.sh") : "" + provision_cmds = join(", ", var.provision_cmds) + is_k8s_template = var.is_k8s_template, + passwd = var.user_passwd, + k8s_join_command = var.k8s_join_command, + ssh_public_key = var.ssh_public_key, } } diff --git a/scripts/provision-k8s-worker b/scripts/provision-k8s-worker new file mode 100755 index 00000000..f141d7e2 --- /dev/null +++ b/scripts/provision-k8s-worker @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# provision-k8s-worker NAME VMID IP[/CIDR] +# +# Clone PVE template 2000 (ubuntu-2404-cloudinit-k8s-template) into a new +# VM, configure resources to match k8s-node3/4 (32G RAM, 8 vCPU, host CPU, +# 256G disk, VLAN 20 on vmbr1), attach the shared cicustom snippet +# (/var/lib/vz/snippets/k8s_cloud_init.yaml), and start it. Cloud-init +# inside the VM installs containerd + kubelet, applies the bundled +# setup script, and runs the kubeadm join. No manual steps after this. +# +# Hostname is derived from `qm set --name $NAME` and read by cloud-init +# from Proxmox metadata — DO NOT hard-code in the snippet. +# +# Idempotent: aborts if VMID already exists or IP is already in use. +# +# Usage: +# ssh root@192.168.1.127 bash -s -- k8s-node6 206 10.0.20.106 < provision-k8s-worker +# or, if the script lives on the PVE host: +# provision-k8s-worker k8s-node6 206 10.0.20.106 +# +# Run on the PVE host (needs qm + /var/lib/vz/snippets access). +set -euo pipefail + +if [ $# -ne 3 ]; then + echo "usage: $0 NAME VMID IP" >&2 + echo " e.g. $0 k8s-node6 206 10.0.20.106" >&2 + exit 2 +fi + +NAME=$1 +VMID=$2 +IP=$3 +CIDR_IP="${IP}/22" +GW="10.0.20.1" +DNS="10.0.20.201" +SEARCH="viktorbarzin.lan" +TEMPLATE_ID=2000 +STORAGE="local-lvm" +USER_SNIPPET="local:snippets/k8s_cloud_init.yaml" +# Per-node meta-data snippet — written below — supplies local-hostname. +# Proxmox's auto-generated metadata DOESN'T include hostname when +# cicustom user=… is set, so the shared user-data snippet alone leaves +# nodes joining as "ubuntu" (image default). Per-node meta-data is the +# clean fix. +META_SNIPPET_FILE="/var/lib/vz/snippets/${NAME}-meta.yaml" +META_SNIPPET="local:snippets/${NAME}-meta.yaml" +BRIDGE="vmbr1" +VLAN=20 + +# Sanity: VMID must be free +if qm status "$VMID" >/dev/null 2>&1; then + echo "ERROR: VM $VMID already exists. Refusing to clobber." >&2 + qm status "$VMID" >&2 + exit 1 +fi + +# Sanity: IP must not be pingable +if ping -c 1 -W 1 "$IP" >/dev/null 2>&1; then + echo "ERROR: $IP is already responding to ping. Refusing to assign." >&2 + exit 1 +fi + +# Sanity: snippet must exist +if [ ! -f "/var/lib/vz/snippets/k8s_cloud_init.yaml" ]; then + echo "ERROR: /var/lib/vz/snippets/k8s_cloud_init.yaml missing." >&2 + echo " Run `tg apply` in infra/stacks/infra/ to regenerate it." >&2 + exit 1 +fi + +# Sanity: template must be a template +if ! qm config "$TEMPLATE_ID" | grep -q '^template: 1'; then + echo "ERROR: VMID $TEMPLATE_ID is not a template." >&2 + exit 1 +fi + +echo "[1/6] write per-node meta-data snippet ($META_SNIPPET_FILE)" +cat > "$META_SNIPPET_FILE" < $VMID ($NAME)" +qm clone "$TEMPLATE_ID" "$VMID" --name "$NAME" --full true --storage "$STORAGE" + +echo "[3/6] qm set $VMID — VM resources + network + cicustom" +qm set "$VMID" \ + --agent 1 \ + --balloon 32768 \ + --cores 8 \ + --cpu host \ + --memory 32768 \ + --net0 "virtio,bridge=$BRIDGE,tag=$VLAN" \ + --ipconfig0 "ip=$CIDR_IP,gw=$GW" \ + --nameserver "$DNS" \ + --searchdomain "$SEARCH" \ + --onboot 1 \ + --startup 'order=5,up=45,down=420' \ + --cicustom "user=$USER_SNIPPET,meta=$META_SNIPPET" + +echo "[4/6] qm resize $VMID scsi0 256G" +qm resize "$VMID" scsi0 256G + +echo "[5/6] qm start $VMID" +qm start "$VMID" + +echo "[6/6] Done. Cloud-init runs now; node should appear in 'kubectl get nodes' within ~6-10 min." +echo " Tail cloud-init: socat -u UNIX-CONNECT:/var/run/qemu-server/$VMID.serial0 STDOUT | strings" +echo " Final config:" +qm config "$VMID" | grep -E '^(name|cores|memory|net0|ipconfig0|cicustom|scsi0|onboot):' diff --git a/stacks/infra/main.tf b/stacks/infra/main.tf index 469dd507..ac8742c2 100644 --- a/stacks/infra/main.tf +++ b/stacks/infra/main.tf @@ -10,8 +10,9 @@ variable "proxmox_host" { type = string } variable "ssh_public_key" { - type = string - default = "" + type = string + default = "" + description = "DEPRECATED: was a tfvars input. Now read from Vault secret/viktor.ssh_public_key directly (see locals.k8s_ssh_public_key) so no apply-time argument can leave the snippet's authorized_keys empty." } variable "k8s_join_command" { type = string } @@ -40,6 +41,12 @@ locals { non_k8s_cloud_init_image_path = "/var/lib/vz/template/iso/noble-server-cloudimg-amd64-non-k8s.img" cloud_init_image_url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" + + # Source of truth for the wizard user's SSH key on every cloud-init + # generated VM. Lives in Vault so we never apply with an empty value + # (which silently locked the wizard account on the node5 v1 boot — + # 2026-05-26). Falls back to var.ssh_public_key for backward compat. + k8s_ssh_public_key = try(data.vault_kv_secret_v2.viktor.data["ssh_public_key"], var.ssh_public_key) } # --------------------------------------------------------------------------- @@ -52,7 +59,7 @@ module "k8s-node-template" { proxmox_user = "root" # SSH user on Proxmox host ssh_private_key = data.vault_kv_secret_v2.secrets.data["ssh_private_key"] - ssh_public_key = var.ssh_public_key + ssh_public_key = local.k8s_ssh_public_key cloud_image_url = local.cloud_init_image_url image_path = local.k8s_cloud_init_image_path @@ -62,167 +69,10 @@ module "k8s-node-template" { is_k8s_template = true # provision cloud init file with k8s deps snippet_name = local.k8s_cloud_init_snippet_name - # Add mirror registry - containerd_config_update_command = <<-EOF - # Set up config_path for per-registry mirror configuration. - # NOTE: containerd v2 writes `config_path = ''` (single quotes) on - # `config default`; v1 writes `config_path = ""`. Match both forms so this - # is idempotent across versions. Without the v2 match, hosts.toml mirror - # config is silently ignored — observed 2026-05-26 on node4 (containerd v2.2.4). - sed -i 's|config_path = .*|config_path = "/etc/containerd/certs.d"|' /etc/containerd/config.toml - - # Create hosts.toml for docker.io (Docker Hub) — high traffic, rate-limited - mkdir -p /etc/containerd/certs.d/docker.io - printf 'server = "https://registry-1.docker.io"\n\n[host."http://10.0.20.10:5000"]\n capabilities = ["pull", "resolve"]\n\n[host."https://registry-1.docker.io"]\n capabilities = ["pull", "resolve"]\n' > /etc/containerd/certs.d/docker.io/hosts.toml - - # Create hosts.toml for ghcr.io — medium traffic - mkdir -p /etc/containerd/certs.d/ghcr.io - printf 'server = "https://ghcr.io"\n\n[host."http://10.0.20.10:5010"]\n capabilities = ["pull", "resolve"]\n\n[host."https://ghcr.io"]\n capabilities = ["pull", "resolve"]\n' > /etc/containerd/certs.d/ghcr.io/hosts.toml - - # Forgejo OCI registry: redirect to in-cluster Traefik LB (10.0.20.200) so - # pulls don't hairpin out through the WAN gateway. Traefik serves the - # *.viktorbarzin.me wildcard so SNI verification still passes. - # registry.viktorbarzin.me / 10.0.20.10:5050 entries removed in Phase 4 of - # the forgejo-registry-consolidation 2026-05-07 — registry-private is gone. - mkdir -p /etc/containerd/certs.d/forgejo.viktorbarzin.me - printf 'server = "https://forgejo.viktorbarzin.me"\n\n[host."https://10.0.20.200"]\n capabilities = ["pull", "resolve"]\n' > /etc/containerd/certs.d/forgejo.viktorbarzin.me/hosts.toml - - # Low-traffic registries (registry.k8s.io, quay.io, reg.kyverno.io) pull directly. - # Pull-through cache removed: caused corrupted images (truncated downloads) - # breaking VPA certgen and Kyverno image pulls. - - sed -i 's/.*max_concurrent_downloads = 3/max_concurrent_downloads = 20/g' /etc/containerd/config.toml # Enable multiple concurrent downloads - - # Configure aggressive garbage collection to prevent disk space exhaustion (node2 incident prevention) - # Set up containerd GC for unused images and containers - cat >> /etc/containerd/config.toml << 'CONTAINERD_GC' - -[plugins."io.containerd.gc.v1.scheduler"] - # Run GC every 30 minutes instead of default 1 hour - pause_threshold = 0.02 - deletion_threshold = 0 - mutation_threshold = 100 - schedule_delay = "1800s" # 30 minutes - -[plugins."io.containerd.runtime.v2.task"] - # More aggressive container cleanup - exit_timeout = "5m" - -[plugins."io.containerd.metadata.v1.bolt"] - # Compact database more frequently - compact_threshold = 5242880 # 5MB instead of default 100MB -CONTAINERD_GC - sudo sed -i '/serializeImagePulls:/d' /var/lib/kubelet/config.yaml && \ - sudo sed -i '/maxParallelImagePulls:/d' /var/lib/kubelet/config.yaml && \ - echo -e 'serializeImagePulls: false\nmaxParallelImagePulls: 50' | sudo tee -a /var/lib/kubelet/config.yaml - - # Memory and disk reservation and eviction — prevent node OOM/disk full - # Aggressive disk eviction settings added after node2 containerd corruption incident (2026-03-13) - # These settings prevent disk space exhaustion that can corrupt containerd image store - sudo sed -i '/systemReserved:/d; /kubeReserved:/d; /evictionHard:/,/^[^ ]/{ /evictionHard:/d; /^ /d }; /evictionSoft:/,/^[^ ]/{ /evictionSoft:/d; /^ /d }; /evictionSoftGracePeriod:/,/^[^ ]/{ /evictionSoftGracePeriod:/d; /^ /d }' /var/lib/kubelet/config.yaml - cat <<'KUBELET_PATCH' | sudo tee -a /var/lib/kubelet/config.yaml -systemReserved: - memory: "512Mi" - cpu: "200m" -kubeReserved: - memory: "512Mi" - cpu: "200m" -evictionHard: - memory.available: "500Mi" - nodefs.available: "15%" # More aggressive: evict at 15% free (was 10%) - imagefs.available: "20%" # Much more aggressive: evict at 20% free to prevent containerd corruption -evictionSoft: - memory.available: "1Gi" - nodefs.available: "20%" # Start warnings at 20% free - imagefs.available: "25%" # Start warnings at 25% free for containerd safety -evictionSoftGracePeriod: - memory.available: "30s" - nodefs.available: "60s" # Grace period for disk space warnings - imagefs.available: "30s" # Shorter grace for critical containerd space -memorySwap: - swapBehavior: "LimitedSwap" -KUBELET_PATCH - - # Remove old 2-bucket shutdown config if present (replaced by priority-based) - sudo sed -i '/^shutdownGracePeriod:/d; /^shutdownGracePeriodCriticalPods:/d' /var/lib/kubelet/config.yaml - # Remove old shutdownGracePeriodByPodPriority block if present (idempotent re-apply) - sudo python3 -c " -import yaml, sys -with open('/var/lib/kubelet/config.yaml') as f: - cfg = yaml.safe_load(f) -cfg.pop('shutdownGracePeriod', None) -cfg.pop('shutdownGracePeriodCriticalPods', None) -cfg.pop('shutdownGracePeriodByPodPriority', None) -# Container log rotation limits — reduces root disk writes (~20-30 GB/day savings) -cfg['containerLogMaxSize'] = '10Mi' -cfg['containerLogMaxFiles'] = 3 -cfg['shutdownGracePeriodByPodPriority'] = [ - {'priority': 0, 'shutdownGracePeriodSeconds': 20}, - {'priority': 200000, 'shutdownGracePeriodSeconds': 20}, - {'priority': 400000, 'shutdownGracePeriodSeconds': 30}, - {'priority': 600000, 'shutdownGracePeriodSeconds': 30}, - {'priority': 800000, 'shutdownGracePeriodSeconds': 90}, - {'priority': 1000000, 'shutdownGracePeriodSeconds': 30}, - {'priority': 1200000, 'shutdownGracePeriodSeconds': 30}, - {'priority': 2000000000, 'shutdownGracePeriodSeconds': 30}, - {'priority': 2000001000, 'shutdownGracePeriodSeconds': 30}, -] -with open('/var/lib/kubelet/config.yaml', 'w') as f: - yaml.dump(cfg, f, default_flow_style=False) -" - - # Systemd: increase InhibitDelayMaxSec so logind doesn't force-kill before kubelet finishes graceful shutdown - # Total kubelet shutdown time: 310s. InhibitDelay must exceed this. - mkdir -p /etc/systemd/logind.conf.d - cat <<'LOGIND_CONF' | sudo tee /etc/systemd/logind.conf.d/kubelet-shutdown.conf -[Login] -InhibitDelayMaxSec=480 -LOGIND_CONF - sudo systemctl restart systemd-logind - - # Systemd: increase kubelet stop timeout to match total shutdown grace period (310s + buffer) - mkdir -p /etc/systemd/system/kubelet.service.d - cat <<'KUBELET_SHUTDOWN' | sudo tee /etc/systemd/system/kubelet.service.d/20-shutdown.conf -[Service] -TimeoutStopSec=420s -KUBELET_SHUTDOWN - sudo systemctl daemon-reload - - # Tune controller-manager + apiserver for faster volume detach on node failure - # Only on master node (has static pod manifests) - if [ -f /etc/kubernetes/manifests/kube-controller-manager.yaml ]; then - sudo python3 -c " -import yaml -# Controller-manager: faster attach-detach reconciliation (15s vs 1m default) -with open('/etc/kubernetes/manifests/kube-controller-manager.yaml') as f: - m = yaml.safe_load(f) -args = m['spec']['containers'][0]['command'] -for flag in ['--attach-detach-reconcile-sync-period=15s']: - key = flag.split('=')[0] - args = [a for a in args if not a.startswith(key)] - args.append(flag) -m['spec']['containers'][0]['command'] = args -with open('/etc/kubernetes/manifests/kube-controller-manager.yaml', 'w') as f: - yaml.dump(m, f, default_flow_style=False) -print('controller-manager: attach-detach-reconcile-sync-period=15s') -" - sudo python3 -c " -import yaml -# API server: faster pod eviction from unreachable nodes (60s vs 300s default) -with open('/etc/kubernetes/manifests/kube-apiserver.yaml') as f: - m = yaml.safe_load(f) -args = m['spec']['containers'][0]['command'] -for flag in ['--default-unreachable-toleration-seconds=60', '--default-not-ready-toleration-seconds=60']: - key = flag.split('=')[0] - args = [a for a in args if not a.startswith(key)] - args.append(flag) -m['spec']['containers'][0]['command'] = args -with open('/etc/kubernetes/manifests/kube-apiserver.yaml', 'w') as f: - yaml.dump(m, f, default_flow_style=False) -print('apiserver: unreachable+not-ready toleration=60s') -" - fi - EOF + # containerd setup script now bundled in the module + # (k8s-node-containerd-setup.sh); the deprecated variable is + # ignored when is_k8s_template=true. + containerd_config_update_command = "" k8s_join_command = var.k8s_join_command } diff --git a/stacks/proxmox-csi/modules/proxmox-csi/main.tf b/stacks/proxmox-csi/modules/proxmox-csi/main.tf index 9103bb0d..ec6b03f2 100644 --- a/stacks/proxmox-csi/modules/proxmox-csi/main.tf +++ b/stacks/proxmox-csi/modules/proxmox-csi/main.tf @@ -107,6 +107,8 @@ locals { "k8s-node2" = { vmid = 202, proxmox_node = "pve" } "k8s-node3" = { vmid = 203, proxmox_node = "pve" } "k8s-node4" = { vmid = 204, proxmox_node = "pve" } + "k8s-node5" = { vmid = 205, proxmox_node = "pve" } + "k8s-node6" = { vmid = 206, proxmox_node = "pve" } } }