fix: restore tree dropped by 6d224861; land stem95su gdrive-sync (10m) [ci skip]

6d224861 came from a --no-checkout worktree whose empty index made the commit drop every file except two. This restores 05b50d2b's full tree and correctly adds stacks/stem95su/gdrive-sync.tf + the service-catalog stem95su entry. Forward-only (parent=6d224861, no force-push); [ci skip] since the live infra was never applied from the broken commit. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-09 08:45:33 +00:00 · 2026-06-09 08:45:33 +00:00 · fd0f4a0365
commit fd0f4a0365
parent 6d224861c4
1166 changed files with 358546 additions and 0 deletions
--- a/modules/create-template-vm/cloud_init.yaml
+++ b/modules/create-template-vm/cloud_init.yaml
@ -0,0 +1,185 @@
+#cloud-config
+# Hostname intentionally NOT set here — cloud-init reads it from
+# Proxmox's auto-generated meta-data (which uses `qm set --name <X>`),
+# so a single shared snippet works for every node.
+manage_etc_hosts: true
+users:
+  - name: wizard
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    ssh_authorized_keys:
+      - ${authorized_ssh_key}
+    passwd: ${passwd}
+    lock_passwd: false # enable passwd login
+    shell: /bin/bash
+package_update: true
+package_upgrade: true
+packages:
+  - htop
+  - vim
+  - curl
+  - jq
+  - tcpdump
+  - tree
+  - tmux
+  - wget
+  - net-tools
+  - zsh
+  - apt-transport-https
+  - ca-certificates
+  - gpg
+  - isc-dhcp-client
+  - cloud-guest-utils # to enable resizing of disk via growpart
+  - qemu-guest-agent
+  - nginx
+  # docker
+  - docker-ce
+  - docker-ce-cli
+  - containerd.io
+  - docker-buildx-plugin
+  - docker-compose-plugin
+  %{if is_k8s_template}
+  # kubernetes
+  - kubeadm
+  - kubelet
+  # iSCSI client for CSI-backed database storage
+  - open-iscsi
+  %{endif}
+
+apt:
+  sources:
+  %{if is_k8s_template}
+    kubernetes:
+      source: "deb https://pkgs.k8s.io/core:/stable:/v1.34/deb/ /"
+      keyid: "DE15B14486CD377B9E876E1A234654DA9A296436"
+      filename: kubernetes.list
+  %{endif}
+    docker:
+      source: "deb https://download.docker.com/linux/ubuntu noble stable"
+      keyid: "9DC858229FC7DD38854AE2D88D81803C0EBFCD88"
+      filename: docker.list
+
+%{if is_k8s_template}
+# Setup script is base64-encoded by the module so YAML whitespace
+# handling never touches the heredoc bodies inside it. Replaces an
+# earlier `indent(6, …)` approach that put `[plugins.*]` TOML
+# sections at col 6 inside `cat >> /etc/containerd/config.toml`
+# heredocs — containerd refused to parse the result and the node5 v1
+# boot failed there (2026-05-26). Source: modules/create-template-vm/k8s-node-containerd-setup.sh
+write_files:
+  - path: /usr/local/bin/k8s-node-containerd-setup.sh
+    permissions: '0755'
+    owner: root:root
+    encoding: b64
+    content: ${k8s_node_setup_script_b64}
+  - path: /usr/local/bin/k8s-node-post-join-tune.sh
+    permissions: '0755'
+    owner: root:root
+    encoding: b64
+    content: ${k8s_node_post_join_script_b64}
+%{endif}
+
+runcmd:
+  # Enable weekly TRIM/discard to reclaim freed blocks in LVM thin pool
+  - systemctl enable --now fstrim.timer
+  # Enable persistent journald logging for crash forensics, with size limits to reduce disk wear
+  - mkdir -p /var/log/journal
+  - sed -i 's/#Storage=auto/Storage=persistent/' /etc/systemd/journald.conf
+  - sed -i 's/#SystemMaxUse=/SystemMaxUse=500M/' /etc/systemd/journald.conf
+  - sed -i 's/#MaxRetentionSec=/MaxRetentionSec=7day/' /etc/systemd/journald.conf
+  - sed -i 's/#MaxFileSec=/MaxFileSec=1day/' /etc/systemd/journald.conf
+  - sed -i 's/#Compress=yes/Compress=yes/' /etc/systemd/journald.conf
+  - systemctl restart systemd-journald
+  %{if is_k8s_template}
+  # systemd-resolved global DNS fallback. Without this, only the
+  # link-level DNS from Proxmox's `qm set --nameserver` (Technitium,
+  # 10.0.20.201) is consulted — and Technitium returns NXDOMAIN for
+  # forgejo.viktorbarzin.me, so kubelet image pulls from the Forgejo
+  # registry break. Public DNS upstream + Technitium fallback matches
+  # the pre-existing manual setup on k8s-node1..4.
+  - mkdir -p /etc/systemd/resolved.conf.d
+  - |
+    cat > /etc/systemd/resolved.conf.d/global-dns.conf <<'EOF'
+    [Resolve]
+    DNS=8.8.8.8 1.1.1.1
+    FallbackDNS=10.0.20.201
+    EOF
+  - systemctl restart systemd-resolved
+  # Re-enabled 2026-05-10: unattended-upgrades is back on, but with a tight
+  # Allowed-Origins list, a Package-Blacklist for k8s/containerd/runc/calico,
+  # and Automatic-Reboot disabled (kured + sentinel-gate handles reboots in a
+  # 24h-soaked rolling window, gated by Prometheus alerts).
+  # Original outage (March 2026) was kernel update → containerd overlayfs corruption.
+  # Mitigations: 24h cool-down between node reboots, Prometheus halt-on-alert,
+  # apt-mark hold on k8s components, Package-Blacklist for runtime components.
+  - apt-get install -y unattended-upgrades update-notifier-common
+  - |
+    cat > /etc/apt/apt.conf.d/52unattended-upgrades-k8s <<'EOF'
+    Unattended-Upgrade::Allowed-Origins {
+        "$${distro_id}:$${distro_codename}";
+        "$${distro_id}:$${distro_codename}-security";
+        "$${distro_id}:$${distro_codename}-updates";
+        "$${distro_id}ESMApps:$${distro_codename}-apps-security";
+        "$${distro_id}ESM:$${distro_codename}-infra-security";
+    };
+    Unattended-Upgrade::Package-Blacklist {
+        "^containerd(\.io)?$$";
+        "^runc$$";
+        "^cri-tools$$";
+        "^kubernetes-cni$$";
+        "^calico-.*";
+        "^cni-plugins-.*";
+        "^docker-ce$$";
+    };
+    Unattended-Upgrade::DevRelease "false";
+    Unattended-Upgrade::Automatic-Reboot "false";
+    EOF
+  - |
+    cat > /etc/apt/apt.conf.d/20auto-upgrades <<'EOF'
+    APT::Periodic::Update-Package-Lists "1";
+    APT::Periodic::Unattended-Upgrade "1";
+    EOF
+  - systemctl unmask unattended-upgrades 2>/dev/null || true
+  - systemctl enable --now unattended-upgrades
+  - apt-mark hold kubelet kubeadm kubectl
+  - apt-mark hold containerd containerd.io runc 2>/dev/null || true
+  - systemctl stop kubelet
+  - containerd config default | sudo tee /etc/containerd/config.toml
+  # The containerd/kubelet setup is delivered as /usr/local/bin/k8s-node-containerd-setup.sh
+  # via the write_files: block at the top of this file. We run it as a single
+  # bash invocation here so cloud-init only sees a one-line runcmd item.
+  # (Previous inline `- $${containerd_config_update_command}` broke YAML parsing
+  # because the heredoc contains mixed-indent inner shell heredocs.)
+  - bash /usr/local/bin/k8s-node-containerd-setup.sh
+  - systemctl restart containerd
+  - systemctl enable --now iscsid
+  # Harden iSCSI: increase recovery timeout (300s vs 120s default) and enable
+  # CRC32C data/header digests to detect bit flips over the network.
+  # Prevents SQLite corruption from transient iSCSI session drops.
+  - sed -i 's/^node.session.timeo.replacement_timeout = .*/node.session.timeo.replacement_timeout = 300/' /etc/iscsi/iscsid.conf
+  - sed -i 's/^node.conn\[0\].timeo.noop_out_interval = .*/node.conn[0].timeo.noop_out_interval = 10/' /etc/iscsi/iscsid.conf
+  - sed -i 's/^node.conn\[0\].timeo.noop_out_timeout = .*/node.conn[0].timeo.noop_out_timeout = 15/' /etc/iscsi/iscsid.conf
+  - |
+    if ! grep -q '^node.conn\[0\].iscsi.HeaderDigest' /etc/iscsi/iscsid.conf; then
+      echo 'node.conn[0].iscsi.HeaderDigest = CRC32C,None' >> /etc/iscsi/iscsid.conf
+      echo 'node.conn[0].iscsi.DataDigest = CRC32C,None' >> /etc/iscsi/iscsid.conf
+    fi
+  - systemctl restart iscsid
+  # Create /sentinel directory for kured reboot gating (sentinel gate DaemonSet)
+  - mkdir -p /sentinel
+  # Disable swap — kubelet defaults to failSwapOn=true and won't start otherwise.
+  # (Previously this snippet created a 4G swapfile for "memory pressure relief"
+  # but never set failSwapOn=false / memorySwap.swapBehavior together, so the
+  # join consistently bricked kubelet — observed on node6 boot v3 2026-05-26.)
+  - swapoff -a
+  - sed -i '/ swap / s/^/#/' /etc/fstab
+  - ${k8s_join_command}
+  - systemctl enable kubelet
+  - systemctl start kubelet
+  # Kubelet tuning runs AFTER kubeadm join — that's when
+  # /var/lib/kubelet/config.yaml gets written. Restarts kubelet at the
+  # end to pick up the patched config.
+  - bash /usr/local/bin/k8s-node-post-join-tune.sh
+  %{ endif }
+  %{ for provision_cmd in provision_cmds ~}
+  - ${provision_cmd}
+  %{ endfor ~}
--- a/modules/create-template-vm/k8s-node-containerd-setup.sh
+++ b/modules/create-template-vm/k8s-node-containerd-setup.sh
@ -0,0 +1,147 @@
+#!/usr/bin/env bash
+#
+# K8s node containerd + kubelet bootstrap. Runs once via cloud-init runcmd.
+# Embedded into the cloud-init snippet base64-encoded by main.tf so YAML
+# whitespace handling never touches the heredoc bodies — TOML / Python
+# blocks below land in /etc/containerd/config.toml etc. with their leading
+# whitespace intact.
+#
+# Layout:
+#   1. /etc/containerd/config.toml — config_path + mirror dirs + GC tuning
+#   2. /etc/containerd/certs.d/*/hosts.toml — per-registry mirror configs
+#   3. /var/lib/kubelet/config.yaml — eviction + shutdown grace + log rotation
+#   4. /etc/systemd/logind.conf.d + kubelet.service.d — graceful shutdown
+#   5. (master-only) /etc/kubernetes/manifests — apiserver + controller flags
+set -euo pipefail
+
+# 1. config_path — match BOTH quote styles. containerd v1 writes `""`,
+# containerd v2.x writes `''`. Without the v2 match, hosts.toml mirror
+# config is silently ignored — observed 2026-05-26 on k8s-node4
+# (containerd v2.2.4) and reproduced on k8s-node5 v1 boot.
+sed -i "s|config_path = \"\"|config_path = \"/etc/containerd/certs.d\"|g" /etc/containerd/config.toml
+sed -i "s|config_path = ''|config_path = \"/etc/containerd/certs.d\"|g" /etc/containerd/config.toml
+
+# 2. Per-registry hosts.toml — pull-through caches on docker-registry VM
+# (10.0.20.10) for high-traffic registries, Traefik LB (10.0.20.200) for
+# forgejo. Low-traffic registries (registry.k8s.io, reg.kyverno.io) skip
+# the cache and pull direct because past pull-through cache attempts
+# truncated downloads and broke VPA certgen + Kyverno image pulls.
+
+mkdir -p /etc/containerd/certs.d/docker.io
+cat > /etc/containerd/certs.d/docker.io/hosts.toml <<'DOCKERIO'
+server = "https://registry-1.docker.io"
+
+[host."http://10.0.20.10:5000"]
+  capabilities = ["pull", "resolve"]
+
+[host."https://registry-1.docker.io"]
+  capabilities = ["pull", "resolve"]
+DOCKERIO
+
+mkdir -p /etc/containerd/certs.d/ghcr.io
+cat > /etc/containerd/certs.d/ghcr.io/hosts.toml <<'GHCR'
+server = "https://ghcr.io"
+
+[host."http://10.0.20.10:5010"]
+  capabilities = ["pull", "resolve"]
+
+[host."https://ghcr.io"]
+  capabilities = ["pull", "resolve"]
+GHCR
+
+# Forgejo OCI registry: prefer in-cluster Traefik LB (10.0.20.200) to
+# avoid hairpin NAT. Traefik serves the *.viktorbarzin.me wildcard so
+# SNI verification succeeds. If the mirror is unreachable, fall back to
+# public DNS resolution (needs the global DNS fallback set up below).
+mkdir -p /etc/containerd/certs.d/forgejo.viktorbarzin.me
+cat > /etc/containerd/certs.d/forgejo.viktorbarzin.me/hosts.toml <<'FORGEJO'
+server = "https://forgejo.viktorbarzin.me"
+
+[host."https://10.0.20.203"]
+  capabilities = ["pull", "resolve"]
+  skip_verify = true
+FORGEJO
+
+# quay.io + registry.k8s.io: include mirror configs that match node4's
+# layout (no real pull-through cache today, server line is the direct
+# upstream). Keeping these present makes the per-node config uniform and
+# lets us flip a cache on later by editing only the [host."..."] block.
+mkdir -p /etc/containerd/certs.d/quay.io
+cat > /etc/containerd/certs.d/quay.io/hosts.toml <<'QUAY'
+server = "https://quay.io"
+
+[host."http://10.0.20.10:5020"]
+  capabilities = ["pull", "resolve"]
+QUAY
+
+mkdir -p /etc/containerd/certs.d/registry.k8s.io
+cat > /etc/containerd/certs.d/registry.k8s.io/hosts.toml <<'K8SREG'
+server = "https://registry.k8s.io"
+
+[host."http://10.0.20.10:5030"]
+  capabilities = ["pull", "resolve"]
+K8SREG
+
+# 3. containerd tuning: parallel pulls + selective GC overrides.
+# containerd v2's `config default` ALREADY emits `[plugins.'io.containerd.gc.v1.scheduler']`,
+# `[plugins.'io.containerd.runtime.v2.task']`, and `[plugins.'io.containerd.metadata.v1.bolt']`
+# sections — declaring them again fails with `toml: table … already exists`
+# (observed on node6 boot 2026-05-26). Patch values in place instead.
+sed -i 's/.*max_concurrent_downloads = 3/max_concurrent_downloads = 20/g' /etc/containerd/config.toml
+# pause_threshold: 0.5 → 0.02 (run GC more aggressively when images dirty %)
+sed -i "s/^[[:space:]]*pause_threshold = .*/  pause_threshold = 0.02/" /etc/containerd/config.toml
+# schedule_delay: 0s/1ms → 30 min (longer cool-down between GC runs)
+sed -i "s/^[[:space:]]*schedule_delay = .*/  schedule_delay = '1800s'/" /etc/containerd/config.toml
+# exit_timeout: 0s → 5m (more aggressive container cleanup)
+sed -i "s/^[[:space:]]*exit_timeout = .*/  exit_timeout = '5m'/" /etc/containerd/config.toml
+
+# 4. (kubelet tuning intentionally NOT here — /var/lib/kubelet/config.yaml
+# only exists AFTER kubeadm join. That work runs in
+# k8s-node-post-join-tune.sh, invoked as a separate cloud-init runcmd
+# step after the join completes.)
+
+# 5. logind + kubelet systemd unit — total kubelet shutdown 310s, so
+# logind InhibitDelay > that and kubelet TimeoutStopSec > that.
+mkdir -p /etc/systemd/logind.conf.d
+cat > /etc/systemd/logind.conf.d/kubelet-shutdown.conf <<'LOGIND_CONF'
+[Login]
+InhibitDelayMaxSec=480
+LOGIND_CONF
+systemctl restart systemd-logind
+
+mkdir -p /etc/systemd/system/kubelet.service.d
+cat > /etc/systemd/system/kubelet.service.d/20-shutdown.conf <<'KUBELET_SHUTDOWN'
+[Service]
+TimeoutStopSec=420s
+KUBELET_SHUTDOWN
+systemctl daemon-reload
+
+# 6. (master-only) faster pod eviction + attach-detach reconcile.
+if [ -f /etc/kubernetes/manifests/kube-controller-manager.yaml ]; then
+    python3 - <<'CM_PATCH'
+import yaml
+with open('/etc/kubernetes/manifests/kube-controller-manager.yaml') as f:
+    m = yaml.safe_load(f)
+args = m['spec']['containers'][0]['command']
+for flag in ['--attach-detach-reconcile-sync-period=15s']:
+    key = flag.split('=')[0]
+    args = [a for a in args if not a.startswith(key)]
+    args.append(flag)
+m['spec']['containers'][0]['command'] = args
+with open('/etc/kubernetes/manifests/kube-controller-manager.yaml', 'w') as f:
+    yaml.dump(m, f, default_flow_style=False)
+CM_PATCH
+    python3 - <<'AS_PATCH'
+import yaml
+with open('/etc/kubernetes/manifests/kube-apiserver.yaml') as f:
+    m = yaml.safe_load(f)
+args = m['spec']['containers'][0]['command']
+for flag in ['--default-unreachable-toleration-seconds=60', '--default-not-ready-toleration-seconds=60']:
+    key = flag.split('=')[0]
+    args = [a for a in args if not a.startswith(key)]
+    args.append(flag)
+m['spec']['containers'][0]['command'] = args
+with open('/etc/kubernetes/manifests/kube-apiserver.yaml', 'w') as f:
+    yaml.dump(m, f, default_flow_style=False)
+AS_PATCH
+fi
--- a/modules/create-template-vm/k8s-node-post-join-tune.sh
+++ b/modules/create-template-vm/k8s-node-post-join-tune.sh
@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+#
+# Runs AFTER `kubeadm join` has written /var/lib/kubelet/config.yaml.
+# Patches kubelet config in place (parallel image pulls, eviction
+# thresholds, priority-based shutdown grace, container log rotation)
+# and (on master) tightens controller-manager / apiserver flags.
+#
+# Embedded into the cloud-init snippet base64-encoded by main.tf so
+# YAML whitespace doesn't touch the heredoc bodies inside.
+set -euo pipefail
+
+if [ ! -f /var/lib/kubelet/config.yaml ]; then
+    echo "post-join-tune: /var/lib/kubelet/config.yaml not found — was kubeadm join run?" >&2
+    exit 1
+fi
+
+# Parallel image pulls.
+sed -i '/serializeImagePulls:/d' /var/lib/kubelet/config.yaml
+sed -i '/maxParallelImagePulls:/d' /var/lib/kubelet/config.yaml
+printf 'serializeImagePulls: false\nmaxParallelImagePulls: 50\n' >> /var/lib/kubelet/config.yaml
+
+# Memory / disk eviction. Aggressive disk thresholds (15%/20%)
+# prevent the 2026-03-13 containerd image-store corruption that took
+# down k8s-node2.
+sed -i '/systemReserved:/d; /kubeReserved:/d; /evictionHard:/,/^[^ ]/{ /evictionHard:/d; /^  /d }; /evictionSoft:/,/^[^ ]/{ /evictionSoft:/d; /^  /d }; /evictionSoftGracePeriod:/,/^[^ ]/{ /evictionSoftGracePeriod:/d; /^  /d }' /var/lib/kubelet/config.yaml
+
+cat >> /var/lib/kubelet/config.yaml <<'KUBELET_PATCH'
+systemReserved:
+  memory: "512Mi"
+  cpu: "200m"
+kubeReserved:
+  memory: "512Mi"
+  cpu: "200m"
+evictionHard:
+  memory.available: "500Mi"
+  nodefs.available: "15%"
+  imagefs.available: "20%"
+evictionSoft:
+  memory.available: "1Gi"
+  nodefs.available: "20%"
+  imagefs.available: "25%"
+evictionSoftGracePeriod:
+  memory.available: "30s"
+  nodefs.available: "60s"
+  imagefs.available: "30s"
+memorySwap:
+  swapBehavior: "LimitedSwap"
+KUBELET_PATCH
+
+# Container log rotation + priority-based shutdown grace.
+sed -i '/^shutdownGracePeriod:/d; /^shutdownGracePeriodCriticalPods:/d' /var/lib/kubelet/config.yaml
+python3 - <<'KUBELET_FINAL'
+import yaml
+with open('/var/lib/kubelet/config.yaml') as f:
+    cfg = yaml.safe_load(f)
+cfg.pop('shutdownGracePeriod', None)
+cfg.pop('shutdownGracePeriodCriticalPods', None)
+cfg.pop('shutdownGracePeriodByPodPriority', None)
+cfg['containerLogMaxSize'] = '10Mi'
+cfg['containerLogMaxFiles'] = 3
+cfg['shutdownGracePeriodByPodPriority'] = [
+    {'priority': 0,          'shutdownGracePeriodSeconds': 20},
+    {'priority': 200000,     'shutdownGracePeriodSeconds': 20},
+    {'priority': 400000,     'shutdownGracePeriodSeconds': 30},
+    {'priority': 600000,     'shutdownGracePeriodSeconds': 30},
+    {'priority': 800000,     'shutdownGracePeriodSeconds': 90},
+    {'priority': 1000000,    'shutdownGracePeriodSeconds': 30},
+    {'priority': 1200000,    'shutdownGracePeriodSeconds': 30},
+    {'priority': 2000000000, 'shutdownGracePeriodSeconds': 30},
+    {'priority': 2000001000, 'shutdownGracePeriodSeconds': 30},
+]
+with open('/var/lib/kubelet/config.yaml', 'w') as f:
+    yaml.dump(cfg, f, default_flow_style=False)
+KUBELET_FINAL
+
+# Reload kubelet to pick up new config (it's already started by the
+# preceding cloud-init runcmd line — restart, not start).
+systemctl restart kubelet
--- a/modules/create-template-vm/main.tf
+++ b/modules/create-template-vm/main.tf
@ -0,0 +1,104 @@
+variable "proxmox_host" { type = string }
+variable "proxmox_user" { type = string }
+variable "cloud_image_url" { type = string }
+variable "image_path" { type = string }
+variable "template_id" {
+  type    = number
+  default = 8000
+}
+variable "template_name" { type = string }
+variable "snippet_name" { type = string }
+variable "user_passwd" { type = string } # hashed pw
+variable "k8s_join_command" {
+  type    = string
+  default = ""
+}
+variable "containerd_config_update_command" {
+  type        = string
+  default     = ""
+  description = "DEPRECATED: was inlined into write_files via indent(); the heredoc-TOML interaction broke containerd config parsing on node5 v1 boot 2026-05-26. The k8s setup script is now bundled inside the module at k8s-node-containerd-setup.sh — pass nothing here. Kept to avoid breaking stacks that still reference it; ignored when is_k8s_template=true."
+}
+variable "is_k8s_template" { type = bool }
+variable "ssh_private_key" {
+  type    = string
+  default = ""
+}
+variable "ssh_public_key" {
+  type    = string
+  default = ""
+}
+variable "provision_cmds" {
+  type    = list(string)
+  default = []
+}
+
+# SSH connection to Proxmox
+resource "null_resource" "create_template_remote" {
+  connection {
+    type        = "ssh"
+    user        = var.proxmox_user
+    host        = var.proxmox_host
+    private_key = var.ssh_private_key
+  }
+
+  # Commands executed *on Proxmox host*
+  provisioner "remote-exec" {
+    inline = [
+      "set -e",
+      # download the cloud image if missing
+      "if [ ! -f ${var.image_path} ]; then wget -O ${var.image_path} ${var.cloud_image_url}; fi",
+      # create template only if not existing
+      "if ! qm status ${var.template_id} >/dev/null 2>&1; then",
+      "  echo 'Creating cloud-init template...';",
+      "  qm create ${var.template_id} --name ${var.template_name} --memory 8192 --cores 8 --net0 virtio,bridge=vmbr0;",
+      "  qm importdisk ${var.template_id} ${var.image_path} local-lvm;",
+      "  qm set ${var.template_id} --scsihw virtio-scsi-pci --scsi0 local-lvm:vm-${var.template_id}-disk-0;",
+      "  qm set ${var.template_id} --ide2 local-lvm:cloudinit;",
+      "  qm set ${var.template_id} --boot c --bootdisk scsi0;",
+      "  qm set ${var.template_id} --serial0 socket --vga serial0;",
+      "  qm template ${var.template_id};",
+      "else",
+      "  echo 'Template ${var.template_id} already exists — skipping.';",
+      "fi"
+    ]
+  }
+}
+
+resource "null_resource" "upload_cloud_init" {
+  connection {
+    type        = "ssh"
+    host        = var.proxmox_host
+    user        = var.proxmox_user
+    private_key = var.ssh_private_key
+  }
+
+  provisioner "remote-exec" {
+    inline = ["mkdir -p /var/lib/vz/snippets"]
+  }
+
+  provisioner "file" {
+    destination = "/var/lib/vz/snippets/${var.snippet_name}"
+    content = templatefile("${path.module}/cloud_init.yaml", {
+      is_k8s_template                = var.is_k8s_template,
+      authorized_ssh_key             = var.ssh_public_key,
+      passwd                         = var.user_passwd,
+      provision_cmds                 = var.provision_cmds,
+      k8s_join_command               = var.k8s_join_command,
+      k8s_node_setup_script_b64      = var.is_k8s_template ? base64encode(file("${path.module}/k8s-node-containerd-setup.sh")) : ""
+      k8s_node_post_join_script_b64  = var.is_k8s_template ? base64encode(file("${path.module}/k8s-node-post-join-tune.sh")) : ""
+      }
+    )
+  }
+
+  # Force recreate when the below changes
+  triggers = {
+    file_hash                   = filesha256("${path.module}/cloud_init.yaml")
+    setup_script_hash           = var.is_k8s_template ? filesha256("${path.module}/k8s-node-containerd-setup.sh") : ""
+    post_join_script_hash       = var.is_k8s_template ? filesha256("${path.module}/k8s-node-post-join-tune.sh") : ""
+    provision_cmds              = join(", ", var.provision_cmds)
+    is_k8s_template             = var.is_k8s_template,
+    passwd                      = var.user_passwd,
+    k8s_join_command            = var.k8s_join_command,
+    ssh_public_key              = var.ssh_public_key,
+  }
+}