Goal: re-clone the worker template, boot, and have it appear as `kubectl get nodes …Ready` with no manual steps. Adds `scripts/provision-k8s-worker NAME VMID IP` and rebuilds the cloud-init pipeline that was failing five distinct ways on a clean boot. Bugs fixed (all hit during the k8s-node5 + k8s-node6 builds today): 1. `indent(6, containerd_config_update_command)` indented the bodies of `cat >> /etc/containerd/config.toml <<'CONTAINERD_GC'` heredocs, so [plugins.*] TOML sections landed in /etc/containerd/config.toml at col 6 — containerd refused to parse them. Source is now a normal .sh file (`modules/create-template-vm/k8s-node-containerd-setup.sh`) base64-embedded into `write_files`; YAML whitespace never touches the heredoc bodies. 2. The same script tried to `cat >> /etc/containerd/config.toml` `[plugins."io.containerd.gc.v1.scheduler"]` etc., which containerd v2.2.4's `config default` ALREADY emits. Result: `toml: table … already exists`. Patched with sed-in-place overrides instead. 3. Kubelet tuning (sed against /var/lib/kubelet/config.yaml) ran from the containerd setup script — BEFORE `kubeadm join` writes that file. Sed aborted with "No such file or directory", `set -e` killed the script, post-script cloud-init steps kept going (cloud-init doesn't stop on runcmd failure). Split into a dedicated `k8s-node-post-join-tune.sh` invoked AFTER kubeadm join. 4. cloud_init.yaml fallocate'd a 4G swapfile and `swapon`'d it BEFORE kubeadm join. kubelet defaults to failSwapOn=true → exited 1 immediately. Replaced the swap setup with `swapoff -a` (node4 already runs this way and the cluster is fine). 5. Without `hostname:` in the shared user-data snippet, Proxmox's auto-generated meta-data does NOT include local-hostname when `cicustom user=…` is set — so cloud-init falls back to the cloud image's default `ubuntu` and `kubeadm join` registers the wrong node name. `provision-k8s-worker` now writes a per-node `<NAME>-meta.yaml` snippet and passes both via `cicustom user=…,meta=…`. Other improvements rolled in while fixing the above: - `ssh_public_key` read from Vault (`secret/viktor.ssh_public_key`, added today) instead of `var.ssh_public_key`. The last `terragrunt apply` was run with that var empty, leaving the snippet's `ssh_authorized_keys` with a single blank entry; the wizard user was effectively locked out of every fresh node. - `cloud_init.yaml` adds `/etc/systemd/resolved.conf.d/global-dns.conf` with `DNS=8.8.8.8 1.1.1.1, FallbackDNS=10.0.20.201`. Without it, systemd-resolved only consulted Technitium (link-level), which returns NXDOMAIN for `forgejo.viktorbarzin.me` — kubelet pulls from the Forgejo registry then failed DNS until I patched it manually on node5. - k8s apt repo bumped v1.32 → v1.34 (matches cluster). - The containerd setup script now creates hosts.toml for forgejo, quay, registry.k8s.io in addition to docker.io + ghcr.io. node3/4 had these added by hand post-bootstrap; now they're baked in. - `config_path` sed matches both `""` (containerd v1) and `''` (containerd v2.x). Without the v2 match, the certs.d mirror dir was silently ignored. - `proxmox-csi` node map adds k8s-node5 + k8s-node6 entries so CSI topology labels (region/zone, max-volume-attachments=28) apply on next `tg apply`. - `stacks/infra/main.tf` shed the 160-line inline containerd setup heredoc — that whole thing now lives in the module as a .sh file. Known unsolved gaps (deferred): - iscsid restart hangs ~90s on first boot before SIGKILL releases it (systemd-resolved restart kicks iscsid via dependency). Adds wall- clock time but doesn't block the join. - `provision-k8s-worker` doesn't run `tg apply` on `proxmox-csi` afterward, so the CSI topology labels need a manual apply after the node joins. Solving cleanly needs the CSI map to derive from `kubectl get nodes` instead of a static local — separate work. - `var.containerd_config_update_command` is now ignored when is_k8s_template=true (replaced by the bundled .sh file). Variable kept with a deprecation note to avoid breaking other call sites. E2E proof: k8s-node6 (VMID 206) boots hands-off from `provision-k8s-worker k8s-node6 206 10.0.20.106` and appears as `kubectl get nodes …Ready` ~7 min later (most of which is the apt package_upgrade — separate optimization). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
104 lines
4 KiB
HCL
104 lines
4 KiB
HCL
variable "proxmox_host" { type = string }
|
|
variable "proxmox_user" { type = string }
|
|
variable "cloud_image_url" { type = string }
|
|
variable "image_path" { type = string }
|
|
variable "template_id" {
|
|
type = number
|
|
default = 8000
|
|
}
|
|
variable "template_name" { type = string }
|
|
variable "snippet_name" { type = string }
|
|
variable "user_passwd" { type = string } # hashed pw
|
|
variable "k8s_join_command" {
|
|
type = string
|
|
default = ""
|
|
}
|
|
variable "containerd_config_update_command" {
|
|
type = string
|
|
default = ""
|
|
description = "DEPRECATED: was inlined into write_files via indent(); the heredoc-TOML interaction broke containerd config parsing on node5 v1 boot 2026-05-26. The k8s setup script is now bundled inside the module at k8s-node-containerd-setup.sh — pass nothing here. Kept to avoid breaking stacks that still reference it; ignored when is_k8s_template=true."
|
|
}
|
|
variable "is_k8s_template" { type = bool }
|
|
variable "ssh_private_key" {
|
|
type = string
|
|
default = ""
|
|
}
|
|
variable "ssh_public_key" {
|
|
type = string
|
|
default = ""
|
|
}
|
|
variable "provision_cmds" {
|
|
type = list(string)
|
|
default = []
|
|
}
|
|
|
|
# SSH connection to Proxmox
|
|
resource "null_resource" "create_template_remote" {
|
|
connection {
|
|
type = "ssh"
|
|
user = var.proxmox_user
|
|
host = var.proxmox_host
|
|
private_key = var.ssh_private_key
|
|
}
|
|
|
|
# Commands executed *on Proxmox host*
|
|
provisioner "remote-exec" {
|
|
inline = [
|
|
"set -e",
|
|
# download the cloud image if missing
|
|
"if [ ! -f ${var.image_path} ]; then wget -O ${var.image_path} ${var.cloud_image_url}; fi",
|
|
# create template only if not existing
|
|
"if ! qm status ${var.template_id} >/dev/null 2>&1; then",
|
|
" echo 'Creating cloud-init template...';",
|
|
" qm create ${var.template_id} --name ${var.template_name} --memory 8192 --cores 8 --net0 virtio,bridge=vmbr0;",
|
|
" qm importdisk ${var.template_id} ${var.image_path} local-lvm;",
|
|
" qm set ${var.template_id} --scsihw virtio-scsi-pci --scsi0 local-lvm:vm-${var.template_id}-disk-0;",
|
|
" qm set ${var.template_id} --ide2 local-lvm:cloudinit;",
|
|
" qm set ${var.template_id} --boot c --bootdisk scsi0;",
|
|
" qm set ${var.template_id} --serial0 socket --vga serial0;",
|
|
" qm template ${var.template_id};",
|
|
"else",
|
|
" echo 'Template ${var.template_id} already exists — skipping.';",
|
|
"fi"
|
|
]
|
|
}
|
|
}
|
|
|
|
resource "null_resource" "upload_cloud_init" {
|
|
connection {
|
|
type = "ssh"
|
|
host = var.proxmox_host
|
|
user = var.proxmox_user
|
|
private_key = var.ssh_private_key
|
|
}
|
|
|
|
provisioner "remote-exec" {
|
|
inline = ["mkdir -p /var/lib/vz/snippets"]
|
|
}
|
|
|
|
provisioner "file" {
|
|
destination = "/var/lib/vz/snippets/${var.snippet_name}"
|
|
content = templatefile("${path.module}/cloud_init.yaml", {
|
|
is_k8s_template = var.is_k8s_template,
|
|
authorized_ssh_key = var.ssh_public_key,
|
|
passwd = var.user_passwd,
|
|
provision_cmds = var.provision_cmds,
|
|
k8s_join_command = var.k8s_join_command,
|
|
k8s_node_setup_script_b64 = var.is_k8s_template ? base64encode(file("${path.module}/k8s-node-containerd-setup.sh")) : ""
|
|
k8s_node_post_join_script_b64 = var.is_k8s_template ? base64encode(file("${path.module}/k8s-node-post-join-tune.sh")) : ""
|
|
}
|
|
)
|
|
}
|
|
|
|
# Force recreate when the below changes
|
|
triggers = {
|
|
file_hash = filesha256("${path.module}/cloud_init.yaml")
|
|
setup_script_hash = var.is_k8s_template ? filesha256("${path.module}/k8s-node-containerd-setup.sh") : ""
|
|
post_join_script_hash = var.is_k8s_template ? filesha256("${path.module}/k8s-node-post-join-tune.sh") : ""
|
|
provision_cmds = join(", ", var.provision_cmds)
|
|
is_k8s_template = var.is_k8s_template,
|
|
passwd = var.user_passwd,
|
|
k8s_join_command = var.k8s_join_command,
|
|
ssh_public_key = var.ssh_public_key,
|
|
}
|
|
}
|