infra/stacks/proxmox-csi/modules/proxmox-csi/main.tf

resource "kubernetes_namespace" "proxmox_csi" {
  metadata {
    name = "proxmox-csi"
    labels = {
      tier                               = var.tier
      "resource-governance/custom-quota" = "true"
      "keel.sh/enrolled" = "true"
    }
  }
  lifecycle {
    # KYVERNO_LIFECYCLE_V1: goldilocks-vpa-auto-mode ClusterPolicy stamps this label on every namespace
    ignore_changes = [metadata[0].labels["goldilocks.fairwinds.com/vpa-update-mode"]]
  }
}

resource "helm_release" "proxmox_csi" {
  namespace        = kubernetes_namespace.proxmox_csi.metadata[0].name
  create_namespace = false
  name             = "proxmox-csi-plugin"
  atomic           = true
  timeout          = 300

  repository = "oci://ghcr.io/sergelogvinov/charts"
  chart      = "proxmox-csi-plugin"
  version    = "0.5.6"

  values = [yamlencode({
    config = {
      clusters = [{
        url          = var.proxmox_url
        insecure     = true
        token_id     = var.proxmox_token_id
        token_secret = var.proxmox_token_secret
        region       = var.proxmox_cluster_name
      }]
    }

    # StorageClass for block volumes on existing HDD thin pool.
    # `resize.topolvm.io/enabled=true` opts the SC into pvc-autoresizer:
    # without it, the controller filters this SC out and never patches its
    # PVCs no matter how full they get. Required alongside the per-PVC
    # threshold/increase/storage_limit annotations.
    storageClass = [
      {
        name                 = "proxmox-lvm"
        storage              = "local-lvm"
        reclaimPolicy        = "Retain"
        fstype               = "ext4"
        ssd                  = false
        cache                = "none"
        volumeBindingMode    = "WaitForFirstConsumer"
        allowVolumeExpansion = true
        annotations = {
          "resize.topolvm.io/enabled" = "true"
        }
      },
      {
        name                 = "proxmox-lvm-encrypted"
        storage              = "local-lvm"
        reclaimPolicy        = "Retain"
        fstype               = "ext4"
        ssd                  = false
        cache                = "none"
        volumeBindingMode    = "WaitForFirstConsumer"
        allowVolumeExpansion = true
        annotations = {
          "resize.topolvm.io/enabled" = "true"
        }
        extraParameters = {
          "csi.storage.k8s.io/node-stage-secret-name"       = "proxmox-csi-encryption"
          "csi.storage.k8s.io/node-stage-secret-namespace"  = "kube-system"
          "csi.storage.k8s.io/node-expand-secret-name"      = "proxmox-csi-encryption"
          "csi.storage.k8s.io/node-expand-secret-namespace" = "kube-system"
        }
      },
    ]

    controller = {
      replicas = 2
      resources = {
        requests = { cpu = "10m", memory = "32Mi" }
        limits   = { memory = "64Mi" }
      }
    }

    # LUKS2 Argon2id key derivation needs ~1GiB memory (memory id=712).
    # Request bumped from 64Mi → 1024Mi (2026-05-23) so the pod is reserved
    # for the unlock burst instead of risking OOM under node pressure.
    node = {
      plugin = {
        resources = {
          requests = { cpu = "10m", memory = "1024Mi" }
          limits   = { memory = "1280Mi" }
        }
      }
    }
  })]
}

# Topology labels on K8s nodes — required for Proxmox CSI to map nodes to Proxmox VMs.
# region = Proxmox cluster name, zone = Proxmox node name (where the VM runs).
# All our VMs run on the single Proxmox node "pve".
locals {
  k8s_nodes = {
    "k8s-master" = { vmid = 200, proxmox_node = "pve" }
    "k8s-node1"  = { vmid = 201, proxmox_node = "pve" }
    "k8s-node2"  = { vmid = 202, proxmox_node = "pve" }
    "k8s-node3"  = { vmid = 203, proxmox_node = "pve" }
    "k8s-node4"  = { vmid = 204, proxmox_node = "pve" }
    "k8s-node5"  = { vmid = 205, proxmox_node = "pve" }
    "k8s-node6"  = { vmid = 206, proxmox_node = "pve" }
  }
}

resource "null_resource" "node_labels" {
  for_each = local.k8s_nodes

  # max-volume-attachments: capped at 28 (4 below plugin's hard ceiling of 30,
  # see VolumesPerNodeHardLimit in sergelogvinov/proxmox-csi-plugin pkg/csi/node.go).
  # Default is 24; bumping to 28 gives ~4-PVC headroom per node while keeping
  # 2 slots for recovery (boot disk + transient attach during reschedule).
  # Without this label the plugin reports 24 and node1 cascades through that
  # ceiling during evictions — see post-mortem 2026-05-25.
  provisioner "local-exec" {
    command = <<-EOT
      kubectl --kubeconfig=${var.kube_config_path} label node ${each.key} \
        topology.kubernetes.io/region=${var.proxmox_cluster_name} \
        topology.kubernetes.io/zone=${each.value.proxmox_node} \
        node.csi.proxmox.sinextra.dev/name=${each.key} \
        csi.proxmox.sinextra.dev/max-volume-attachments=28 \
        --overwrite
    EOT
  }

  triggers = {
    region      = var.proxmox_cluster_name
    zone        = each.value.proxmox_node
    max_volumes = "28"
  }
}

# --- RBAC for PVE host snapshot restore script ---
# Provides kubectl access from the Proxmox host for the lvm-pvc-snapshot restore subcommand.
# Minimal permissions: read PVs/PVCs/Pods, scale Deployments/StatefulSets.

resource "kubernetes_service_account" "pve_snapshot_admin" {
  metadata {
    name      = "pve-snapshot-admin"
    namespace = "kube-system"
  }
}

resource "kubernetes_secret" "pve_snapshot_admin_token" {
  metadata {
    name      = "pve-snapshot-admin-token"
    namespace = "kube-system"
    annotations = {
      "kubernetes.io/service-account.name" = kubernetes_service_account.pve_snapshot_admin.metadata[0].name
    }
  }
  type = "kubernetes.io/service-account-token"
}

resource "kubernetes_cluster_role" "pve_snapshot_admin" {
  metadata {
    name = "pve-snapshot-admin"
  }

  rule {
    api_groups = [""]
    resources  = ["persistentvolumes", "persistentvolumeclaims", "pods"]
    verbs      = ["get", "list"]
  }

  rule {
    api_groups = ["apps"]
    resources  = ["deployments", "statefulsets", "replicasets"]
    verbs      = ["get", "list", "update", "patch"]
  }

  rule {
    api_groups = ["apps"]
    resources  = ["deployments/scale", "statefulsets/scale"]
    verbs      = ["get", "update", "patch"]
  }
}

resource "kubernetes_cluster_role_binding" "pve_snapshot_admin" {
  metadata {
    name = "pve-snapshot-admin"
  }

  role_ref {
    api_group = "rbac.authorization.k8s.io"
    kind      = "ClusterRole"
    name      = kubernetes_cluster_role.pve_snapshot_admin.metadata[0].name
  }

  subject {
    kind      = "ServiceAccount"
    name      = kubernetes_service_account.pve_snapshot_admin.metadata[0].name
    namespace = "kube-system"
  }
}

# --- ExternalSecret for LUKS encryption passphrase ---
# Creates K8s Secret "proxmox-csi-encryption" in kube-system from Vault KV.
# Referenced by the proxmox-lvm-encrypted StorageClass for node-stage and node-expand.
resource "kubernetes_manifest" "external_secret_encryption" {
  manifest = {
    apiVersion = "external-secrets.io/v1beta1"
    kind       = "ExternalSecret"
    metadata = {
      name      = "proxmox-csi-encryption"
      namespace = "kube-system"
    }
    spec = {
      refreshInterval = "1h"
      secretStoreRef = {
        kind = "ClusterSecretStore"
        name = "vault-kv"
      }
      target = {
        name           = "proxmox-csi-encryption"
        creationPolicy = "Owner"
        deletionPolicy = "Retain"
      }
      data = [{
        secretKey = "encryption-passphrase"
        remoteRef = {
          key      = "viktor"
          property = "proxmox_csi_encryption_passphrase"
        }
      }]
    }
  }
}