diff --git a/stacks/dbaas/modules/dbaas/main.tf b/stacks/dbaas/modules/dbaas/main.tf index a0157235..61fb21ac 100644 --- a/stacks/dbaas/modules/dbaas/main.tf +++ b/stacks/dbaas/modules/dbaas/main.tf @@ -156,7 +156,7 @@ resource "helm_release" "mysql_cluster" { } datadirVolumeClaimTemplate = { - storageClassName = "iscsi-truenas" + storageClassName = "proxmox-lvm" resources = { requests = { storage = "30Gi" @@ -189,7 +189,7 @@ resource "helm_release" "mysql_cluster" { memory = "2Gi" } limits = { - memory = "5Gi" + memory = "4Gi" } } @@ -225,7 +225,7 @@ resource "helm_release" "mysql_cluster" { cpu = "250m" } limits = { - memory = "5Gi" + memory = "4Gi" } } }] @@ -876,7 +876,7 @@ resource "null_resource" "pg_cluster" { instances = "2" image = "ghcr.io/cloudnative-pg/postgis:16" storage_size = "20Gi" - storage_class = "iscsi-truenas" + storage_class = "proxmox-lvm" memory_limit = "512Mi" } @@ -899,7 +899,7 @@ resource "null_resource" "pg_cluster" { enableSuperuserAccess: true storage: size: 20Gi - storageClass: iscsi-truenas + storageClass: proxmox-lvm resources: requests: cpu: "50m" diff --git a/stacks/monitoring/modules/monitoring/prometheus.tf b/stacks/monitoring/modules/monitoring/prometheus.tf index 0f99cabc..7b12ff31 100644 --- a/stacks/monitoring/modules/monitoring/prometheus.tf +++ b/stacks/monitoring/modules/monitoring/prometheus.tf @@ -2,13 +2,13 @@ resource "kubernetes_persistent_volume_claim" "prometheus_server_pvc" { metadata { - name = "prometheus-data" + name = "prometheus-data-proxmox" namespace = kubernetes_namespace.monitoring.metadata[0].name } spec { access_modes = ["ReadWriteOnce"] - storage_class_name = "iscsi-truenas" + storage_class_name = "proxmox-lvm" resources { requests = { storage = "200Gi" diff --git a/stacks/monitoring/modules/monitoring/prometheus_chart_values.tpl b/stacks/monitoring/modules/monitoring/prometheus_chart_values.tpl index 219984df..a014be95 100755 --- a/stacks/monitoring/modules/monitoring/prometheus_chart_values.tpl +++ b/stacks/monitoring/modules/monitoring/prometheus_chart_values.tpl @@ -155,7 +155,7 @@ server: - "storage.tsdb.wal-compression" persistentVolume: # enabled: false - existingClaim: prometheus-data + existingClaim: prometheus-data-proxmox # storageClass: rook-cephfs retention: "52w" # NOTE: Memory must be >= 4Gi. The WAL tmpfs (2Gi, medium: Memory) shares diff --git a/stacks/nextcloud/main.tf b/stacks/nextcloud/main.tf index 7c38b684..4349865d 100644 --- a/stacks/nextcloud/main.tf +++ b/stacks/nextcloud/main.tf @@ -186,12 +186,12 @@ resource "kubernetes_config_map" "apache_tuning" { resource "kubernetes_persistent_volume_claim" "nextcloud_data_iscsi" { metadata { - name = "nextcloud-data-iscsi" + name = "nextcloud-data-proxmox" namespace = kubernetes_namespace.nextcloud.metadata[0].name } spec { access_modes = ["ReadWriteOnce"] - storage_class_name = "iscsi-truenas" + storage_class_name = "proxmox-lvm" resources { requests = { storage = "20Gi" diff --git a/stacks/novelapp/main.tf b/stacks/novelapp/main.tf index b9b7fb51..2a65b64f 100644 --- a/stacks/novelapp/main.tf +++ b/stacks/novelapp/main.tf @@ -48,12 +48,12 @@ module "tls_secret" { resource "kubernetes_persistent_volume_claim" "novelapp-data" { metadata { - name = "novelapp-data" + name = "novelapp-data-proxmox" namespace = kubernetes_namespace.novelapp.metadata[0].name } spec { access_modes = ["ReadWriteOnce"] - storage_class_name = "iscsi-truenas" + storage_class_name = "proxmox-lvm" resources { requests = { storage = "1Gi" diff --git a/stacks/platform/modules/monitoring/prometheus.tf b/stacks/platform/modules/monitoring/prometheus.tf index 4b8283eb..5aad77a1 100644 --- a/stacks/platform/modules/monitoring/prometheus.tf +++ b/stacks/platform/modules/monitoring/prometheus.tf @@ -2,13 +2,13 @@ resource "kubernetes_persistent_volume_claim" "prometheus_server_pvc" { metadata { - name = "prometheus-data" + name = "prometheus-data-proxmox" namespace = kubernetes_namespace.monitoring.metadata[0].name } spec { access_modes = ["ReadWriteOnce"] - storage_class_name = "iscsi-truenas" + storage_class_name = "proxmox-lvm" resources { requests = { storage = "200Gi" diff --git a/stacks/platform/modules/monitoring/prometheus_chart_values.tpl b/stacks/platform/modules/monitoring/prometheus_chart_values.tpl index 8d5a63a3..174882f1 100755 --- a/stacks/platform/modules/monitoring/prometheus_chart_values.tpl +++ b/stacks/platform/modules/monitoring/prometheus_chart_values.tpl @@ -155,7 +155,7 @@ server: - "storage.tsdb.wal-compression" persistentVolume: # enabled: false - existingClaim: prometheus-data + existingClaim: prometheus-data-proxmox # storageClass: rook-cephfs retention: "52w" resources: diff --git a/stacks/plotting-book/main.tf b/stacks/plotting-book/main.tf index 7b19a886..b7cfb4b6 100644 --- a/stacks/plotting-book/main.tf +++ b/stacks/plotting-book/main.tf @@ -48,12 +48,12 @@ module "tls_secret" { resource "kubernetes_persistent_volume_claim" "plotting-book-data" { metadata { - name = "plotting-book-data" + name = "plotting-book-data-proxmox" namespace = kubernetes_namespace.plotting-book.metadata[0].name } spec { access_modes = ["ReadWriteOnce"] - storage_class_name = "iscsi-truenas" + storage_class_name = "proxmox-lvm" resources { requests = { storage = "1Gi" diff --git a/stacks/proxmox-csi/main.tf b/stacks/proxmox-csi/main.tf new file mode 100644 index 00000000..b8d7f516 --- /dev/null +++ b/stacks/proxmox-csi/main.tf @@ -0,0 +1,16 @@ +variable "nfs_server" { type = string } + +data "vault_kv_secret_v2" "secrets" { + mount = "secret" + name = "viktor" +} + +module "proxmox-csi" { + source = "./modules/proxmox-csi" + tier = local.tiers.cluster + proxmox_url = "https://192.168.1.127:8006/api2/json" + proxmox_token_id = data.vault_kv_secret_v2.secrets.data["proxmox_csi_token_id"] + proxmox_token_secret = data.vault_kv_secret_v2.secrets.data["proxmox_csi_token_secret"] + proxmox_cluster_name = "pve" + kube_config_path = var.kube_config_path +} diff --git a/stacks/proxmox-csi/modules/proxmox-csi/main.tf b/stacks/proxmox-csi/modules/proxmox-csi/main.tf new file mode 100644 index 00000000..bf96c40d --- /dev/null +++ b/stacks/proxmox-csi/modules/proxmox-csi/main.tf @@ -0,0 +1,91 @@ +resource "kubernetes_namespace" "proxmox_csi" { + metadata { + name = "proxmox-csi" + labels = { + tier = var.tier + "resource-governance/custom-quota" = "true" + } + } +} + +resource "helm_release" "proxmox_csi" { + namespace = kubernetes_namespace.proxmox_csi.metadata[0].name + create_namespace = false + name = "proxmox-csi-plugin" + atomic = true + timeout = 300 + + repository = "oci://ghcr.io/sergelogvinov/charts" + chart = "proxmox-csi-plugin" + + values = [yamlencode({ + config = { + clusters = [{ + url = var.proxmox_url + insecure = true + token_id = var.proxmox_token_id + token_secret = var.proxmox_token_secret + region = var.proxmox_cluster_name + }] + } + + # StorageClass for block volumes on existing HDD thin pool + storageClass = [{ + name = "proxmox-lvm" + storage = "local-lvm" + reclaimPolicy = "Retain" + fstype = "ext4" + ssd = false + cache = "none" + volumeBindingMode = "WaitForFirstConsumer" + allowVolumeExpansion = true + }] + + controller = { + replicas = 2 + resources = { + requests = { cpu = "10m", memory = "32Mi" } + limits = { memory = "64Mi" } + } + } + + node = { + resources = { + requests = { cpu = "10m", memory = "32Mi" } + limits = { memory = "64Mi" } + } + } + })] +} + +# Topology labels on K8s nodes — required for Proxmox CSI to map nodes to Proxmox VMs. +# region = Proxmox cluster name, zone = Proxmox node name (where the VM runs). +# All our VMs run on the single Proxmox node "pve". +locals { + k8s_nodes = { + "k8s-master" = { vmid = 200, proxmox_node = "pve" } + "k8s-node1" = { vmid = 201, proxmox_node = "pve" } + "k8s-node2" = { vmid = 202, proxmox_node = "pve" } + "k8s-node3" = { vmid = 203, proxmox_node = "pve" } + "k8s-node4" = { vmid = 204, proxmox_node = "pve" } + } +} + +resource "null_resource" "node_labels" { + for_each = local.k8s_nodes + + provisioner "local-exec" { + command = <<-EOT + kubectl --kubeconfig=${var.kube_config_path} label node ${each.key} \ + topology.kubernetes.io/region=${var.proxmox_cluster_name} \ + topology.kubernetes.io/zone=${each.value.proxmox_node} \ + node.csi.proxmox.sinextra.dev/name=${each.key} \ + --overwrite + EOT + } + + triggers = { + region = var.proxmox_cluster_name + zone = each.value.proxmox_node + } +} diff --git a/stacks/proxmox-csi/modules/proxmox-csi/variables.tf b/stacks/proxmox-csi/modules/proxmox-csi/variables.tf new file mode 100644 index 00000000..06efa478 --- /dev/null +++ b/stacks/proxmox-csi/modules/proxmox-csi/variables.tf @@ -0,0 +1,18 @@ +variable "tier" { type = string } +variable "proxmox_url" { type = string } +variable "proxmox_token_id" { + type = string + sensitive = true +} +variable "proxmox_token_secret" { + type = string + sensitive = true +} +variable "proxmox_cluster_name" { + type = string + default = "pve" +} +variable "kube_config_path" { + type = string + default = "" +} diff --git a/stacks/proxmox-csi/terragrunt.hcl b/stacks/proxmox-csi/terragrunt.hcl new file mode 100644 index 00000000..4f16dddf --- /dev/null +++ b/stacks/proxmox-csi/terragrunt.hcl @@ -0,0 +1,8 @@ +include "root" { + path = find_in_parent_folders() +} + +dependency "infra" { + config_path = "../infra" + skip_outputs = true +} diff --git a/stacks/redis/modules/redis/main.tf b/stacks/redis/modules/redis/main.tf index 2450eda7..557b1eb3 100644 --- a/stacks/redis/modules/redis/main.tf +++ b/stacks/redis/modules/redis/main.tf @@ -59,7 +59,7 @@ resource "helm_release" "redis" { master = { persistence = { enabled = true - storageClass = "iscsi-truenas" + storageClass = "proxmox-lvm" size = "2Gi" } @@ -79,7 +79,7 @@ resource "helm_release" "redis" { persistence = { enabled = true - storageClass = "iscsi-truenas" + storageClass = "proxmox-lvm" size = "2Gi" } diff --git a/stacks/vaultwarden/modules/vaultwarden/main.tf b/stacks/vaultwarden/modules/vaultwarden/main.tf index 03c49fe8..cc9c2199 100644 --- a/stacks/vaultwarden/modules/vaultwarden/main.tf +++ b/stacks/vaultwarden/modules/vaultwarden/main.tf @@ -22,12 +22,12 @@ module "tls_secret" { resource "kubernetes_persistent_volume_claim" "vaultwarden_data" { metadata { - name = "vaultwarden-data-iscsi" + name = "vaultwarden-data-proxmox" namespace = kubernetes_namespace.vaultwarden.metadata[0].name } spec { access_modes = ["ReadWriteOnce"] - storage_class_name = "iscsi-truenas" + storage_class_name = "proxmox-lvm" resources { requests = { storage = "1Gi"