diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 2d0a717a..f9144bc7 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -73,22 +73,3 @@ provider "registry.terraform.io/hashicorp/random" { "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a", ] } - -provider "registry.terraform.io/hashicorp/vsphere" { - version = "2.0.2" - constraints = "2.0.2" - hashes = [ - "h1:/9qlE9Ni7mhl3cTHLhmYg1AFuyBnXqie53Q5Ujor6oU=", - "zh:0b72856d2a89b118adf64dcfc75a0b0f7d8875e9872c637a9a0549d4a3dd9383", - "zh:2cf50f17464feeac60039f4d36656835895260798d4d9167a1ff2d4e1e464dce", - "zh:39d5f1c99bd4024d88b95892711f234afeb2b83700cdbd3df68a60c6b3bb0ac7", - "zh:551b68f91f5eefa4daee4d826d27d2bc10236e0642fdafd066604884b2a06816", - "zh:8454b5e7dbbf0300bbab1ecd73c12511cf35c0c493ad18e31742d58798e11026", - "zh:84a8233a60240e0020de513668a20d015a7fbc760ad6ab639d52e6ddabfa3376", - "zh:95574b7f092e77f9c629770b802d1635903f14d8d88d3828a96f58ca072876a5", - "zh:a262c779c112e4d6ac47152d6c6303308180dfd4d5929b980cf24707ca9218e9", - "zh:c7622cfa8bab6a6f122c0ab3d4767ecf0f52ccf32d64020aa4e9581f91da5586", - "zh:e01cfc79d21c7151c97047afdc79dfe7bf37c0063908829200480a03359e4c2e", - "zh:f64c3f283c31477886c3d16592426dbefa2cab55f2885cb3351cef094ad898a0", - ] -} diff --git a/modules/kubernetes/main.tf b/modules/kubernetes/main.tf index 200b7dbb..a03beb4d 100644 --- a/modules/kubernetes/main.tf +++ b/modules/kubernetes/main.tf @@ -340,6 +340,6 @@ module "immich" { postgresql_password = var.immich_postgresql_password } -# module "nginx-ingress" { -# source = "./nginx-ingress" -# } +module "nginx-ingress" { + source = "./nginx-ingress" +} diff --git a/modules/kubernetes/nginx-ingress/main.tf b/modules/kubernetes/nginx-ingress/main.tf new file mode 100644 index 00000000..df1bb9ae --- /dev/null +++ b/modules/kubernetes/nginx-ingress/main.tf @@ -0,0 +1,674 @@ +# module "nginx-controller" { +# source = "terraform-iaac/nginx-controller/helm" +# namespace = "ingress-nginx-test" +# create_namespace = true +# atomic = true +# ingress_class_is_default = false +# ingress_class_name = "nginx-test" +# } +resource "kubernetes_namespace" "ingress_nginx" { + metadata { + name = "ingress-nginx" + labels = { + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + } + } +} +resource "kubernetes_service_account" "ingress_nginx" { + metadata { + name = "ingress-nginx" + namespace = "ingress-nginx" + labels = { + "app.kubernetes.io/component" = "controller" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + "app.kubernetes.io/part-of" = "ingress-nginx" + "app.kubernetes.io/version" = "1.8.2" + } + } + automount_service_account_token = true +} + +# Jobs create a cert and modify this secret. This is problematic as TF recreates it every time +# Instead, on each fresh install, uncomment this, get nginx working and comment it. +# Also rm from state: tf state rm module.kubernetes_cluster.module.nginx-ingress.kubernetes_service_account.ingress_nginx_admission +# resource "kubernetes_service_account" "ingress_nginx_admission" { +# metadata { +# name = "ingress-nginx-admission" +# namespace = "ingress-nginx" +# labels = { +# "app.kubernetes.io/component" = "admission-webhook" +# "app.kubernetes.io/instance" = "ingress-nginx" +# "app.kubernetes.io/name" = "ingress-nginx" +# "app.kubernetes.io/part-of" = "ingress-nginx" +# "app.kubernetes.io/version" = "1.8.2" +# } +# } +# } +resource "kubernetes_role" "ingress_nginx" { + metadata { + name = "ingress-nginx" + namespace = "ingress-nginx" + labels = { + "app.kubernetes.io/component" = "controller" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + "app.kubernetes.io/part-of" = "ingress-nginx" + "app.kubernetes.io/version" = "1.8.2" + } + } + rule { + verbs = ["get"] + api_groups = [""] + resources = ["namespaces"] + } + rule { + verbs = ["get", "list", "watch", "update", "create", "delete"] + api_groups = [""] + resources = ["configmaps", "pods", "secrets", "endpoints"] + } + rule { + verbs = ["get", "list", "watch"] + api_groups = [""] + resources = ["services"] + } + rule { + verbs = ["get", "list", "watch"] + api_groups = ["networking.k8s.io"] + resources = ["ingresses"] + } + rule { + verbs = ["update"] + api_groups = ["networking.k8s.io"] + resources = ["ingresses/status"] + } + rule { + verbs = ["get", "list", "watch"] + api_groups = ["networking.k8s.io"] + resources = ["ingressclasses"] + } + rule { + verbs = ["get", "update"] + api_groups = ["coordination.k8s.io"] + resources = ["leases"] + resource_names = ["ingress-nginx-leader"] + } + rule { + verbs = ["create"] + api_groups = ["coordination.k8s.io"] + resources = ["leases"] + } + rule { + verbs = ["create", "patch"] + api_groups = [""] + resources = ["events"] + } + rule { + verbs = ["list", "watch", "get"] + api_groups = ["discovery.k8s.io"] + resources = ["endpointslices"] + } +} +resource "kubernetes_role" "ingress_nginx_admission" { + metadata { + name = "ingress-nginx-admission" + namespace = "ingress-nginx" + labels = { + "app.kubernetes.io/component" = "admission-webhook" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + "app.kubernetes.io/part-of" = "ingress-nginx" + "app.kubernetes.io/version" = "1.8.2" + } + } + rule { + verbs = ["get", "create", "patch", "update", "watch", "list"] + api_groups = [""] + resources = ["secrets"] + } +} +resource "kubernetes_cluster_role" "ingress_nginx" { + metadata { + name = "ingress-nginx" + labels = { + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + "app.kubernetes.io/part-of" = "ingress-nginx" + "app.kubernetes.io/version" = "1.8.2" + } + } + rule { + verbs = ["list", "watch"] + api_groups = [""] + resources = ["configmaps", "endpoints", "nodes", "pods", "secrets", "namespaces"] + } + rule { + verbs = ["list", "watch"] + api_groups = ["coordination.k8s.io"] + resources = ["leases"] + } + rule { + verbs = ["get"] + api_groups = [""] + resources = ["nodes"] + } + rule { + verbs = ["get", "list", "watch"] + api_groups = [""] + resources = ["services"] + } + rule { + verbs = ["get", "list", "watch"] + api_groups = ["networking.k8s.io"] + resources = ["ingresses"] + } + rule { + verbs = ["create", "patch"] + api_groups = [""] + resources = ["events"] + } + rule { + verbs = ["update"] + api_groups = ["networking.k8s.io"] + resources = ["ingresses/status"] + } + rule { + verbs = ["get", "list", "watch"] + api_groups = ["networking.k8s.io"] + resources = ["ingressclasses"] + } + rule { + verbs = ["list", "watch", "get"] + api_groups = ["discovery.k8s.io"] + resources = ["endpointslices"] + } +} +resource "kubernetes_cluster_role" "ingress_nginx_admission" { + metadata { + name = "ingress-nginx-admission" + labels = { + "app.kubernetes.io/component" = "admission-webhook" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + "app.kubernetes.io/part-of" = "ingress-nginx" + "app.kubernetes.io/version" = "1.8.2" + } + } + rule { + verbs = ["get", "update"] + api_groups = ["admissionregistration.k8s.io"] + resources = ["validatingwebhookconfigurations"] + } +} +resource "kubernetes_role_binding" "ingress_nginx" { + metadata { + name = "ingress-nginx" + namespace = "ingress-nginx" + labels = { + "app.kubernetes.io/component" = "controller" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + "app.kubernetes.io/part-of" = "ingress-nginx" + "app.kubernetes.io/version" = "1.8.2" + } + } + subject { + kind = "ServiceAccount" + name = "ingress-nginx" + namespace = "ingress-nginx" + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "Role" + name = "ingress-nginx" + } +} +resource "kubernetes_role_binding" "ingress_nginx_admission" { + metadata { + name = "ingress-nginx-admission" + namespace = "ingress-nginx" + labels = { + "app.kubernetes.io/component" = "admission-webhook" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + "app.kubernetes.io/part-of" = "ingress-nginx" + "app.kubernetes.io/version" = "1.8.2" + } + } + subject { + kind = "ServiceAccount" + name = "ingress-nginx-admission" + namespace = "ingress-nginx" + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "Role" + name = "ingress-nginx-admission" + } +} +resource "kubernetes_cluster_role_binding" "ingress_nginx" { + metadata { + name = "ingress-nginx" + labels = { + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + "app.kubernetes.io/part-of" = "ingress-nginx" + "app.kubernetes.io/version" = "1.8.2" + } + } + subject { + kind = "ServiceAccount" + name = "ingress-nginx" + namespace = "ingress-nginx" + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "ingress-nginx" + } +} +resource "kubernetes_cluster_role_binding" "ingress_nginx_admission" { + metadata { + name = "ingress-nginx-admission" + labels = { + "app.kubernetes.io/component" = "admission-webhook" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + "app.kubernetes.io/part-of" = "ingress-nginx" + "app.kubernetes.io/version" = "1.8.2" + } + } + subject { + kind = "ServiceAccount" + name = "ingress-nginx-admission" + namespace = "ingress-nginx" + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "ingress-nginx-admission" + } +} +resource "kubernetes_config_map" "ingress_nginx_controller" { + metadata { + name = "ingress-nginx-controller" + namespace = "ingress-nginx" + labels = { + "app.kubernetes.io/component" = "controller" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + "app.kubernetes.io/part-of" = "ingress-nginx" + "app.kubernetes.io/version" = "1.8.2" + } + } + data = { + allow-snippet-annotations = "true" + } +} +resource "kubernetes_service" "ingress_nginx_controller" { + metadata { + name = "ingress-nginx-controller" + namespace = "ingress-nginx" + labels = { + "app.kubernetes.io/component" = "controller" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + "app.kubernetes.io/part-of" = "ingress-nginx" + "app.kubernetes.io/version" = "1.8.2" + } + } + spec { + port { + name = "http" + protocol = "TCP" + port = 80 + target_port = "http" + } + port { + name = "https" + protocol = "TCP" + port = 443 + target_port = "https" + } + selector = { + "app.kubernetes.io/component" = "controller" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + } + type = "LoadBalancer" + # ip_families = ["IPv4"] + } +} +resource "kubernetes_service" "ingress_nginx_controller_admission" { + metadata { + name = "ingress-nginx-controller-admission" + namespace = "ingress-nginx" + labels = { + "app.kubernetes.io/component" = "controller" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + "app.kubernetes.io/part-of" = "ingress-nginx" + "app.kubernetes.io/version" = "1.8.2" + } + } + spec { + port { + name = "https-webhook" + port = 443 + target_port = "webhook" + } + selector = { + "app.kubernetes.io/component" = "controller" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + } + type = "ClusterIP" + } +} +resource "kubernetes_deployment" "ingress_nginx_controller" { + metadata { + name = "ingress-nginx-controller" + namespace = "ingress-nginx" + labels = { + "app.kubernetes.io/component" = "controller" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + "app.kubernetes.io/part-of" = "ingress-nginx" + "app.kubernetes.io/version" = "1.8.2" + } + } + spec { + selector { + match_labels = { + "app.kubernetes.io/component" = "controller" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + } + } + template { + metadata { + labels = { + "app.kubernetes.io/component" = "controller" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + "app.kubernetes.io/part-of" = "ingress-nginx" + "app.kubernetes.io/version" = "1.8.2" + } + } + spec { + volume { + name = "webhook-cert" + secret { + secret_name = "ingress-nginx-admission" + } + } + container { + name = "controller" + image = "registry.k8s.io/ingress-nginx/controller:v1.8.2@sha256:74834d3d25b336b62cabeb8bf7f1d788706e2cf1cfd64022de4137ade8881ff2" + args = ["/nginx-ingress-controller", "--election-id=ingress-nginx-leader", "--controller-class=k8s.io/ingress-nginx", "--ingress-class=nginx", "--configmap=$(POD_NAMESPACE)/ingress-nginx-controller", "--validating-webhook=:8443", "--validating-webhook-certificate=/usr/local/certificates/cert", "--validating-webhook-key=/usr/local/certificates/key"] + port { + name = "http" + container_port = 80 + protocol = "TCP" + } + port { + name = "https" + container_port = 443 + protocol = "TCP" + } + port { + name = "webhook" + container_port = 8443 + protocol = "TCP" + } + env { + name = "POD_NAME" + value_from { + field_ref { + field_path = "metadata.name" + } + } + } + env { + name = "POD_NAMESPACE" + value_from { + field_ref { + field_path = "metadata.namespace" + } + } + } + env { + name = "LD_PRELOAD" + value = "/usr/local/lib/libmimalloc.so" + } + resources { + requests = { + cpu = "100m" + memory = "90Mi" + } + } + volume_mount { + name = "webhook-cert" + read_only = true + mount_path = "/usr/local/certificates/" + } + liveness_probe { + http_get { + path = "/healthz" + port = "10254" + scheme = "HTTP" + } + initial_delay_seconds = 10 + timeout_seconds = 1 + period_seconds = 10 + success_threshold = 1 + failure_threshold = 5 + } + readiness_probe { + http_get { + path = "/healthz" + port = "10254" + scheme = "HTTP" + } + initial_delay_seconds = 10 + timeout_seconds = 1 + period_seconds = 10 + success_threshold = 1 + failure_threshold = 3 + } + lifecycle { + pre_stop { + exec { + command = ["/wait-shutdown"] + } + } + } + image_pull_policy = "IfNotPresent" + security_context { + capabilities { + add = ["NET_BIND_SERVICE"] + drop = ["ALL"] + } + run_as_user = 101 + allow_privilege_escalation = true + } + } + termination_grace_period_seconds = 300 + dns_policy = "ClusterFirst" + node_selector = { + "kubernetes.io/os" = "linux" + } + service_account_name = "ingress-nginx" + } + } + strategy { + type = "RollingUpdate" + rolling_update { + max_unavailable = "1" + } + } + revision_history_limit = 10 + } +} +resource "kubernetes_job" "ingress_nginx_admission_create" { + metadata { + name = "ingress-nginx-admission-create" + namespace = "ingress-nginx" + labels = { + "app.kubernetes.io/component" = "admission-webhook" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + "app.kubernetes.io/part-of" = "ingress-nginx" + "app.kubernetes.io/version" = "1.8.2" + } + } + spec { + template { + metadata { + name = "ingress-nginx-admission-create" + labels = { + "app.kubernetes.io/component" = "admission-webhook" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + "app.kubernetes.io/part-of" = "ingress-nginx" + "app.kubernetes.io/version" = "1.8.2" + } + } + spec { + container { + name = "create" + image = "registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20230407@sha256:543c40fd093964bc9ab509d3e791f9989963021f1e9e4c9c7b6700b02bfb227b" + args = ["create", "--host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc", "--namespace=$(POD_NAMESPACE)", "--secret-name=ingress-nginx-admission"] + env { + name = "POD_NAMESPACE" + value_from { + field_ref { + field_path = "metadata.namespace" + } + } + } + image_pull_policy = "IfNotPresent" + } + restart_policy = "OnFailure" + node_selector = { + "kubernetes.io/os" = "linux" + } + service_account_name = "ingress-nginx-admission" + security_context { + run_as_user = 2000 + run_as_non_root = true + fs_group = 2000 + } + } + } + } +} + +# Jobs create a cert and modify this secret. This is problematic as TF recreates it every time +# Instead, on each fresh install, uncomment this, get nginx working and comment it. +# Also rm from state: tf state rm module.kubernetes_cluster.module.nginx-ingress.kubernetes_job.ingress_nginx_admission_patch +# resource "kubernetes_job" "ingress_nginx_admission_patch" { +# metadata { +# name = "ingress-nginx-admission-patch" +# namespace = "ingress-nginx" +# labels = { +# "app.kubernetes.io/component" = "admission-webhook" +# "app.kubernetes.io/instance" = "ingress-nginx" +# "app.kubernetes.io/name" = "ingress-nginx" +# "app.kubernetes.io/part-of" = "ingress-nginx" +# "app.kubernetes.io/version" = "1.8.2" +# } +# } +# spec { +# template { +# metadata { +# name = "ingress-nginx-admission-patch" +# labels = { +# "app.kubernetes.io/component" = "admission-webhook" +# "app.kubernetes.io/instance" = "ingress-nginx" +# "app.kubernetes.io/name" = "ingress-nginx" +# "app.kubernetes.io/part-of" = "ingress-nginx" +# "app.kubernetes.io/version" = "1.8.2" +# } +# } +# spec { +# container { +# name = "patch" +# image = "registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20230407@sha256:543c40fd093964bc9ab509d3e791f9989963021f1e9e4c9c7b6700b02bfb227b" +# args = ["patch", "--webhook-name=ingress-nginx-admission", "--namespace=$(POD_NAMESPACE)", "--patch-mutating=false", "--secret-name=ingress-nginx-admission", "--patch-failure-policy=Fail"] +# env { +# name = "POD_NAMESPACE" +# value_from { +# field_ref { +# field_path = "metadata.namespace" +# } +# } +# } +# image_pull_policy = "IfNotPresent" +# } +# restart_policy = "OnFailure" +# node_selector = { +# "kubernetes.io/os" = "linux" +# } +# service_account_name = "ingress-nginx-admission" +# security_context { +# run_as_user = 2000 +# run_as_non_root = true +# fs_group = 2000 +# } +# } +# } +# } +# } +resource "kubernetes_ingress_class" "nginx" { + metadata { + name = "nginx" + labels = { + "app.kubernetes.io/component" = "controller" + "app.kubernetes.io/instance" = "ingress-nginx" + "app.kubernetes.io/name" = "ingress-nginx" + "app.kubernetes.io/part-of" = "ingress-nginx" + "app.kubernetes.io/version" = "1.8.2" + } + } + spec { + controller = "k8s.io/ingress-nginx" + } +} + +# Jobs create a cert and modify this secret. This is problematic as TF recreates it every time +# Instead, on each fresh install, uncomment this, get nginx working and comment it. +# Also rm from state: tf state rm module.kubernetes_cluster.module.nginx-ingress.kubernetes_service_account.ingress_nginx_admission +# resource "kubernetes_validating_webhook_configuration" "ingress_nginx_admission" { +# metadata { +# name = "ingress-nginx-admission" +# labels = { +# "app.kubernetes.io/component" = "admission-webhook" +# "app.kubernetes.io/instance" = "ingress-nginx" +# "app.kubernetes.io/name" = "ingress-nginx" +# "app.kubernetes.io/part-of" = "ingress-nginx" +# "app.kubernetes.io/version" = "1.8.2" +# } +# } +# webhook { +# name = "validate.nginx.ingress.kubernetes.io" +# client_config { +# service { +# namespace = "ingress-nginx" +# name = "ingress-nginx-controller-admission" +# path = "/networking/v1/ingresses" +# } +# } +# rule { +# api_versions = ["v1"] +# api_groups = ["networking.k8s.io"] +# resources = ["ingresses"] +# operations = ["CREATE", "UPDATE"] +# } +# failure_policy = "Fail" +# match_policy = "Equivalent" +# side_effects = "None" +# admission_review_versions = ["v1"] +# } +# } diff --git a/terraform.tfstate b/terraform.tfstate index f5c37665..f76d13d0 100644 Binary files a/terraform.tfstate and b/terraform.tfstate differ diff --git a/terraform.tfvars b/terraform.tfvars index f03cb012..2a27b3dc 100644 Binary files a/terraform.tfvars and b/terraform.tfvars differ