add nginx ingress as tf module [ci skip]

This commit is contained in:
Viktor Barzin 2023-11-18 16:18:26 +00:00
parent 54fea4724b
commit dc565c4d87
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
5 changed files with 677 additions and 22 deletions

19
.terraform.lock.hcl generated
View file

@ -73,22 +73,3 @@ provider "registry.terraform.io/hashicorp/random" {
"zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a",
]
}
provider "registry.terraform.io/hashicorp/vsphere" {
version = "2.0.2"
constraints = "2.0.2"
hashes = [
"h1:/9qlE9Ni7mhl3cTHLhmYg1AFuyBnXqie53Q5Ujor6oU=",
"zh:0b72856d2a89b118adf64dcfc75a0b0f7d8875e9872c637a9a0549d4a3dd9383",
"zh:2cf50f17464feeac60039f4d36656835895260798d4d9167a1ff2d4e1e464dce",
"zh:39d5f1c99bd4024d88b95892711f234afeb2b83700cdbd3df68a60c6b3bb0ac7",
"zh:551b68f91f5eefa4daee4d826d27d2bc10236e0642fdafd066604884b2a06816",
"zh:8454b5e7dbbf0300bbab1ecd73c12511cf35c0c493ad18e31742d58798e11026",
"zh:84a8233a60240e0020de513668a20d015a7fbc760ad6ab639d52e6ddabfa3376",
"zh:95574b7f092e77f9c629770b802d1635903f14d8d88d3828a96f58ca072876a5",
"zh:a262c779c112e4d6ac47152d6c6303308180dfd4d5929b980cf24707ca9218e9",
"zh:c7622cfa8bab6a6f122c0ab3d4767ecf0f52ccf32d64020aa4e9581f91da5586",
"zh:e01cfc79d21c7151c97047afdc79dfe7bf37c0063908829200480a03359e4c2e",
"zh:f64c3f283c31477886c3d16592426dbefa2cab55f2885cb3351cef094ad898a0",
]
}

View file

@ -340,6 +340,6 @@ module "immich" {
postgresql_password = var.immich_postgresql_password
}
# module "nginx-ingress" {
# source = "./nginx-ingress"
# }
module "nginx-ingress" {
source = "./nginx-ingress"
}

View file

@ -0,0 +1,674 @@
# module "nginx-controller" {
# source = "terraform-iaac/nginx-controller/helm"
# namespace = "ingress-nginx-test"
# create_namespace = true
# atomic = true
# ingress_class_is_default = false
# ingress_class_name = "nginx-test"
# }
resource "kubernetes_namespace" "ingress_nginx" {
metadata {
name = "ingress-nginx"
labels = {
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
}
}
}
resource "kubernetes_service_account" "ingress_nginx" {
metadata {
name = "ingress-nginx"
namespace = "ingress-nginx"
labels = {
"app.kubernetes.io/component" = "controller"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
"app.kubernetes.io/part-of" = "ingress-nginx"
"app.kubernetes.io/version" = "1.8.2"
}
}
automount_service_account_token = true
}
# Jobs create a cert and modify this secret. This is problematic as TF recreates it every time
# Instead, on each fresh install, uncomment this, get nginx working and comment it.
# Also rm from state: tf state rm module.kubernetes_cluster.module.nginx-ingress.kubernetes_service_account.ingress_nginx_admission
# resource "kubernetes_service_account" "ingress_nginx_admission" {
# metadata {
# name = "ingress-nginx-admission"
# namespace = "ingress-nginx"
# labels = {
# "app.kubernetes.io/component" = "admission-webhook"
# "app.kubernetes.io/instance" = "ingress-nginx"
# "app.kubernetes.io/name" = "ingress-nginx"
# "app.kubernetes.io/part-of" = "ingress-nginx"
# "app.kubernetes.io/version" = "1.8.2"
# }
# }
# }
resource "kubernetes_role" "ingress_nginx" {
metadata {
name = "ingress-nginx"
namespace = "ingress-nginx"
labels = {
"app.kubernetes.io/component" = "controller"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
"app.kubernetes.io/part-of" = "ingress-nginx"
"app.kubernetes.io/version" = "1.8.2"
}
}
rule {
verbs = ["get"]
api_groups = [""]
resources = ["namespaces"]
}
rule {
verbs = ["get", "list", "watch", "update", "create", "delete"]
api_groups = [""]
resources = ["configmaps", "pods", "secrets", "endpoints"]
}
rule {
verbs = ["get", "list", "watch"]
api_groups = [""]
resources = ["services"]
}
rule {
verbs = ["get", "list", "watch"]
api_groups = ["networking.k8s.io"]
resources = ["ingresses"]
}
rule {
verbs = ["update"]
api_groups = ["networking.k8s.io"]
resources = ["ingresses/status"]
}
rule {
verbs = ["get", "list", "watch"]
api_groups = ["networking.k8s.io"]
resources = ["ingressclasses"]
}
rule {
verbs = ["get", "update"]
api_groups = ["coordination.k8s.io"]
resources = ["leases"]
resource_names = ["ingress-nginx-leader"]
}
rule {
verbs = ["create"]
api_groups = ["coordination.k8s.io"]
resources = ["leases"]
}
rule {
verbs = ["create", "patch"]
api_groups = [""]
resources = ["events"]
}
rule {
verbs = ["list", "watch", "get"]
api_groups = ["discovery.k8s.io"]
resources = ["endpointslices"]
}
}
resource "kubernetes_role" "ingress_nginx_admission" {
metadata {
name = "ingress-nginx-admission"
namespace = "ingress-nginx"
labels = {
"app.kubernetes.io/component" = "admission-webhook"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
"app.kubernetes.io/part-of" = "ingress-nginx"
"app.kubernetes.io/version" = "1.8.2"
}
}
rule {
verbs = ["get", "create", "patch", "update", "watch", "list"]
api_groups = [""]
resources = ["secrets"]
}
}
resource "kubernetes_cluster_role" "ingress_nginx" {
metadata {
name = "ingress-nginx"
labels = {
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
"app.kubernetes.io/part-of" = "ingress-nginx"
"app.kubernetes.io/version" = "1.8.2"
}
}
rule {
verbs = ["list", "watch"]
api_groups = [""]
resources = ["configmaps", "endpoints", "nodes", "pods", "secrets", "namespaces"]
}
rule {
verbs = ["list", "watch"]
api_groups = ["coordination.k8s.io"]
resources = ["leases"]
}
rule {
verbs = ["get"]
api_groups = [""]
resources = ["nodes"]
}
rule {
verbs = ["get", "list", "watch"]
api_groups = [""]
resources = ["services"]
}
rule {
verbs = ["get", "list", "watch"]
api_groups = ["networking.k8s.io"]
resources = ["ingresses"]
}
rule {
verbs = ["create", "patch"]
api_groups = [""]
resources = ["events"]
}
rule {
verbs = ["update"]
api_groups = ["networking.k8s.io"]
resources = ["ingresses/status"]
}
rule {
verbs = ["get", "list", "watch"]
api_groups = ["networking.k8s.io"]
resources = ["ingressclasses"]
}
rule {
verbs = ["list", "watch", "get"]
api_groups = ["discovery.k8s.io"]
resources = ["endpointslices"]
}
}
resource "kubernetes_cluster_role" "ingress_nginx_admission" {
metadata {
name = "ingress-nginx-admission"
labels = {
"app.kubernetes.io/component" = "admission-webhook"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
"app.kubernetes.io/part-of" = "ingress-nginx"
"app.kubernetes.io/version" = "1.8.2"
}
}
rule {
verbs = ["get", "update"]
api_groups = ["admissionregistration.k8s.io"]
resources = ["validatingwebhookconfigurations"]
}
}
resource "kubernetes_role_binding" "ingress_nginx" {
metadata {
name = "ingress-nginx"
namespace = "ingress-nginx"
labels = {
"app.kubernetes.io/component" = "controller"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
"app.kubernetes.io/part-of" = "ingress-nginx"
"app.kubernetes.io/version" = "1.8.2"
}
}
subject {
kind = "ServiceAccount"
name = "ingress-nginx"
namespace = "ingress-nginx"
}
role_ref {
api_group = "rbac.authorization.k8s.io"
kind = "Role"
name = "ingress-nginx"
}
}
resource "kubernetes_role_binding" "ingress_nginx_admission" {
metadata {
name = "ingress-nginx-admission"
namespace = "ingress-nginx"
labels = {
"app.kubernetes.io/component" = "admission-webhook"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
"app.kubernetes.io/part-of" = "ingress-nginx"
"app.kubernetes.io/version" = "1.8.2"
}
}
subject {
kind = "ServiceAccount"
name = "ingress-nginx-admission"
namespace = "ingress-nginx"
}
role_ref {
api_group = "rbac.authorization.k8s.io"
kind = "Role"
name = "ingress-nginx-admission"
}
}
resource "kubernetes_cluster_role_binding" "ingress_nginx" {
metadata {
name = "ingress-nginx"
labels = {
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
"app.kubernetes.io/part-of" = "ingress-nginx"
"app.kubernetes.io/version" = "1.8.2"
}
}
subject {
kind = "ServiceAccount"
name = "ingress-nginx"
namespace = "ingress-nginx"
}
role_ref {
api_group = "rbac.authorization.k8s.io"
kind = "ClusterRole"
name = "ingress-nginx"
}
}
resource "kubernetes_cluster_role_binding" "ingress_nginx_admission" {
metadata {
name = "ingress-nginx-admission"
labels = {
"app.kubernetes.io/component" = "admission-webhook"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
"app.kubernetes.io/part-of" = "ingress-nginx"
"app.kubernetes.io/version" = "1.8.2"
}
}
subject {
kind = "ServiceAccount"
name = "ingress-nginx-admission"
namespace = "ingress-nginx"
}
role_ref {
api_group = "rbac.authorization.k8s.io"
kind = "ClusterRole"
name = "ingress-nginx-admission"
}
}
resource "kubernetes_config_map" "ingress_nginx_controller" {
metadata {
name = "ingress-nginx-controller"
namespace = "ingress-nginx"
labels = {
"app.kubernetes.io/component" = "controller"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
"app.kubernetes.io/part-of" = "ingress-nginx"
"app.kubernetes.io/version" = "1.8.2"
}
}
data = {
allow-snippet-annotations = "true"
}
}
resource "kubernetes_service" "ingress_nginx_controller" {
metadata {
name = "ingress-nginx-controller"
namespace = "ingress-nginx"
labels = {
"app.kubernetes.io/component" = "controller"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
"app.kubernetes.io/part-of" = "ingress-nginx"
"app.kubernetes.io/version" = "1.8.2"
}
}
spec {
port {
name = "http"
protocol = "TCP"
port = 80
target_port = "http"
}
port {
name = "https"
protocol = "TCP"
port = 443
target_port = "https"
}
selector = {
"app.kubernetes.io/component" = "controller"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
}
type = "LoadBalancer"
# ip_families = ["IPv4"]
}
}
resource "kubernetes_service" "ingress_nginx_controller_admission" {
metadata {
name = "ingress-nginx-controller-admission"
namespace = "ingress-nginx"
labels = {
"app.kubernetes.io/component" = "controller"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
"app.kubernetes.io/part-of" = "ingress-nginx"
"app.kubernetes.io/version" = "1.8.2"
}
}
spec {
port {
name = "https-webhook"
port = 443
target_port = "webhook"
}
selector = {
"app.kubernetes.io/component" = "controller"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
}
type = "ClusterIP"
}
}
resource "kubernetes_deployment" "ingress_nginx_controller" {
metadata {
name = "ingress-nginx-controller"
namespace = "ingress-nginx"
labels = {
"app.kubernetes.io/component" = "controller"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
"app.kubernetes.io/part-of" = "ingress-nginx"
"app.kubernetes.io/version" = "1.8.2"
}
}
spec {
selector {
match_labels = {
"app.kubernetes.io/component" = "controller"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
}
}
template {
metadata {
labels = {
"app.kubernetes.io/component" = "controller"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
"app.kubernetes.io/part-of" = "ingress-nginx"
"app.kubernetes.io/version" = "1.8.2"
}
}
spec {
volume {
name = "webhook-cert"
secret {
secret_name = "ingress-nginx-admission"
}
}
container {
name = "controller"
image = "registry.k8s.io/ingress-nginx/controller:v1.8.2@sha256:74834d3d25b336b62cabeb8bf7f1d788706e2cf1cfd64022de4137ade8881ff2"
args = ["/nginx-ingress-controller", "--election-id=ingress-nginx-leader", "--controller-class=k8s.io/ingress-nginx", "--ingress-class=nginx", "--configmap=$(POD_NAMESPACE)/ingress-nginx-controller", "--validating-webhook=:8443", "--validating-webhook-certificate=/usr/local/certificates/cert", "--validating-webhook-key=/usr/local/certificates/key"]
port {
name = "http"
container_port = 80
protocol = "TCP"
}
port {
name = "https"
container_port = 443
protocol = "TCP"
}
port {
name = "webhook"
container_port = 8443
protocol = "TCP"
}
env {
name = "POD_NAME"
value_from {
field_ref {
field_path = "metadata.name"
}
}
}
env {
name = "POD_NAMESPACE"
value_from {
field_ref {
field_path = "metadata.namespace"
}
}
}
env {
name = "LD_PRELOAD"
value = "/usr/local/lib/libmimalloc.so"
}
resources {
requests = {
cpu = "100m"
memory = "90Mi"
}
}
volume_mount {
name = "webhook-cert"
read_only = true
mount_path = "/usr/local/certificates/"
}
liveness_probe {
http_get {
path = "/healthz"
port = "10254"
scheme = "HTTP"
}
initial_delay_seconds = 10
timeout_seconds = 1
period_seconds = 10
success_threshold = 1
failure_threshold = 5
}
readiness_probe {
http_get {
path = "/healthz"
port = "10254"
scheme = "HTTP"
}
initial_delay_seconds = 10
timeout_seconds = 1
period_seconds = 10
success_threshold = 1
failure_threshold = 3
}
lifecycle {
pre_stop {
exec {
command = ["/wait-shutdown"]
}
}
}
image_pull_policy = "IfNotPresent"
security_context {
capabilities {
add = ["NET_BIND_SERVICE"]
drop = ["ALL"]
}
run_as_user = 101
allow_privilege_escalation = true
}
}
termination_grace_period_seconds = 300
dns_policy = "ClusterFirst"
node_selector = {
"kubernetes.io/os" = "linux"
}
service_account_name = "ingress-nginx"
}
}
strategy {
type = "RollingUpdate"
rolling_update {
max_unavailable = "1"
}
}
revision_history_limit = 10
}
}
resource "kubernetes_job" "ingress_nginx_admission_create" {
metadata {
name = "ingress-nginx-admission-create"
namespace = "ingress-nginx"
labels = {
"app.kubernetes.io/component" = "admission-webhook"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
"app.kubernetes.io/part-of" = "ingress-nginx"
"app.kubernetes.io/version" = "1.8.2"
}
}
spec {
template {
metadata {
name = "ingress-nginx-admission-create"
labels = {
"app.kubernetes.io/component" = "admission-webhook"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
"app.kubernetes.io/part-of" = "ingress-nginx"
"app.kubernetes.io/version" = "1.8.2"
}
}
spec {
container {
name = "create"
image = "registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20230407@sha256:543c40fd093964bc9ab509d3e791f9989963021f1e9e4c9c7b6700b02bfb227b"
args = ["create", "--host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc", "--namespace=$(POD_NAMESPACE)", "--secret-name=ingress-nginx-admission"]
env {
name = "POD_NAMESPACE"
value_from {
field_ref {
field_path = "metadata.namespace"
}
}
}
image_pull_policy = "IfNotPresent"
}
restart_policy = "OnFailure"
node_selector = {
"kubernetes.io/os" = "linux"
}
service_account_name = "ingress-nginx-admission"
security_context {
run_as_user = 2000
run_as_non_root = true
fs_group = 2000
}
}
}
}
}
# Jobs create a cert and modify this secret. This is problematic as TF recreates it every time
# Instead, on each fresh install, uncomment this, get nginx working and comment it.
# Also rm from state: tf state rm module.kubernetes_cluster.module.nginx-ingress.kubernetes_job.ingress_nginx_admission_patch
# resource "kubernetes_job" "ingress_nginx_admission_patch" {
# metadata {
# name = "ingress-nginx-admission-patch"
# namespace = "ingress-nginx"
# labels = {
# "app.kubernetes.io/component" = "admission-webhook"
# "app.kubernetes.io/instance" = "ingress-nginx"
# "app.kubernetes.io/name" = "ingress-nginx"
# "app.kubernetes.io/part-of" = "ingress-nginx"
# "app.kubernetes.io/version" = "1.8.2"
# }
# }
# spec {
# template {
# metadata {
# name = "ingress-nginx-admission-patch"
# labels = {
# "app.kubernetes.io/component" = "admission-webhook"
# "app.kubernetes.io/instance" = "ingress-nginx"
# "app.kubernetes.io/name" = "ingress-nginx"
# "app.kubernetes.io/part-of" = "ingress-nginx"
# "app.kubernetes.io/version" = "1.8.2"
# }
# }
# spec {
# container {
# name = "patch"
# image = "registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20230407@sha256:543c40fd093964bc9ab509d3e791f9989963021f1e9e4c9c7b6700b02bfb227b"
# args = ["patch", "--webhook-name=ingress-nginx-admission", "--namespace=$(POD_NAMESPACE)", "--patch-mutating=false", "--secret-name=ingress-nginx-admission", "--patch-failure-policy=Fail"]
# env {
# name = "POD_NAMESPACE"
# value_from {
# field_ref {
# field_path = "metadata.namespace"
# }
# }
# }
# image_pull_policy = "IfNotPresent"
# }
# restart_policy = "OnFailure"
# node_selector = {
# "kubernetes.io/os" = "linux"
# }
# service_account_name = "ingress-nginx-admission"
# security_context {
# run_as_user = 2000
# run_as_non_root = true
# fs_group = 2000
# }
# }
# }
# }
# }
resource "kubernetes_ingress_class" "nginx" {
metadata {
name = "nginx"
labels = {
"app.kubernetes.io/component" = "controller"
"app.kubernetes.io/instance" = "ingress-nginx"
"app.kubernetes.io/name" = "ingress-nginx"
"app.kubernetes.io/part-of" = "ingress-nginx"
"app.kubernetes.io/version" = "1.8.2"
}
}
spec {
controller = "k8s.io/ingress-nginx"
}
}
# Jobs create a cert and modify this secret. This is problematic as TF recreates it every time
# Instead, on each fresh install, uncomment this, get nginx working and comment it.
# Also rm from state: tf state rm module.kubernetes_cluster.module.nginx-ingress.kubernetes_service_account.ingress_nginx_admission
# resource "kubernetes_validating_webhook_configuration" "ingress_nginx_admission" {
# metadata {
# name = "ingress-nginx-admission"
# labels = {
# "app.kubernetes.io/component" = "admission-webhook"
# "app.kubernetes.io/instance" = "ingress-nginx"
# "app.kubernetes.io/name" = "ingress-nginx"
# "app.kubernetes.io/part-of" = "ingress-nginx"
# "app.kubernetes.io/version" = "1.8.2"
# }
# }
# webhook {
# name = "validate.nginx.ingress.kubernetes.io"
# client_config {
# service {
# namespace = "ingress-nginx"
# name = "ingress-nginx-controller-admission"
# path = "/networking/v1/ingresses"
# }
# }
# rule {
# api_versions = ["v1"]
# api_groups = ["networking.k8s.io"]
# resources = ["ingresses"]
# operations = ["CREATE", "UPDATE"]
# }
# failure_policy = "Fail"
# match_policy = "Equivalent"
# side_effects = "None"
# admission_review_versions = ["v1"]
# }
# }

Binary file not shown.

Binary file not shown.