diff --git a/modules/kubernetes/descheduler/main.tf b/modules/kubernetes/descheduler/main.tf index c2594b21..56abaa97 100644 --- a/modules/kubernetes/descheduler/main.tf +++ b/modules/kubernetes/descheduler/main.tf @@ -14,7 +14,7 @@ resource "kubernetes_cluster_role" "descheduler" { verbs = ["create", "update"] } rule { - api_groups = [""] + api_groups = ["metrics.k8s.io"] resources = ["nodes"] verbs = ["get", "watch", "list"] } @@ -24,7 +24,7 @@ resource "kubernetes_cluster_role" "descheduler" { verbs = ["get", "list", "watch"] } rule { - api_groups = [""] + api_groups = ["metrics.k8s.io"] resources = ["pods"] verbs = ["get", "watch", "list", "delete"] } @@ -74,117 +74,14 @@ resource "kubernetes_cluster_role_binding" "descheduler" { } } -resource "kubernetes_config_map" "policy" { - metadata { - namespace = "descheduler" - name = "policy-configmap" - } - data = { - # "policy.yaml" = <<-EOF - # apiVersion: "descheduler/v1alpha1" - # maxNoOfPodsToEvictPerNode: 20 - # kind: "DeschedulerPolicy" - # strategies: - # "RemoveDuplicates": - # enabled: true - # "RemovePodsViolatingInterPodAntiAffinity": - # enabled: true - # "LowNodeUtilization": - # enabled: true - # params: - # nodeResourceUtilizationThresholds: - # thresholds: - # "cpu" : 50 - # "memory": 30 - # "pods": 20 - # targetThresholds: - # "cpu" : 70 - # "memory": 30 - # "pods": 50 - # "HighNodeUtilization": - # enabled: true - # params: - # nodeResourceUtilizationThresholds: - # thresholds: - # "cpu" : 20 - # "memory": 80 - # "pods": 20 - # "PodLifeTime": - # enabled: true - # params: - # podLifeTime: - # maxPodLifeTimeSeconds: 604800 - # namespaces: - # exclude: - # - "monitoring" - # - "kube-system" - # EOF - "policy.yaml" = <<-EOF - capiVersion: "descheduler/v1alpha2" - kind: "DeschedulerPolicy" - profiles: - - name: ProfileName - pluginConfig: - - name: "LowNodeUtilization" - args: - thresholds: - "cpu" : 20 - "memory": 20 - "pods": 20 - targetThresholds: - "cpu" : 20 - "memory": 20 - "pods": 20 - metricsUtilization: - metricsServer: true - plugins: - balance: - enabled: - - "LowNodeUtilization" - EOF - } -} +resource "helm_release" "prometheus" { + namespace = "descheduler" + name = "descheduler" -resource "kubernetes_cron_job_v1" "descheduler" { - metadata { - name = "descheduler" - namespace = "descheduler" - } - spec { - schedule = "0 0 * * *" - concurrency_policy = "Forbid" - job_template { - metadata { - name = "descheduler" - } - spec { - template { - metadata { - name = "descheduler" - } - spec { - priority_class_name = "system-cluster-critical" - container { - name = "descheduler" - image = "k8s.gcr.io/descheduler/descheduler:v0.28.0" - volume_mount { - mount_path = "/policy-dir" - name = "policy-volume" - } - command = ["/bin/descheduler"] - args = ["--policy-config-file", "/policy-dir/policy.yaml", "--v", "4"] - } - restart_policy = "Never" - service_account_name = "descheduler-sa" - volume { - name = "policy-volume" - config_map { - name = "policy-configmap" - } - } - } - } - } - } - } + repository = "https://kubernetes-sigs.github.io/descheduler/" + chart = "descheduler" + + + + values = [templatefile("${path.module}/values.yaml", {})] } diff --git a/modules/kubernetes/descheduler/values.yaml b/modules/kubernetes/descheduler/values.yaml new file mode 100644 index 00000000..c56cbe31 --- /dev/null +++ b/modules/kubernetes/descheduler/values.yaml @@ -0,0 +1,264 @@ +# Source from https://github.com/kubernetes-sigs/descheduler/blob/master/charts/descheduler/values.yaml + +# Default values for descheduler. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# CronJob or Deployment +kind: CronJob + +image: + repository: registry.k8s.io/descheduler/descheduler + # Overrides the image tag whose default is the chart version + tag: "" + pullPolicy: IfNotPresent + +imagePullSecrets: +# - name: container-registry-secret + +resources: + requests: + cpu: 500m + memory: 256Mi + limits: + cpu: 500m + memory: 256Mi + +ports: + - containerPort: 10258 + protocol: TCP + +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + +# podSecurityContext -- [Security context for pod](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) +podSecurityContext: + {} + # fsGroup: 1000 + +nameOverride: "" +fullnameOverride: "" + +# -- Override the deployment namespace; defaults to .Release.Namespace +namespaceOverride: "" + +# labels that'll be applied to all resources +commonLabels: {} + +cronJobApiVersion: "batch/v1" +schedule: "0 * * * *" +suspend: false +# startingDeadlineSeconds: 200 +# successfulJobsHistoryLimit: 3 +# failedJobsHistoryLimit: 1 +# ttlSecondsAfterFinished 600 +# timeZone: Etc/UTC + +# Required when running as a Deployment +deschedulingInterval: 5m + +# Specifies the replica count for Deployment +# Set leaderElection if you want to use more than 1 replica +# Set affinity.podAntiAffinity rule if you want to schedule onto a node +# only if that node is in the same zone as at least one already-running descheduler +replicas: 1 + +# Specifies whether Leader Election resources should be created +# Required when running as a Deployment +# NOTE: Leader election can't be activated if DryRun enabled +leaderElection: {} +# enabled: true +# leaseDuration: 15s +# renewDeadline: 10s +# retryPeriod: 2s +# resourceLock: "leases" +# resourceName: "descheduler" +# resourceNamespace: "kube-system" + +command: + - "/bin/descheduler" + +cmdOptions: + v: 3 + +# Recommended to use the latest Policy API version supported by the Descheduler app version +deschedulerPolicyAPIVersion: "descheduler/v1alpha2" + +# deschedulerPolicy contains the policies the descheduler will execute. +# To use policies stored in an existing configMap use: +# NOTE: The name of the cm should comply to {{ template "descheduler.fullname" . }} +# deschedulerPolicy: {} +deschedulerPolicy: + # nodeSelector: "key1=value1,key2=value2" + # maxNoOfPodsToEvictPerNode: 10 + # maxNoOfPodsToEvictPerNamespace: 10 + # ignorePvcPods: true + # evictLocalStoragePods: true + # evictDaemonSetPods: true + # tracing: + # collectorEndpoint: otel-collector.observability.svc.cluster.local:4317 + # transportCert: "" + # serviceName: "" + # serviceNamespace: "" + # sampleRate: 1.0 + # fallbackToNoOpProviderOnError: true + + metricsCollector: + enabled: true + profiles: + - name: default + pluginConfig: + - name: DefaultEvictor + args: + ignorePvcPods: true + evictLocalStoragePods: true + - name: RemoveDuplicates + - name: RemovePodsHavingTooManyRestarts + args: + podRestartThreshold: 100 + includingInitContainers: true + - name: RemovePodsViolatingNodeAffinity + args: + nodeAffinityType: + - requiredDuringSchedulingIgnoredDuringExecution + - name: RemovePodsViolatingNodeTaints + - name: RemovePodsViolatingInterPodAntiAffinity + - name: RemovePodsViolatingTopologySpreadConstraint + - name: LowNodeUtilization + args: + namespaces: + exclude: + - "dbaas" # let's not meddle with the dbs + thresholds: + cpu: 50 + memory: 50 + pods: 20 + targetThresholds: + cpu: 80 + memory: 80 + pods: 30 + metricsUtilization: + metricsServer: true + plugins: + balance: + enabled: + - RemoveDuplicates + - RemovePodsViolatingTopologySpreadConstraint + - LowNodeUtilization + deschedule: + enabled: + - RemovePodsHavingTooManyRestarts + - RemovePodsViolatingNodeTaints + - RemovePodsViolatingNodeAffinity + - RemovePodsViolatingInterPodAntiAffinity + +priorityClassName: system-cluster-critical + +nodeSelector: {} +# foo: bar + +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: kubernetes.io/e2e-az-name +# operator: In +# values: +# - e2e-az1 +# - e2e-az2 +# podAntiAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# - labelSelector: +# matchExpressions: +# - key: app.kubernetes.io/name +# operator: In +# values: +# - descheduler +# topologyKey: "kubernetes.io/hostname" +topologySpreadConstraints: [] +# - maxSkew: 1 +# topologyKey: kubernetes.io/hostname +# whenUnsatisfiable: DoNotSchedule +# labelSelector: +# matchLabels: +# app.kubernetes.io/name: descheduler +tolerations: [] +# - key: 'management' +# operator: 'Equal' +# value: 'tool' +# effect: 'NoSchedule' + +rbac: + # Specifies whether RBAC resources should be created + create: true + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: false + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: "descheduler-sa" + # Specifies custom annotations for the serviceAccount + annotations: {} + +podAnnotations: {} + +podLabels: {} + +dnsConfig: {} + +livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10258 + scheme: HTTPS + initialDelaySeconds: 3 + periodSeconds: 10 + +service: + enabled: false + # @param service.ipFamilyPolicy [string], support SingleStack, PreferDualStack and RequireDualStack + # + ipFamilyPolicy: "" + # @param service.ipFamilies [array] List of IP families (e.g. IPv4, IPv6) assigned to the service. + # Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/ + # E.g. + # ipFamilies: + # - IPv6 + # - IPv4 + ipFamilies: [] + +serviceMonitor: + enabled: false + # The namespace where Prometheus expects to find service monitors. + # namespace: "" + # Add custom labels to the ServiceMonitor resource + additionalLabels: + {} + # prometheus: kube-prometheus-stack + interval: "" + # honorLabels: true + insecureSkipVerify: true + serverName: null + metricRelabelings: + [] + # - action: keep + # regex: 'descheduler_(build_info|pods_evicted)' + # sourceLabels: [__name__] + relabelings: + [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace diff --git a/modules/kubernetes/immich/main.tf b/modules/kubernetes/immich/main.tf index 25a77696..2b229268 100644 --- a/modules/kubernetes/immich/main.tf +++ b/modules/kubernetes/immich/main.tf @@ -277,148 +277,3 @@ resource "kubernetes_cron_job_v1" "postgresql-backup" { } # POWER TOOLS - -resource "kubernetes_deployment" "powertools" { - metadata { - name = "powertools" - namespace = "immich" - labels = { - app = "powertools" - } - annotations = { - "reloader.stakater.com/search" = "true" - } - } - spec { - replicas = 1 - strategy { - type = "Recreate" - } - selector { - match_labels = { - app = "powertools" - } - } - template { - metadata { - labels = { - app = "powertools" - } - annotations = { - "diun.enable" = "true" - "diun.include_tags" = "latest" - } - } - spec { - - container { - image = "ghcr.io/varun-raj/immich-power-tools:latest" - name = "owntracks" - port { - name = "http" - container_port = 3000 - } - env { - name = "IMMICH_API_KEY" - value = "bLz8OIoRrtUFumDaBrXjkgwXkU5YoJBXcwRr8F90KQ" - } - env { - name = "IMMICH_URL" - # value = "http://immich-server.immich.svc.cluster.local" - value = "http://10.109.73.214" - } - env { - name = "EXTERNAL_IMMICH_URL" - value = "https://immich.viktorbarzin.me" - } - env { - name = "DB_USERNAME" - value = "immich" - } - env { - name = "DB_PASSWORD" - value = var.postgresql_password - } - env { - name = "DB_HOST" - # value = "immich-postgresql.immich.svc.cluster.local" - value = "10.102.122.16" - } - # env { - # name = "DB_PORT" - # value = "5432" - # } - env { - name = "DB_DATABASE_NAME" - value = "immich" - } - env { - name = "NODE_ENV" - value = "development" - } - - } - } - } - } -} - - -resource "kubernetes_service" "powertools" { - metadata { - name = "powertools" - namespace = "immich" - labels = { - "app" = "powertools" - } - } - - spec { - selector = { - app = "powertools" - } - port { - name = "http" - port = 80 - target_port = 3000 - protocol = "TCP" - } - } -} - -resource "kubernetes_ingress_v1" "powertools" { - metadata { - name = "powertools" - namespace = "immich" - annotations = { - "kubernetes.io/ingress.class" = "nginx" - "nginx.ingress.kubernetes.io/auth-url" : "http://ak-outpost-authentik-embedded-outpost.authentik.svc.cluster.local:9000/outpost.goauthentik.io/auth/nginx" - "nginx.ingress.kubernetes.io/auth-signin" : "https://authentik.viktorbarzin.me/outpost.goauthentik.io/start?rd=$scheme%3A%2F%2F$host$escaped_request_uri" - "nginx.ingress.kubernetes.io/auth-response-headers" : "Set-Cookie,X-authentik-username,X-authentik-groups,X-authentik-email,X-authentik-name,X-authentik-uid" - "nginx.ingress.kubernetes.io/auth-snippet" : "proxy_set_header X-Forwarded-Host $http_host;" - } - } - - spec { - tls { - hosts = ["immich-powertools.viktorbarzin.me"] - secret_name = var.tls_secret_name - } - rule { - host = "immich-powertools.viktorbarzin.me" - http { - path { - path = "/" - backend { - service { - name = "powertools" - port { - number = 80 - } - } - } - } - } - } - } -}