fix descheduler [ci skip]
This commit is contained in:
Viktor Barzin
parent
e7a222360c
commit
216a74f819
3 changed files with 275 additions and 259 deletions
|
|
@ -14,7 +14,7 @@ resource "kubernetes_cluster_role" "descheduler" {
|
|||
verbs = ["create", "update"]
|
||||
}
|
||||
rule {
|
||||
api_groups = [""]
|
||||
api_groups = ["metrics.k8s.io"]
|
||||
resources = ["nodes"]
|
||||
verbs = ["get", "watch", "list"]
|
||||
}
|
||||
|
|
@ -24,7 +24,7 @@ resource "kubernetes_cluster_role" "descheduler" {
|
|||
verbs = ["get", "list", "watch"]
|
||||
}
|
||||
rule {
|
||||
api_groups = [""]
|
||||
api_groups = ["metrics.k8s.io"]
|
||||
resources = ["pods"]
|
||||
verbs = ["get", "watch", "list", "delete"]
|
||||
}
|
||||
|
|
@ -74,117 +74,14 @@ resource "kubernetes_cluster_role_binding" "descheduler" {
|
|||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_config_map" "policy" {
|
||||
metadata {
|
||||
namespace = "descheduler"
|
||||
name = "policy-configmap"
|
||||
}
|
||||
data = {
|
||||
# "policy.yaml" = <<-EOF
|
||||
# apiVersion: "descheduler/v1alpha1"
|
||||
# maxNoOfPodsToEvictPerNode: 20
|
||||
# kind: "DeschedulerPolicy"
|
||||
# strategies:
|
||||
# "RemoveDuplicates":
|
||||
# enabled: true
|
||||
# "RemovePodsViolatingInterPodAntiAffinity":
|
||||
# enabled: true
|
||||
# "LowNodeUtilization":
|
||||
# enabled: true
|
||||
# params:
|
||||
# nodeResourceUtilizationThresholds:
|
||||
# thresholds:
|
||||
# "cpu" : 50
|
||||
# "memory": 30
|
||||
# "pods": 20
|
||||
# targetThresholds:
|
||||
# "cpu" : 70
|
||||
# "memory": 30
|
||||
# "pods": 50
|
||||
# "HighNodeUtilization":
|
||||
# enabled: true
|
||||
# params:
|
||||
# nodeResourceUtilizationThresholds:
|
||||
# thresholds:
|
||||
# "cpu" : 20
|
||||
# "memory": 80
|
||||
# "pods": 20
|
||||
# "PodLifeTime":
|
||||
# enabled: true
|
||||
# params:
|
||||
# podLifeTime:
|
||||
# maxPodLifeTimeSeconds: 604800
|
||||
# namespaces:
|
||||
# exclude:
|
||||
# - "monitoring"
|
||||
# - "kube-system"
|
||||
# EOF
|
||||
"policy.yaml" = <<-EOF
|
||||
capiVersion: "descheduler/v1alpha2"
|
||||
kind: "DeschedulerPolicy"
|
||||
profiles:
|
||||
- name: ProfileName
|
||||
pluginConfig:
|
||||
- name: "LowNodeUtilization"
|
||||
args:
|
||||
thresholds:
|
||||
"cpu" : 20
|
||||
"memory": 20
|
||||
"pods": 20
|
||||
targetThresholds:
|
||||
"cpu" : 20
|
||||
"memory": 20
|
||||
"pods": 20
|
||||
metricsUtilization:
|
||||
metricsServer: true
|
||||
plugins:
|
||||
balance:
|
||||
enabled:
|
||||
- "LowNodeUtilization"
|
||||
EOF
|
||||
}
|
||||
}
|
||||
resource "helm_release" "prometheus" {
|
||||
namespace = "descheduler"
|
||||
name = "descheduler"
|
||||
|
||||
resource "kubernetes_cron_job_v1" "descheduler" {
|
||||
metadata {
|
||||
name = "descheduler"
|
||||
namespace = "descheduler"
|
||||
}
|
||||
spec {
|
||||
schedule = "0 0 * * *"
|
||||
concurrency_policy = "Forbid"
|
||||
job_template {
|
||||
metadata {
|
||||
name = "descheduler"
|
||||
}
|
||||
spec {
|
||||
template {
|
||||
metadata {
|
||||
name = "descheduler"
|
||||
}
|
||||
spec {
|
||||
priority_class_name = "system-cluster-critical"
|
||||
container {
|
||||
name = "descheduler"
|
||||
image = "k8s.gcr.io/descheduler/descheduler:v0.28.0"
|
||||
volume_mount {
|
||||
mount_path = "/policy-dir"
|
||||
name = "policy-volume"
|
||||
}
|
||||
command = ["/bin/descheduler"]
|
||||
args = ["--policy-config-file", "/policy-dir/policy.yaml", "--v", "4"]
|
||||
}
|
||||
restart_policy = "Never"
|
||||
service_account_name = "descheduler-sa"
|
||||
volume {
|
||||
name = "policy-volume"
|
||||
config_map {
|
||||
name = "policy-configmap"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
repository = "https://kubernetes-sigs.github.io/descheduler/"
|
||||
chart = "descheduler"
|
||||
|
||||
|
||||
|
||||
values = [templatefile("${path.module}/values.yaml", {})]
|
||||
}
|
||||
|
|
|
|||
264
modules/kubernetes/descheduler/values.yaml
Normal file
264
modules/kubernetes/descheduler/values.yaml
Normal file
|
|
@ -0,0 +1,264 @@
|
|||
# Source from https://github.com/kubernetes-sigs/descheduler/blob/master/charts/descheduler/values.yaml
|
||||
|
||||
# Default values for descheduler.
|
||||
# This is a YAML-formatted file.
|
||||
# Declare variables to be passed into your templates.
|
||||
|
||||
# CronJob or Deployment
|
||||
kind: CronJob
|
||||
|
||||
image:
|
||||
repository: registry.k8s.io/descheduler/descheduler
|
||||
# Overrides the image tag whose default is the chart version
|
||||
tag: ""
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
imagePullSecrets:
|
||||
# - name: container-registry-secret
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 256Mi
|
||||
|
||||
ports:
|
||||
- containerPort: 10258
|
||||
protocol: TCP
|
||||
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
privileged: false
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
|
||||
# podSecurityContext -- [Security context for pod](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/)
|
||||
podSecurityContext:
|
||||
{}
|
||||
# fsGroup: 1000
|
||||
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
|
||||
# -- Override the deployment namespace; defaults to .Release.Namespace
|
||||
namespaceOverride: ""
|
||||
|
||||
# labels that'll be applied to all resources
|
||||
commonLabels: {}
|
||||
|
||||
cronJobApiVersion: "batch/v1"
|
||||
schedule: "0 * * * *"
|
||||
suspend: false
|
||||
# startingDeadlineSeconds: 200
|
||||
# successfulJobsHistoryLimit: 3
|
||||
# failedJobsHistoryLimit: 1
|
||||
# ttlSecondsAfterFinished 600
|
||||
# timeZone: Etc/UTC
|
||||
|
||||
# Required when running as a Deployment
|
||||
deschedulingInterval: 5m
|
||||
|
||||
# Specifies the replica count for Deployment
|
||||
# Set leaderElection if you want to use more than 1 replica
|
||||
# Set affinity.podAntiAffinity rule if you want to schedule onto a node
|
||||
# only if that node is in the same zone as at least one already-running descheduler
|
||||
replicas: 1
|
||||
|
||||
# Specifies whether Leader Election resources should be created
|
||||
# Required when running as a Deployment
|
||||
# NOTE: Leader election can't be activated if DryRun enabled
|
||||
leaderElection: {}
|
||||
# enabled: true
|
||||
# leaseDuration: 15s
|
||||
# renewDeadline: 10s
|
||||
# retryPeriod: 2s
|
||||
# resourceLock: "leases"
|
||||
# resourceName: "descheduler"
|
||||
# resourceNamespace: "kube-system"
|
||||
|
||||
command:
|
||||
- "/bin/descheduler"
|
||||
|
||||
cmdOptions:
|
||||
v: 3
|
||||
|
||||
# Recommended to use the latest Policy API version supported by the Descheduler app version
|
||||
deschedulerPolicyAPIVersion: "descheduler/v1alpha2"
|
||||
|
||||
# deschedulerPolicy contains the policies the descheduler will execute.
|
||||
# To use policies stored in an existing configMap use:
|
||||
# NOTE: The name of the cm should comply to {{ template "descheduler.fullname" . }}
|
||||
# deschedulerPolicy: {}
|
||||
deschedulerPolicy:
|
||||
# nodeSelector: "key1=value1,key2=value2"
|
||||
# maxNoOfPodsToEvictPerNode: 10
|
||||
# maxNoOfPodsToEvictPerNamespace: 10
|
||||
# ignorePvcPods: true
|
||||
# evictLocalStoragePods: true
|
||||
# evictDaemonSetPods: true
|
||||
# tracing:
|
||||
# collectorEndpoint: otel-collector.observability.svc.cluster.local:4317
|
||||
# transportCert: ""
|
||||
# serviceName: ""
|
||||
# serviceNamespace: ""
|
||||
# sampleRate: 1.0
|
||||
# fallbackToNoOpProviderOnError: true
|
||||
|
||||
metricsCollector:
|
||||
enabled: true
|
||||
profiles:
|
||||
- name: default
|
||||
pluginConfig:
|
||||
- name: DefaultEvictor
|
||||
args:
|
||||
ignorePvcPods: true
|
||||
evictLocalStoragePods: true
|
||||
- name: RemoveDuplicates
|
||||
- name: RemovePodsHavingTooManyRestarts
|
||||
args:
|
||||
podRestartThreshold: 100
|
||||
includingInitContainers: true
|
||||
- name: RemovePodsViolatingNodeAffinity
|
||||
args:
|
||||
nodeAffinityType:
|
||||
- requiredDuringSchedulingIgnoredDuringExecution
|
||||
- name: RemovePodsViolatingNodeTaints
|
||||
- name: RemovePodsViolatingInterPodAntiAffinity
|
||||
- name: RemovePodsViolatingTopologySpreadConstraint
|
||||
- name: LowNodeUtilization
|
||||
args:
|
||||
namespaces:
|
||||
exclude:
|
||||
- "dbaas" # let's not meddle with the dbs
|
||||
thresholds:
|
||||
cpu: 50
|
||||
memory: 50
|
||||
pods: 20
|
||||
targetThresholds:
|
||||
cpu: 80
|
||||
memory: 80
|
||||
pods: 30
|
||||
metricsUtilization:
|
||||
metricsServer: true
|
||||
plugins:
|
||||
balance:
|
||||
enabled:
|
||||
- RemoveDuplicates
|
||||
- RemovePodsViolatingTopologySpreadConstraint
|
||||
- LowNodeUtilization
|
||||
deschedule:
|
||||
enabled:
|
||||
- RemovePodsHavingTooManyRestarts
|
||||
- RemovePodsViolatingNodeTaints
|
||||
- RemovePodsViolatingNodeAffinity
|
||||
- RemovePodsViolatingInterPodAntiAffinity
|
||||
|
||||
priorityClassName: system-cluster-critical
|
||||
|
||||
nodeSelector: {}
|
||||
# foo: bar
|
||||
|
||||
affinity: {}
|
||||
# nodeAffinity:
|
||||
# requiredDuringSchedulingIgnoredDuringExecution:
|
||||
# nodeSelectorTerms:
|
||||
# - matchExpressions:
|
||||
# - key: kubernetes.io/e2e-az-name
|
||||
# operator: In
|
||||
# values:
|
||||
# - e2e-az1
|
||||
# - e2e-az2
|
||||
# podAntiAffinity:
|
||||
# requiredDuringSchedulingIgnoredDuringExecution:
|
||||
# - labelSelector:
|
||||
# matchExpressions:
|
||||
# - key: app.kubernetes.io/name
|
||||
# operator: In
|
||||
# values:
|
||||
# - descheduler
|
||||
# topologyKey: "kubernetes.io/hostname"
|
||||
topologySpreadConstraints: []
|
||||
# - maxSkew: 1
|
||||
# topologyKey: kubernetes.io/hostname
|
||||
# whenUnsatisfiable: DoNotSchedule
|
||||
# labelSelector:
|
||||
# matchLabels:
|
||||
# app.kubernetes.io/name: descheduler
|
||||
tolerations: []
|
||||
# - key: 'management'
|
||||
# operator: 'Equal'
|
||||
# value: 'tool'
|
||||
# effect: 'NoSchedule'
|
||||
|
||||
rbac:
|
||||
# Specifies whether RBAC resources should be created
|
||||
create: true
|
||||
|
||||
serviceAccount:
|
||||
# Specifies whether a ServiceAccount should be created
|
||||
create: false
|
||||
# The name of the ServiceAccount to use.
|
||||
# If not set and create is true, a name is generated using the fullname template
|
||||
name: "descheduler-sa"
|
||||
# Specifies custom annotations for the serviceAccount
|
||||
annotations: {}
|
||||
|
||||
podAnnotations: {}
|
||||
|
||||
podLabels: {}
|
||||
|
||||
dnsConfig: {}
|
||||
|
||||
livenessProbe:
|
||||
failureThreshold: 3
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: 10258
|
||||
scheme: HTTPS
|
||||
initialDelaySeconds: 3
|
||||
periodSeconds: 10
|
||||
|
||||
service:
|
||||
enabled: false
|
||||
# @param service.ipFamilyPolicy [string], support SingleStack, PreferDualStack and RequireDualStack
|
||||
#
|
||||
ipFamilyPolicy: ""
|
||||
# @param service.ipFamilies [array] List of IP families (e.g. IPv4, IPv6) assigned to the service.
|
||||
# Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/
|
||||
# E.g.
|
||||
# ipFamilies:
|
||||
# - IPv6
|
||||
# - IPv4
|
||||
ipFamilies: []
|
||||
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
# The namespace where Prometheus expects to find service monitors.
|
||||
# namespace: ""
|
||||
# Add custom labels to the ServiceMonitor resource
|
||||
additionalLabels:
|
||||
{}
|
||||
# prometheus: kube-prometheus-stack
|
||||
interval: ""
|
||||
# honorLabels: true
|
||||
insecureSkipVerify: true
|
||||
serverName: null
|
||||
metricRelabelings:
|
||||
[]
|
||||
# - action: keep
|
||||
# regex: 'descheduler_(build_info|pods_evicted)'
|
||||
# sourceLabels: [__name__]
|
||||
relabelings:
|
||||
[]
|
||||
# - sourceLabels: [__meta_kubernetes_pod_node_name]
|
||||
# separator: ;
|
||||
# regex: ^(.*)$
|
||||
# targetLabel: nodename
|
||||
# replacement: $1
|
||||
# action: replace
|
||||
|
|
@ -277,148 +277,3 @@ resource "kubernetes_cron_job_v1" "postgresql-backup" {
|
|||
}
|
||||
|
||||
# POWER TOOLS
|
||||
|
||||
resource "kubernetes_deployment" "powertools" {
|
||||
metadata {
|
||||
name = "powertools"
|
||||
namespace = "immich"
|
||||
labels = {
|
||||
app = "powertools"
|
||||
}
|
||||
annotations = {
|
||||
"reloader.stakater.com/search" = "true"
|
||||
}
|
||||
}
|
||||
spec {
|
||||
replicas = 1
|
||||
strategy {
|
||||
type = "Recreate"
|
||||
}
|
||||
selector {
|
||||
match_labels = {
|
||||
app = "powertools"
|
||||
}
|
||||
}
|
||||
template {
|
||||
metadata {
|
||||
labels = {
|
||||
app = "powertools"
|
||||
}
|
||||
annotations = {
|
||||
"diun.enable" = "true"
|
||||
"diun.include_tags" = "latest"
|
||||
}
|
||||
}
|
||||
spec {
|
||||
|
||||
container {
|
||||
image = "ghcr.io/varun-raj/immich-power-tools:latest"
|
||||
name = "owntracks"
|
||||
port {
|
||||
name = "http"
|
||||
container_port = 3000
|
||||
}
|
||||
env {
|
||||
name = "IMMICH_API_KEY"
|
||||
value = "bLz8OIoRrtUFumDaBrXjkgwXkU5YoJBXcwRr8F90KQ"
|
||||
}
|
||||
env {
|
||||
name = "IMMICH_URL"
|
||||
# value = "http://immich-server.immich.svc.cluster.local"
|
||||
value = "http://10.109.73.214"
|
||||
}
|
||||
env {
|
||||
name = "EXTERNAL_IMMICH_URL"
|
||||
value = "https://immich.viktorbarzin.me"
|
||||
}
|
||||
env {
|
||||
name = "DB_USERNAME"
|
||||
value = "immich"
|
||||
}
|
||||
env {
|
||||
name = "DB_PASSWORD"
|
||||
value = var.postgresql_password
|
||||
}
|
||||
env {
|
||||
name = "DB_HOST"
|
||||
# value = "immich-postgresql.immich.svc.cluster.local"
|
||||
value = "10.102.122.16"
|
||||
}
|
||||
# env {
|
||||
# name = "DB_PORT"
|
||||
# value = "5432"
|
||||
# }
|
||||
env {
|
||||
name = "DB_DATABASE_NAME"
|
||||
value = "immich"
|
||||
}
|
||||
env {
|
||||
name = "NODE_ENV"
|
||||
value = "development"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
resource "kubernetes_service" "powertools" {
|
||||
metadata {
|
||||
name = "powertools"
|
||||
namespace = "immich"
|
||||
labels = {
|
||||
"app" = "powertools"
|
||||
}
|
||||
}
|
||||
|
||||
spec {
|
||||
selector = {
|
||||
app = "powertools"
|
||||
}
|
||||
port {
|
||||
name = "http"
|
||||
port = 80
|
||||
target_port = 3000
|
||||
protocol = "TCP"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_ingress_v1" "powertools" {
|
||||
metadata {
|
||||
name = "powertools"
|
||||
namespace = "immich"
|
||||
annotations = {
|
||||
"kubernetes.io/ingress.class" = "nginx"
|
||||
"nginx.ingress.kubernetes.io/auth-url" : "http://ak-outpost-authentik-embedded-outpost.authentik.svc.cluster.local:9000/outpost.goauthentik.io/auth/nginx"
|
||||
"nginx.ingress.kubernetes.io/auth-signin" : "https://authentik.viktorbarzin.me/outpost.goauthentik.io/start?rd=$scheme%3A%2F%2F$host$escaped_request_uri"
|
||||
"nginx.ingress.kubernetes.io/auth-response-headers" : "Set-Cookie,X-authentik-username,X-authentik-groups,X-authentik-email,X-authentik-name,X-authentik-uid"
|
||||
"nginx.ingress.kubernetes.io/auth-snippet" : "proxy_set_header X-Forwarded-Host $http_host;"
|
||||
}
|
||||
}
|
||||
|
||||
spec {
|
||||
tls {
|
||||
hosts = ["immich-powertools.viktorbarzin.me"]
|
||||
secret_name = var.tls_secret_name
|
||||
}
|
||||
rule {
|
||||
host = "immich-powertools.viktorbarzin.me"
|
||||
http {
|
||||
path {
|
||||
path = "/"
|
||||
backend {
|
||||
service {
|
||||
name = "powertools"
|
||||
port {
|
||||
number = 80
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue