Cluster health remediation: cleanup CronJob, disable Collabora, fix GPU probe, add NFS exports [ci skip]
- Add daily CronJob to auto-clean Failed/Evicted pods cluster-wide (infra-maintenance) - Disable Collabora in Nextcloud (broken HPA caused scaling storm; using OnlyOffice instead) - Increase gpu-pod-exporter liveness probe timeout from 1s to 5s - Add osm-routing NFS exports (osrm-data, otp-data)
This commit is contained in:
parent
3da35166ab
commit
a73f3fcb6b
4 changed files with 71 additions and 5 deletions
|
|
@ -141,3 +141,72 @@ resource "kubernetes_cron_job_v1" "backup-etcd" {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Clean up evicted/failed pods cluster-wide daily
|
||||||
|
resource "kubernetes_cron_job_v1" "cleanup-failed-pods" {
|
||||||
|
metadata {
|
||||||
|
name = "cleanup-failed-pods"
|
||||||
|
namespace = "default"
|
||||||
|
}
|
||||||
|
spec {
|
||||||
|
schedule = "0 2 * * *"
|
||||||
|
successful_jobs_history_limit = 1
|
||||||
|
failed_jobs_history_limit = 1
|
||||||
|
concurrency_policy = "Forbid"
|
||||||
|
job_template {
|
||||||
|
metadata {
|
||||||
|
name = "cleanup-failed-pods"
|
||||||
|
}
|
||||||
|
spec {
|
||||||
|
template {
|
||||||
|
metadata {
|
||||||
|
name = "cleanup-failed-pods"
|
||||||
|
}
|
||||||
|
spec {
|
||||||
|
service_account_name = kubernetes_service_account.cleanup_sa.metadata[0].name
|
||||||
|
container {
|
||||||
|
name = "cleanup"
|
||||||
|
image = "bitnami/kubectl:latest"
|
||||||
|
command = ["/bin/sh", "-c", "kubectl delete pods -A --field-selector=status.phase=Failed --ignore-not-found"]
|
||||||
|
}
|
||||||
|
restart_policy = "Never"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "kubernetes_service_account" "cleanup_sa" {
|
||||||
|
metadata {
|
||||||
|
name = "failed-pod-cleanup"
|
||||||
|
namespace = "default"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "kubernetes_cluster_role" "cleanup_role" {
|
||||||
|
metadata {
|
||||||
|
name = "failed-pod-cleanup"
|
||||||
|
}
|
||||||
|
rule {
|
||||||
|
api_groups = [""]
|
||||||
|
resources = ["pods"]
|
||||||
|
verbs = ["list", "delete"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "kubernetes_cluster_role_binding" "cleanup_binding" {
|
||||||
|
metadata {
|
||||||
|
name = "failed-pod-cleanup"
|
||||||
|
}
|
||||||
|
role_ref {
|
||||||
|
api_group = "rbac.authorization.k8s.io"
|
||||||
|
kind = "ClusterRole"
|
||||||
|
name = kubernetes_cluster_role.cleanup_role.metadata[0].name
|
||||||
|
}
|
||||||
|
subject {
|
||||||
|
kind = "ServiceAccount"
|
||||||
|
name = kubernetes_service_account.cleanup_sa.metadata[0].name
|
||||||
|
namespace = "default"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -59,11 +59,7 @@ podAnnotations:
|
||||||
diun.include_tags: "^[0-9]+(?:.[0-9]+)?(?:.[0-9]+)?.*"
|
diun.include_tags: "^[0-9]+(?:.[0-9]+)?(?:.[0-9]+)?.*"
|
||||||
|
|
||||||
collabora:
|
collabora:
|
||||||
enabled: true # Currently the app is disabled as using onlyoffice instead
|
enabled: false # Using onlyoffice instead
|
||||||
|
|
||||||
autoscaling:
|
|
||||||
# enable autocaling, please check collabora README.md first
|
|
||||||
enabled: true
|
|
||||||
|
|
||||||
cronjob:
|
cronjob:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
|
|
||||||
|
|
@ -605,6 +605,7 @@ resource "kubernetes_daemonset" "gpu_pod_exporter" {
|
||||||
}
|
}
|
||||||
initial_delay_seconds = 30
|
initial_delay_seconds = 30
|
||||||
period_seconds = 30
|
period_seconds = 30
|
||||||
|
timeout_seconds = 5
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Binary file not shown.
Loading…
Add table
Add a link
Reference in a new issue