fix: eliminate memory overcommit to prevent node OOM crashes
Set requests = limits (Guaranteed QoS) across LimitRange defaults and explicit pod resources. Node2 crashed 2026-03-14 from 250% memory overcommit (61GB limits on 24GB node). Changes: - LimitRange: default = defaultRequest for all 6 tiers - Grafana: 3 → 2 replicas - Grampsweb: document why replicas=0 - Prometheus: 1Gi/4Gi → 3Gi/3Gi - OpenClaw: 512Mi/2Gi → 768Mi/768Mi - Immich server: 256Mi/2Gi → 512Mi/512Mi - Immich postgresql: 256Mi/1Gi → 512Mi/512Mi - Calibre: 256Mi/1536Mi → 256Mi/256Mi - Linkwarden: 256Mi/1536Mi → 768Mi/768Mi - N8N: 256Mi/1Gi → 512Mi/512Mi - MySQL cluster: 1Gi/3-4Gi → 2Gi/2Gi - pg-cluster (CNPG): 512Mi/4Gi → 512Mi/512Mi - DBaaS ResourceQuota limits.memory: 64Gi → 12Gi [ci skip]
This commit is contained in:
Viktor Barzin
parent
27fa8ea18f
commit
2be858f616
10 changed files with 70 additions and 30 deletions
|
|
@ -36,7 +36,7 @@ resource "kubernetes_resource_quota" "dbaas" {
|
|||
hard = {
|
||||
"requests.cpu" = "8"
|
||||
"requests.memory" = "12Gi"
|
||||
"limits.memory" = "64Gi"
|
||||
"limits.memory" = "12Gi"
|
||||
pods = "30"
|
||||
}
|
||||
}
|
||||
|
|
@ -181,10 +181,10 @@ resource "helm_release" "mysql_cluster" {
|
|||
resources = {
|
||||
requests = {
|
||||
cpu = "250m"
|
||||
memory = "1Gi"
|
||||
memory = "2Gi"
|
||||
}
|
||||
limits = {
|
||||
memory = "4Gi"
|
||||
memory = "2Gi"
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -216,11 +216,11 @@ resource "helm_release" "mysql_cluster" {
|
|||
name = "mysql"
|
||||
resources = {
|
||||
requests = {
|
||||
memory = "1Gi"
|
||||
memory = "2Gi"
|
||||
cpu = "250m"
|
||||
}
|
||||
limits = {
|
||||
memory = "3Gi"
|
||||
memory = "2Gi"
|
||||
}
|
||||
}
|
||||
}]
|
||||
|
|
@ -229,21 +229,21 @@ resource "helm_release" "mysql_cluster" {
|
|||
name = "fixdatadir"
|
||||
resources = {
|
||||
requests = { memory = "64Mi", cpu = "25m" }
|
||||
limits = { memory = "256Mi" }
|
||||
limits = { memory = "64Mi" }
|
||||
}
|
||||
},
|
||||
{
|
||||
name = "initconf"
|
||||
resources = {
|
||||
requests = { memory = "256Mi", cpu = "50m" }
|
||||
limits = { memory = "1Gi" }
|
||||
limits = { memory = "256Mi" }
|
||||
}
|
||||
},
|
||||
{
|
||||
name = "initmysql"
|
||||
resources = {
|
||||
requests = { memory = "512Mi", cpu = "250m" }
|
||||
limits = { memory = "2Gi" }
|
||||
limits = { memory = "512Mi" }
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
@ -842,7 +842,7 @@ resource "null_resource" "pg_cluster" {
|
|||
image = "ghcr.io/cloudnative-pg/postgis:16"
|
||||
storage_size = "20Gi"
|
||||
storage_class = "iscsi-truenas"
|
||||
memory_limit = "4Gi"
|
||||
memory_limit = "512Mi"
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -870,7 +870,7 @@ resource "null_resource" "pg_cluster" {
|
|||
cpu: "50m"
|
||||
memory: "512Mi"
|
||||
limits:
|
||||
memory: "4Gi"
|
||||
memory: "512Mi"
|
||||
EOF
|
||||
EOT
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,10 @@
|
|||
# =============================================================================
|
||||
# Tier-Based Resource Governance
|
||||
# =============================================================================
|
||||
# default (limit) = defaultRequest (request) to give Guaranteed QoS and prevent
|
||||
# memory overcommit. Changed 2026-03-14 after node2 OOM crash caused by 250%
|
||||
# memory overcommit (61GB limits on 24GB node).
|
||||
#
|
||||
# Four layers of protection against noisy neighbor issues:
|
||||
# 1. PriorityClasses - critical services survive resource pressure
|
||||
# 2. LimitRange defaults (Kyverno generate) - auto-inject defaults for containers without resources
|
||||
|
|
@ -130,7 +134,7 @@ resource "kubernetes_manifest" "generate_limitrange_by_tier" {
|
|||
{
|
||||
type = "Container"
|
||||
default = {
|
||||
memory = "512Mi"
|
||||
memory = "256Mi"
|
||||
}
|
||||
defaultRequest = {
|
||||
cpu = "100m"
|
||||
|
|
@ -187,7 +191,7 @@ resource "kubernetes_manifest" "generate_limitrange_by_tier" {
|
|||
{
|
||||
type = "Container"
|
||||
default = {
|
||||
memory = "512Mi"
|
||||
memory = "256Mi"
|
||||
}
|
||||
defaultRequest = {
|
||||
cpu = "100m"
|
||||
|
|
@ -244,7 +248,7 @@ resource "kubernetes_manifest" "generate_limitrange_by_tier" {
|
|||
{
|
||||
type = "Container"
|
||||
default = {
|
||||
memory = "2Gi"
|
||||
memory = "1Gi"
|
||||
}
|
||||
defaultRequest = {
|
||||
cpu = "200m"
|
||||
|
|
@ -301,7 +305,7 @@ resource "kubernetes_manifest" "generate_limitrange_by_tier" {
|
|||
{
|
||||
type = "Container"
|
||||
default = {
|
||||
memory = "256Mi"
|
||||
memory = "128Mi"
|
||||
}
|
||||
defaultRequest = {
|
||||
cpu = "50m"
|
||||
|
|
@ -358,7 +362,7 @@ resource "kubernetes_manifest" "generate_limitrange_by_tier" {
|
|||
{
|
||||
type = "Container"
|
||||
default = {
|
||||
memory = "256Mi"
|
||||
memory = "128Mi"
|
||||
}
|
||||
defaultRequest = {
|
||||
cpu = "50m"
|
||||
|
|
@ -374,6 +378,7 @@ resource "kubernetes_manifest" "generate_limitrange_by_tier" {
|
|||
}
|
||||
},
|
||||
# Fallback: namespaces without a tier label get aux-level defaults
|
||||
# requests = limits to prevent memory overcommit (2026-03-14 node2 OOM incident)
|
||||
{
|
||||
name = "limitrange-no-tier-fallback"
|
||||
match = {
|
||||
|
|
@ -418,7 +423,7 @@ resource "kubernetes_manifest" "generate_limitrange_by_tier" {
|
|||
{
|
||||
type = "Container"
|
||||
default = {
|
||||
memory = "256Mi"
|
||||
memory = "128Mi"
|
||||
}
|
||||
defaultRequest = {
|
||||
cpu = "50m"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
deploymentStrategy:
|
||||
type: RollingUpdate
|
||||
replicas: 3
|
||||
replicas: 2
|
||||
adminPassword: "${grafana_admin_password}"
|
||||
resources:
|
||||
requests:
|
||||
|
|
|
|||
|
|
@ -141,9 +141,9 @@ server:
|
|||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 1Gi
|
||||
memory: 3Gi
|
||||
limits:
|
||||
memory: 4Gi
|
||||
memory: 3Gi
|
||||
strategy:
|
||||
type: Recreate
|
||||
baseURL: "https://prometheus.viktorbarzin.me"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue