2026-03-07 14:30:36 +00:00
|
|
|
variable "tls_secret_name" {
|
2026-03-14 08:51:45 +00:00
|
|
|
type = string
|
2026-03-07 14:30:36 +00:00
|
|
|
sensitive = true
|
|
|
|
|
}
|
2026-02-22 13:56:34 +00:00
|
|
|
|
|
|
|
|
|
2026-02-22 15:13:55 +00:00
|
|
|
module "tls_secret" {
|
|
|
|
|
source = "../../modules/kubernetes/setup_tls_secret"
|
|
|
|
|
namespace = kubernetes_namespace.dashy.metadata[0].name
|
2026-02-22 14:38:14 +00:00
|
|
|
tls_secret_name = var.tls_secret_name
|
2026-02-22 15:13:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
resource "kubernetes_namespace" "dashy" {
|
|
|
|
|
metadata {
|
|
|
|
|
name = "dashy"
|
|
|
|
|
labels = {
|
|
|
|
|
"istio-injection" : "disabled"
|
|
|
|
|
tier = local.tiers.aux
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
resource "kubernetes_config_map" "config" {
|
|
|
|
|
metadata {
|
|
|
|
|
name = "config"
|
|
|
|
|
namespace = kubernetes_namespace.dashy.metadata[0].name
|
|
|
|
|
|
|
|
|
|
annotations = {
|
|
|
|
|
"reloader.stakater.com/match" = "true"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
data = {
|
|
|
|
|
"conf.yml" = file("${path.module}/conf.yml")
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
resource "kubernetes_deployment" "dashy" {
|
|
|
|
|
metadata {
|
|
|
|
|
name = "dashy"
|
|
|
|
|
namespace = kubernetes_namespace.dashy.metadata[0].name
|
|
|
|
|
labels = {
|
|
|
|
|
app = "dashy"
|
|
|
|
|
tier = local.tiers.aux
|
|
|
|
|
}
|
|
|
|
|
annotations = {
|
|
|
|
|
"reloader.stakater.com/search" = "true"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
spec {
|
2026-03-14 21:12:44 +00:00
|
|
|
# Disabled: reduce cluster memory pressure (2026-03-14 OOM incident)
|
|
|
|
|
replicas = 0
|
2026-02-22 15:13:55 +00:00
|
|
|
selector {
|
|
|
|
|
match_labels = {
|
|
|
|
|
app = "dashy"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
template {
|
|
|
|
|
metadata {
|
|
|
|
|
annotations = {
|
|
|
|
|
# "diun.enable" = "true"
|
|
|
|
|
}
|
|
|
|
|
labels = {
|
|
|
|
|
app = "dashy"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
spec {
|
|
|
|
|
container {
|
|
|
|
|
image = "lissy93/dashy:latest"
|
|
|
|
|
name = "dashy"
|
|
|
|
|
|
|
|
|
|
resources {
|
|
|
|
|
requests = {
|
Right-size CPU requests cluster-wide and remove missed CPU limits
Increase requests for under-requested pods (dashy 50m→250m, frigate 500m→1500m,
clickhouse 100m→500m, otp 100m→300m, linkwarden 25m→50m, authentik worker 50m→100m).
Reduce requests for over-requested pods (crowdsec agent/lapi 500m→25m each,
prometheus 200m→100m, dbaas mysql 1800m→100m, pg-cluster 250m→50m,
shlink-web 250m→10m, gpu-pod-exporter 50m→10m, stirling-pdf 100m→25m,
technitium 100m→25m, celery 50m→15m). Reduce crowdsec quota from 8→1 CPU.
Remove missed CPU limits in prometheus (cpu: "2") and dbaas (cpu: "3600m") tpl files.
2026-03-14 09:22:24 +00:00
|
|
|
cpu = "250m"
|
right-size memory: set requests=limits based on actual usage
- Set memory requests = limits across 56 stacks to prevent overcommit
- Right-sized limits based on actual pod usage (2x actual, rounded up)
- Scaled down trading-bot (replicas=0) to free memory
- Fixed OOMKilled services: forgejo, dawarich, health, meshcentral,
paperless-ngx, vault auto-unseal, rybbit, whisper, openclaw, clickhouse
- Added startup+liveness probes to calibre-web
- Bumped inotify limits on nodes 2,3 (max_user_instances 128->8192)
Post node2 OOM incident (2026-03-14). Previous kubelet config had no
kubeReserved/systemReserved set, allowing pods to starve the kernel.
2026-03-14 21:01:24 +00:00
|
|
|
memory = "1Gi"
|
2026-02-22 15:13:55 +00:00
|
|
|
}
|
|
|
|
|
limits = {
|
resource quota review: fix OOM risks, close quota gaps, add HA protections
Phase 1 - OOM fixes:
- dashy: increase memory limit 512Mi→1Gi (was at 99% utilization)
- caretta DaemonSet: set explicit resources 300Mi/512Mi (was at 85-98%)
- mysql-operator: add Helm resource values 256Mi/512Mi, create namespace
with tier label (was at 92% of LimitRange default)
- prowlarr, flaresolverr, annas-archive-stacks: add explicit resources
(outgrowing 256Mi LimitRange defaults)
- real-estate-crawler celery: add resources 512Mi/3Gi (608Mi actual, no
explicit resources)
Phase 2 - Close quota gaps:
- nvidia, real-estate-crawler, trading-bot: remove custom-quota=true
labels so Kyverno generates tier-appropriate quotas
- descheduler: add tier=1-cluster label for proper classification
Phase 3 - Reduce excessive quotas:
- monitoring: limits.memory 240Gi→64Gi, limits.cpu 120→64
- woodpecker: limits.memory 128Gi→32Gi, limits.cpu 64→16
- GPU tier default: limits.memory 96Gi→32Gi, limits.cpu 48→16
Phase 4 - Kubelet protection:
- Add cpu: 200m to systemReserved and kubeReserved in kubelet template
Phase 5 - HA improvements:
- cloudflared: add topology spread (ScheduleAnyway) + PDB (maxUnavailable:1)
- grafana: add topology spread + PDB via Helm values
- crowdsec LAPI: add topology spread + PDB via Helm values
- authentik server: add topology spread via Helm values
- authentik worker: add topology spread + PDB via Helm values
2026-03-08 18:17:46 +00:00
|
|
|
memory = "1Gi"
|
2026-02-22 15:13:55 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
port {
|
|
|
|
|
container_port = 8080
|
|
|
|
|
}
|
|
|
|
|
volume_mount {
|
|
|
|
|
name = "config"
|
|
|
|
|
mount_path = "/app/user-data/"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
volume {
|
|
|
|
|
name = "config"
|
|
|
|
|
config_map {
|
|
|
|
|
name = "config"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
resource "kubernetes_service" "dashy" {
|
|
|
|
|
metadata {
|
|
|
|
|
name = "dashy"
|
|
|
|
|
namespace = kubernetes_namespace.dashy.metadata[0].name
|
|
|
|
|
labels = {
|
|
|
|
|
app = "dashy"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
spec {
|
|
|
|
|
selector = {
|
|
|
|
|
app = "dashy"
|
|
|
|
|
}
|
|
|
|
|
port {
|
|
|
|
|
name = "http"
|
|
|
|
|
port = 80
|
|
|
|
|
target_port = 8080
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
module "ingress" {
|
|
|
|
|
source = "../../modules/kubernetes/ingress_factory"
|
|
|
|
|
namespace = kubernetes_namespace.dashy.metadata[0].name
|
|
|
|
|
name = "dashy"
|
|
|
|
|
tls_secret_name = var.tls_secret_name
|
|
|
|
|
protected = true # hidden as we use homepage now
|
2026-02-22 13:56:34 +00:00
|
|
|
}
|