ce79bd5c04
- Add journald collection to Alloy (loki.source.journal) for kernel OOM, panic, hung task, and soft lockup detection — ships system logs off-node so they survive hard resets - Add 5 Loki alerting rules (KernelOOMKiller, KernelPanic, KernelHungTask, KernelSoftLockup, ContainerdDown) evaluating against node-journal logs - Fix Loki ruler config: correct rules mount path (/var/loki/rules/fake), add alertmanager_url and enable_api - Add Prometheus alerts: NodeMemoryPressureTrending (>85%), NodeExporterDown, NodeHighIOWait (>30%) - Add caretta tolerations for control-plane and GPU nodes - Scale down chromium-based services to 0 for cluster stability: f1-stream, flaresolverr, changedetection, resume/printer
74 lines
1.5 KiB
HCL
74 lines
1.5 KiB
HCL
resource "helm_release" "caretta" {
|
|
namespace = kubernetes_namespace.monitoring.metadata[0].name
|
|
create_namespace = true
|
|
name = "caretta"
|
|
|
|
repository = "https://helm.groundcover.com/"
|
|
chart = "caretta"
|
|
version = "0.0.16"
|
|
|
|
values = [yamlencode({
|
|
grafana = {
|
|
enabled = false
|
|
}
|
|
victoria-metrics-single = {
|
|
enabled = false
|
|
}
|
|
tolerations = [
|
|
{
|
|
key = "node-role.kubernetes.io/control-plane"
|
|
operator = "Exists"
|
|
effect = "NoSchedule"
|
|
},
|
|
{
|
|
key = "nvidia.com/gpu"
|
|
operator = "Exists"
|
|
effect = "NoSchedule"
|
|
}
|
|
]
|
|
resources = {
|
|
requests = {
|
|
cpu = "10m"
|
|
memory = "300Mi"
|
|
}
|
|
limits = {
|
|
cpu = "200m"
|
|
memory = "512Mi"
|
|
}
|
|
}
|
|
})]
|
|
}
|
|
|
|
resource "kubernetes_service" "caretta_metrics" {
|
|
metadata {
|
|
name = "caretta-metrics"
|
|
namespace = kubernetes_namespace.monitoring.metadata[0].name
|
|
labels = {
|
|
app = "caretta"
|
|
}
|
|
}
|
|
spec {
|
|
selector = {
|
|
app = "caretta"
|
|
}
|
|
port {
|
|
name = "metrics"
|
|
port = 7117
|
|
target_port = 7117
|
|
protocol = "TCP"
|
|
}
|
|
}
|
|
}
|
|
|
|
resource "kubernetes_config_map" "caretta_grafana_dashboard" {
|
|
metadata {
|
|
name = "caretta-grafana-dashboard"
|
|
namespace = kubernetes_namespace.monitoring.metadata[0].name
|
|
labels = {
|
|
grafana_dashboard = "1"
|
|
}
|
|
}
|
|
data = {
|
|
"caretta-dashboard.json" = file("${path.module}/dashboards/caretta-dashboard.json")
|
|
}
|
|
}
|