From aba061cf2e6efa607c74ab90b9a92723fac2e762 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Thu, 21 May 2026 08:27:34 +0000 Subject: [PATCH] alloy: switch pod log shipping from apiserver to file-tail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaced 'loki.source.kubernetes' with 'loki.source.file' in alloy DS config. discovery.relabel.pod_logs already sets __path__ to the kubelet log path (/var/log/pods/*//*.log) and varlog host-mount was already present, so this is a one-line swap. Why: apiserver was burning ~700m sustained on 'CONNECT pods/log' streams (13 req/s, ~2200 sec/s of long-lived TCP connections). Streaming pod logs through the apiserver instead of tailing kubelet's log files was the dominant residual cost after the recent Loki/Alloy onboarding. Measured before/after: - Alloy DS: ~620m total (5 x ~125m) -> ~92m total (5 x ~18m) - kube-apiserver: peak 1959m midnight burst, settled 632m (Stuck-pod recovery: alloy-7zg7t on k8s-master needed --force delete during rollout — FailedKillPod 'unable to signal init: permission denied' on runc, transient runtime issue, unrelated to this change.) --- stacks/monitoring/modules/monitoring/alloy.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/stacks/monitoring/modules/monitoring/alloy.yaml b/stacks/monitoring/modules/monitoring/alloy.yaml index 1c7c7ad7..900cbb25 100644 --- a/stacks/monitoring/modules/monitoring/alloy.yaml +++ b/stacks/monitoring/modules/monitoring/alloy.yaml @@ -81,8 +81,11 @@ alloy: } } - // loki.source.kubernetes tails logs from Kubernetes containers using the Kubernetes API. - loki.source.kubernetes "pod_logs" { + // loki.source.file tails pod logs from /var/log/pods/* on the host filesystem. + // Previously used loki.source.kubernetes (apiserver streaming) which drove + // kube-apiserver `CONNECT pods/log` to ~13 req/s + ~2200 sec/s of streams. + // discovery.relabel.pod_logs already sets __path__ to the kubelet log path. + loki.source.file "pod_logs" { targets = discovery.relabel.pod_logs.output forward_to = [loki.process.pod_logs.receiver] }