dd2a8e640f
monitoring-quota requests.memory sat at 89% (18.2/20Gi), tripping the ResourceQuota>80% WARN. Root cause was over-provisioned requests, not real usage: loki requested 3Gi but its VPA upperBound is 364Mi and actual ~315Mi. prometheus's 4Gi is legitimately required (2Gi tmpfs WAL shares the cgroup; OOMs at 3Gi during WAL replay) so it stays; grafana's main container is already 512Mi. Trimmed loki to 1Gi request (~3x its observed ceiling; 4Gi Burstable limit preserves query-spike headroom) -> quota 78.8%, clears the WARN. NOTE: alloy DaemonSet (562Mi/node) grows with node count, so revisit (bump the 20Gi quota) as the cluster expands.
115 lines
2.5 KiB
YAML
115 lines
2.5 KiB
YAML
loki:
|
|
commonConfig:
|
|
replication_factor: 1
|
|
schemaConfig:
|
|
configs:
|
|
- from: "2025-04-01"
|
|
store: tsdb
|
|
object_store: filesystem
|
|
schema: v13
|
|
index:
|
|
prefix: loki_index_
|
|
period: 24h
|
|
ingester:
|
|
chunk_idle_period: 12h
|
|
max_chunk_age: 24h
|
|
chunk_retain_period: 1m
|
|
chunk_target_size: 1572864
|
|
wal:
|
|
dir: /loki-wal
|
|
pattern_ingester:
|
|
enabled: true
|
|
limits_config:
|
|
allow_structured_metadata: true
|
|
volume_enabled: true
|
|
retention_period: 720h
|
|
compactor:
|
|
retention_enabled: true
|
|
working_directory: /var/loki/compactor
|
|
compaction_interval: 1h
|
|
delete_request_store: filesystem
|
|
ruler:
|
|
enable_api: true
|
|
storage:
|
|
type: local
|
|
local:
|
|
directory: /var/loki/rules
|
|
alertmanager_url: http://prometheus-alertmanager.monitoring.svc.cluster.local:9093
|
|
ring:
|
|
kvstore:
|
|
store: inmemory
|
|
rule_path: /var/loki/scratch
|
|
storage:
|
|
type: "filesystem"
|
|
auth_enabled: false
|
|
|
|
minio:
|
|
enabled: false
|
|
|
|
deploymentMode: SingleBinary
|
|
|
|
singleBinary:
|
|
replicas: 1
|
|
persistence:
|
|
enabled: true
|
|
size: 50Gi
|
|
storageClass: "proxmox-lvm"
|
|
extraVolumes:
|
|
- name: wal
|
|
emptyDir:
|
|
medium: Memory
|
|
sizeLimit: 2Gi
|
|
- name: rules
|
|
configMap:
|
|
name: loki-alert-rules
|
|
extraVolumeMounts:
|
|
- name: wal
|
|
mountPath: /loki-wal
|
|
- name: rules
|
|
mountPath: /var/loki/rules/fake
|
|
resources:
|
|
requests:
|
|
cpu: 250m
|
|
# Right-sized 2026-06-04 (3Gi->1Gi): VPA upperBound 364Mi, actual ~315Mi.
|
|
# 1Gi request is ~3x the observed ceiling; the 4Gi limit (Burstable)
|
|
# keeps headroom for query spikes. Frees 2Gi of monitoring-quota
|
|
# requests.memory, taking it 89%->~79% (under the >80% WARN). NOTE: the
|
|
# alloy DaemonSet (562Mi/node) grows with node count, so this can creep
|
|
# back over 80% as the cluster expands — bump the quota then.
|
|
memory: 1Gi
|
|
limits:
|
|
memory: 4Gi
|
|
|
|
# Zero out replica counts of other deployment modes
|
|
backend:
|
|
replicas: 0
|
|
read:
|
|
replicas: 0
|
|
write:
|
|
replicas: 0
|
|
ingester:
|
|
replicas: 0
|
|
querier:
|
|
replicas: 0
|
|
queryFrontend:
|
|
replicas: 0
|
|
queryScheduler:
|
|
replicas: 0
|
|
distributor:
|
|
replicas: 0
|
|
compactor:
|
|
replicas: 0
|
|
indexGateway:
|
|
replicas: 0
|
|
bloomCompactor:
|
|
replicas: 0
|
|
bloomGateway:
|
|
replicas: 0
|
|
|
|
# Disable optional components for single binary mode
|
|
gateway:
|
|
enabled: false
|
|
chunksCache:
|
|
enabled: false
|
|
resultsCache:
|
|
enabled: false
|