viktor/infra
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

backup & DR: add alerting, fix rotation, secure MySQL password, add runbooks

Browse source 
Phase 1: Add 12 PrometheusRules for backup health alerting
- PostgreSQL, MySQL, Vault, Vaultwarden, Redis staleness + never-succeeded alerts
- CSIDriverCrashLoop alert for nfs-csi/iscsi-csi namespaces
- Generic BackupCronJobFailed alert

Phase 2: Fix backup rotation
- etcd: timestamped snapshots instead of overwriting single file
- Redis: timestamped RDB files with 7-day retention purge
- PostgreSQL: retention increased from 7 to 14 days

Phase 3: Fix MySQL password exposure
- Move root password from command line arg to MYSQL_PWD env var via secretKeyRef

Phase 5: Add restore runbooks
- PostgreSQL, MySQL, Vault, etcd, Vaultwarden, full cluster rebuild
This commit is contained in:
Viktor Barzin 2026-03-19 20:34:33 +00:00
parent 62d42657e6
commit af2222fce8
9 changed files with 657 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

84
stacks/monitoring/modules/monitoring/prometheus_chart_values.tpl
View file

@ -502,6 +502,90 @@ serverFiles:
severity: critical
annotations:
summary: "etcd backup CronJob has never completed successfully"
- alert: PostgreSQLBackupStale
expr: (time() - kube_cronjob_status_last_successful_time{cronjob="postgresql-backup", namespace="dbaas"}) > 129600
for: 30m
labels:
severity: critical
annotations:
summary: "PostgreSQL backup is {{ $value | humanizeDuration }} old (threshold: 36h)"
- alert: PostgreSQLBackupNeverSucceeded
expr: kube_cronjob_status_last_successful_time{cronjob="postgresql-backup", namespace="dbaas"} == 0
for: 1h
labels:
severity: critical
annotations:
summary: "PostgreSQL backup CronJob has never completed successfully"
- alert: MySQLBackupStale
expr: (time() - kube_cronjob_status_last_successful_time{cronjob="mysql-backup", namespace="dbaas"}) > 129600
for: 30m
labels:
severity: critical
annotations:
summary: "MySQL backup is {{ $value | humanizeDuration }} old (threshold: 36h)"
- alert: MySQLBackupNeverSucceeded
expr: kube_cronjob_status_last_successful_time{cronjob="mysql-backup", namespace="dbaas"} == 0
for: 1h
labels:
severity: critical
annotations:
summary: "MySQL backup CronJob has never completed successfully"
- alert: VaultBackupStale
expr: (time() - kube_cronjob_status_last_successful_time{cronjob="vault-raft-backup", namespace="vault"}) > 129600
for: 30m
labels:
severity: critical
annotations:
summary: "Vault backup is {{ $value | humanizeDuration }} old (threshold: 36h)"
- alert: VaultBackupNeverSucceeded
expr: kube_cronjob_status_last_successful_time{cronjob="vault-raft-backup", namespace="vault"} == 0
for: 1h
labels:
severity: critical
annotations:
summary: "Vault backup CronJob has never completed successfully"
- alert: VaultwardenBackupStale
expr: (time() - kube_cronjob_status_last_successful_time{cronjob="vaultwarden-backup", namespace="vaultwarden"}) > 129600
for: 30m
labels:
severity: critical
annotations:
summary: "Vaultwarden backup is {{ $value | humanizeDuration }} old (threshold: 36h)"
- alert: VaultwardenBackupNeverSucceeded
expr: kube_cronjob_status_last_successful_time{cronjob="vaultwarden-backup", namespace="vaultwarden"} == 0
for: 1h
labels:
severity: critical
annotations:
summary: "Vaultwarden backup CronJob has never completed successfully"
- alert: RedisBackupStale
expr: (time() - kube_cronjob_status_last_successful_time{cronjob="redis-backup", namespace="redis"}) > 14400
for: 30m
labels:
severity: critical
annotations:
summary: "Redis backup is {{ $value | humanizeDuration }} old (threshold: 4h)"
- alert: RedisBackupNeverSucceeded
expr: kube_cronjob_status_last_successful_time{cronjob="redis-backup", namespace="redis"} == 0
for: 1h
labels:
severity: critical
annotations:
summary: "Redis backup CronJob has never completed successfully"
- alert: CSIDriverCrashLoop
expr: kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", namespace=~"nfs-csi|iscsi-csi"} > 0
for: 10m
labels:
severity: critical
annotations:
summary: "CSI driver CrashLoopBackOff in {{ $labels.namespace }}/{{ $labels.pod }} — storage-layer failure risk"
- alert: BackupCronJobFailed
expr: kube_job_status_failed{job_name=~".*backup.*"} > 0
for: 15m
labels:
severity: warning
annotations:
summary: "Backup job failed: {{ $labels.namespace }}/{{ $labels.job_name }}"
- alert: NewTailscaleClient
expr: irate(headscale_machine_registrations_total{action="reauth"}[5m]) > 0
for: 5m