2026-03-17 18:11:53 +00:00
|
|
|
|
resource "kubernetes_config_map" "pgbouncer_config" {
|
|
|
|
|
|
metadata {
|
|
|
|
|
|
name = "pgbouncer-config"
|
|
|
|
|
|
namespace = "authentik"
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
data = {
|
|
|
|
|
|
"pgbouncer.ini" = templatefile("${path.module}/pgbouncer.ini", { password = var.postgres_password })
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
# --- 2️⃣ Secret for user credentials ---
|
|
|
|
|
|
resource "kubernetes_secret" "pgbouncer_auth" {
|
|
|
|
|
|
metadata {
|
|
|
|
|
|
name = "pgbouncer-auth"
|
|
|
|
|
|
namespace = "authentik"
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
data = {
|
|
|
|
|
|
"userlist.txt" = templatefile("${path.module}/userlist.txt", { password = var.postgres_password })
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
type = "Opaque"
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
# --- 3️⃣ Deployment ---
|
|
|
|
|
|
resource "kubernetes_deployment" "pgbouncer" {
|
|
|
|
|
|
metadata {
|
|
|
|
|
|
name = "pgbouncer"
|
|
|
|
|
|
namespace = "authentik"
|
|
|
|
|
|
labels = {
|
|
|
|
|
|
app = "pgbouncer"
|
|
|
|
|
|
tier = var.tier
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
spec {
|
|
|
|
|
|
replicas = 3
|
|
|
|
|
|
|
|
|
|
|
|
selector {
|
|
|
|
|
|
match_labels = {
|
|
|
|
|
|
app = "pgbouncer"
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
template {
|
|
|
|
|
|
metadata {
|
|
|
|
|
|
labels = {
|
|
|
|
|
|
app = "pgbouncer"
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
spec {
|
|
|
|
|
|
affinity {
|
|
|
|
|
|
pod_anti_affinity {
|
|
|
|
|
|
required_during_scheduling_ignored_during_execution {
|
|
|
|
|
|
label_selector {
|
|
|
|
|
|
match_expressions {
|
|
|
|
|
|
key = "component"
|
|
|
|
|
|
operator = "In"
|
|
|
|
|
|
values = ["server"]
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
topology_key = "kubernetes.io/hostname"
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
container {
|
2026-05-10 16:22:24 +00:00
|
|
|
|
name = "pgbouncer"
|
|
|
|
|
|
image = "edoburu/pgbouncer:latest"
|
|
|
|
|
|
# `:latest` tag — keep `Always` so pod restarts pick up upstream
|
|
|
|
|
|
# updates. The previous `IfNotPresent` value was declared at module
|
|
|
|
|
|
# creation but the live cluster has reconciled to `Always` (likely
|
|
|
|
|
|
# via a Helm/operator default). Match reality to drop the drift.
|
|
|
|
|
|
image_pull_policy = "Always"
|
2026-03-17 18:11:53 +00:00
|
|
|
|
|
|
|
|
|
|
port {
|
|
|
|
|
|
container_port = 6432
|
|
|
|
|
|
}
|
|
|
|
|
|
|
[authentik] Phase 1 hardening — 3 replicas, PgBouncer PDB/probes, perf env
## Context
Following the 2026-04-18 /dev/shm ENOSPC P0 and a 5-subagent research pass,
this is Phase 1 of the authentik reliability + performance hardening epic
(beads code-cwj). Scope: everything that is safe, additive, and does not
require DB restart, architectural migration, or the 43-service auth path
to go through a risky validation window.
Five research findings drove the deltas:
1. **Server/worker at 2 replicas** conflicts with the documented convention
"critical path services scaled to 3" in .claude/CLAUDE.md (Traefik,
Authentik, CrowdSec LAPI, PgBouncer, Cloudflared). PDB minAvailable was
still 1 — a single-pod outage could take auth down.
2. **PgBouncer had no resource requests/limits** — silently capped at the
Kyverno tier-defaults LimitRange (256Mi), no PDB, no probes. Pool
failures undetected until connection timeouts.
3. **Authentik 2026.2 has no Redis** (the cache moved to Postgres in
2025.10). Persistent Django connections + longer flow/policy cache TTLs
are the two knobs that move the needle most without DB tuning. Both are
safe because PgBouncer runs in session mode.
4. **Gunicorn defaults** (2 workers × 4 threads on server, 1 process × 2
threads on worker) don't use the pod's 1.5 Gi headroom. Each worker
preloads Django at ~500 MiB — bumping to 3 workers needs a memory bump
to 2 Gi first.
5. **AUTHENTIK_WORKER__CONCURRENCY was renamed AUTHENTIK_WORKER__THREADS**
in 2025.8 — the old name is aliased but the canonical config key changed.
## This change
### values.yaml
- server.replicas 2 → 3 (PDB minAvailable 1 → 2)
- worker.replicas 2 → 3
- server/worker limits.memory 1.5 Gi → 2 Gi (headroom for gunicorn workers)
- authentik.postgresql.conn_max_age = 60 (persistent connections; safe
with pgbouncer session mode, conn_max_age < server_idle_timeout=600s)
- authentik.postgresql.conn_health_checks = true
- authentik.cache.timeout_flows = 1800 (30 min; was 300)
- authentik.cache.timeout_policies = 900 (15 min; was 300)
- authentik.web.workers = 3, threads = 4
- authentik.worker.threads = 4 (was 2)
### pgbouncer.tf
- container resources: requests cpu=50m/mem=128Mi, limits mem=512Mi
(observed live usage is 1-3 m CPU, 2-4 MiB RSS — huge headroom,
safely above Kyverno 256Mi tier-default cap)
- readiness probe: TCP :6432, 10s period
- liveness probe: TCP :6432, 30s period, 30s delay
- kubernetes_pod_disruption_budget_v1.pgbouncer: minAvailable=2
(3 replicas; single drain rolls cleanly, two-node simultaneous
outage correctly blocked)
## What is NOT in this change (deferred as Phase 2 follow-ups)
- Codify outpost /dev/shm patch in Terraform (currently applied via
Authentik API, not in code). Needs authentik_outpost resource.
- Migrate embedded outpost → dedicated outpost Deployment with 2
replicas + sticky sessions. Only HA path per GH issue #18098; requires
flow design because outpost sessions are in-process memory only.
- PG max_connections 100 → 200 + shared_buffers 512MB → 768MB + CNPG
pod memory 2Gi → 3Gi. Needs coordinated DB restart.
- Enable pg_stat_statements on CNPG cluster for Authentik DB
observability (currently shared_preload_libraries is empty).
- PgBouncer pool_mode session → transaction + django_channels layer
split. Needs atomic change + psycopg3 prepared-statement support.
- authentik_tasks_tasklog 7-day retention (198k rows, unbounded).
- Traefik forward-auth plugin caching via
xabinapal/traefik-authentik-forward-plugin.
- Grafana dashboard 14837 import + recording rule for
authentik_flow_execution_duration (reported broken: values in ns
while default buckets are seconds — upstream discussion #7156).
## Test plan
### Automated
$ cd stacks/authentik && ../../scripts/tg plan
Plan: 1 to add, 3 to change, 0 to destroy.
$ ../../scripts/tg apply --non-interactive
module.authentik.kubernetes_pod_disruption_budget_v1.pgbouncer: Creation complete after 0s
module.authentik.kubernetes_deployment.pgbouncer: Modifications complete after 45s
module.authentik.helm_release.authentik: Modifications complete after 2m47s
Apply complete! Resources: 1 added, 3 changed, 0 destroyed.
### Manual Verification
1. **Pod topology and PDBs**:
$ kubectl -n authentik get pods,pdb
pod/goauthentik-server-5fc69b6cc6-ctvkp 1/1 Running 0 3m14s k8s-node2
pod/goauthentik-server-5fc69b6cc6-fkn8x 1/1 Running 0 3m45s k8s-node3
pod/goauthentik-server-5fc69b6cc6-jtjjd 1/1 Running 0 5m6s k8s-node1
pod/goauthentik-worker-5cfb7dc9bf-b2rlr 1/1 Running 0 3m44s k8s-node2
pod/goauthentik-worker-5cfb7dc9bf-fkfm4 1/1 Running 0 5m6s k8s-node1
pod/goauthentik-worker-5cfb7dc9bf-hxdg6 1/1 Running 0 3m3s k8s-node4
pod/pgbouncer-64746f955f-st567 1/1 Running 0 4m58s k8s-node4
pod/pgbouncer-64746f955f-xss9c 1/1 Running 0 5m11s k8s-node2
pod/pgbouncer-64746f955f-zvfkw 1/1 Running 0 4m45s k8s-node3
poddisruptionbudget/goauthentik-server 2 N/A 1
poddisruptionbudget/goauthentik-worker N/A 1 1
poddisruptionbudget/pgbouncer 2 N/A 1
All three workloads spread across 3+ nodes, PDBs allow 1 disruption.
2. **Authentik server health**:
$ curl -sS -o /dev/null -w "%{http_code}\n" \
https://authentik.viktorbarzin.me/-/health/ready/
200
3. **Forward-auth redirect on protected service**:
$ curl -sS -o /dev/null -w "%{http_code}\n" -L \
https://wealthfolio.viktorbarzin.me/
200
4. **Outpost /dev/shm still within sizeLimit** (patches from the
2026-04-18 post-mortem were not regressed):
$ kubectl -n authentik exec deploy/ak-outpost-authentik-embedded-outpost \
-c proxy -- df -h /dev/shm
tmpfs 2.0G 58M 2.0G 3% /dev/shm
5. **PgBouncer port reachable from other pods**:
$ kubectl -n authentik exec deploy/pgbouncer -- nc -zv 127.0.0.1 6432
127.0.0.1 (127.0.0.1:6432) open
## Reproduce locally
1. `cd stacks/authentik && ../../scripts/tg plan` — expect 0/0/0 (No changes).
2. `kubectl -n authentik get pdb pgbouncer` — expect MIN AVAILABLE 2.
3. `kubectl -n authentik get deploy goauthentik-server -o jsonpath='{.spec.replicas}'` — expect 3.
Closes: code-cwj
2026-04-19 11:52:41 +00:00
|
|
|
|
resources {
|
|
|
|
|
|
requests = {
|
|
|
|
|
|
cpu = "50m"
|
|
|
|
|
|
memory = "128Mi"
|
|
|
|
|
|
}
|
|
|
|
|
|
limits = {
|
|
|
|
|
|
memory = "512Mi"
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
readiness_probe {
|
|
|
|
|
|
tcp_socket {
|
|
|
|
|
|
port = 6432
|
|
|
|
|
|
}
|
|
|
|
|
|
initial_delay_seconds = 5
|
|
|
|
|
|
period_seconds = 10
|
|
|
|
|
|
timeout_seconds = 3
|
|
|
|
|
|
failure_threshold = 3
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
liveness_probe {
|
|
|
|
|
|
tcp_socket {
|
|
|
|
|
|
port = 6432
|
|
|
|
|
|
}
|
|
|
|
|
|
initial_delay_seconds = 30
|
|
|
|
|
|
period_seconds = 30
|
|
|
|
|
|
timeout_seconds = 5
|
|
|
|
|
|
failure_threshold = 3
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-03-17 18:11:53 +00:00
|
|
|
|
volume_mount {
|
|
|
|
|
|
name = "config"
|
|
|
|
|
|
mount_path = "/etc/pgbouncer/pgbouncer.ini"
|
|
|
|
|
|
sub_path = "pgbouncer.ini"
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
volume_mount {
|
|
|
|
|
|
name = "auth"
|
|
|
|
|
|
mount_path = "/etc/pgbouncer/userlist.txt"
|
|
|
|
|
|
sub_path = "userlist.txt"
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
env {
|
|
|
|
|
|
name = "DATABASES_AUTHENTIK"
|
|
|
|
|
|
value = "host=postgres port=5432 dbname=authentik user=authentik password=${var.postgres_password}"
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
volume {
|
|
|
|
|
|
name = "config"
|
|
|
|
|
|
config_map {
|
|
|
|
|
|
name = kubernetes_config_map.pgbouncer_config.metadata[0].name
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
volume {
|
|
|
|
|
|
name = "auth"
|
|
|
|
|
|
secret {
|
|
|
|
|
|
secret_name = kubernetes_secret.pgbouncer_auth.metadata[0].name
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
dns_config {
|
|
|
|
|
|
option {
|
|
|
|
|
|
name = "ndots"
|
|
|
|
|
|
value = "2"
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
depends_on = [kubernetes_secret.pgbouncer_auth]
|
[infra] Sweep dns_config ignore_changes across all pod-owning resources [ci skip]
## Context
Wave 3A (commit c9d221d5) added the `# KYVERNO_LIFECYCLE_V1` marker to the
27 pre-existing `ignore_changes = [...dns_config]` sites so they could be
grepped and audited. It did NOT address pod-owning resources that were
simply missing the suppression entirely. Post-Wave-3A sampling (2026-04-18)
found that navidrome, f1-stream, frigate, servarr, monitoring, crowdsec,
and many other stacks showed perpetual `dns_config` drift every plan
because their `kubernetes_deployment` / `kubernetes_stateful_set` /
`kubernetes_cron_job_v1` resources had no `lifecycle {}` block at all.
Root cause (same as Wave 3A): Kyverno's admission webhook stamps
`dns_config { option { name = "ndots"; value = "2" } }` on every pod's
`spec.template.spec.dns_config` to prevent NxDomain search-domain flooding
(see `k8s-ndots-search-domain-nxdomain-flood` skill). Without `ignore_changes`
on every Terraform-managed pod-owner, Terraform repeatedly tries to strip
the injected field.
## This change
Extends the Wave 3A convention by sweeping EVERY `kubernetes_deployment`,
`kubernetes_stateful_set`, `kubernetes_daemon_set`, `kubernetes_cron_job_v1`,
`kubernetes_job_v1` (+ their `_v1` variants) in the repo and ensuring each
carries the right `ignore_changes` path:
- **kubernetes_deployment / stateful_set / daemon_set / job_v1**:
`spec[0].template[0].spec[0].dns_config`
- **kubernetes_cron_job_v1**:
`spec[0].job_template[0].spec[0].template[0].spec[0].dns_config`
(extra `job_template[0]` nesting — the CronJob's PodTemplateSpec is
one level deeper)
Each injection / extension is tagged `# KYVERNO_LIFECYCLE_V1: Kyverno
admission webhook mutates dns_config with ndots=2` inline so the
suppression is discoverable via `rg 'KYVERNO_LIFECYCLE_V1' stacks/`.
Two insertion paths are handled by a Python pass (`/tmp/add_dns_config_ignore.py`):
1. **No existing `lifecycle {}`**: inject a brand-new block just before the
resource's closing `}`. 108 new blocks on 93 files.
2. **Existing `lifecycle {}` (usually for `DRIFT_WORKAROUND: CI owns image tag`
from Wave 4, commit a62b43d1)**: extend its `ignore_changes` list with the
dns_config path. Handles both inline (`= [x]`) and multiline
(`= [\n x,\n]`) forms; ensures the last pre-existing list item carries
a trailing comma so the extended list is valid HCL. 34 extensions.
The script skips anything already mentioning `dns_config` inside an
`ignore_changes`, so re-running is a no-op.
## Scale
- 142 total lifecycle injections/extensions
- 93 `.tf` files touched
- 108 brand-new `lifecycle {}` blocks + 34 extensions of existing ones
- Every Tier 0 and Tier 1 stack with a pod-owning resource is covered
- Together with Wave 3A's 27 pre-existing markers → **169 greppable
`KYVERNO_LIFECYCLE_V1` dns_config sites across the repo**
## What is NOT in this change
- `stacks/trading-bot/main.tf` — entirely commented-out block (`/* … */`).
Python script touched the file, reverted manually.
- `_template/main.tf.example` skeleton — kept minimal on purpose; any
future stack created from it should either inherit the Wave 3A one-line
form or add its own on first `kubernetes_deployment`.
- `terraform fmt` fixes to pre-existing alignment issues in meshcentral,
nvidia/modules/nvidia, vault — unrelated to this commit. Left for a
separate fmt-only pass.
- Non-pod resources (`kubernetes_service`, `kubernetes_secret`,
`kubernetes_manifest`, etc.) — they don't own pods so they don't get
Kyverno dns_config mutation.
## Verification
Random sample post-commit:
```
$ cd stacks/navidrome && ../../scripts/tg plan → No changes.
$ cd stacks/f1-stream && ../../scripts/tg plan → No changes.
$ cd stacks/frigate && ../../scripts/tg plan → No changes.
$ rg -c 'KYVERNO_LIFECYCLE_V1' stacks/ --include='*.tf' --include='*.tf.example' \
| awk -F: '{s+=$2} END {print s}'
169
```
## Reproduce locally
1. `git pull`
2. `rg 'KYVERNO_LIFECYCLE_V1' stacks/ | wc -l` → 169+
3. `cd stacks/navidrome && ../../scripts/tg plan` → expect 0 drift on
the deployment's dns_config field.
Refs: code-seq (Wave 3B dns_config class closed; kubernetes_manifest
annotation class handled separately in 8d94688d for tls_secret)
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-18 21:19:48 +00:00
|
|
|
|
lifecycle {
|
|
|
|
|
|
# KYVERNO_LIFECYCLE_V1: Kyverno admission webhook mutates dns_config with ndots=2
|
|
|
|
|
|
ignore_changes = [spec[0].template[0].spec[0].dns_config]
|
|
|
|
|
|
}
|
2026-03-17 18:11:53 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
[authentik] Phase 1 hardening — 3 replicas, PgBouncer PDB/probes, perf env
## Context
Following the 2026-04-18 /dev/shm ENOSPC P0 and a 5-subagent research pass,
this is Phase 1 of the authentik reliability + performance hardening epic
(beads code-cwj). Scope: everything that is safe, additive, and does not
require DB restart, architectural migration, or the 43-service auth path
to go through a risky validation window.
Five research findings drove the deltas:
1. **Server/worker at 2 replicas** conflicts with the documented convention
"critical path services scaled to 3" in .claude/CLAUDE.md (Traefik,
Authentik, CrowdSec LAPI, PgBouncer, Cloudflared). PDB minAvailable was
still 1 — a single-pod outage could take auth down.
2. **PgBouncer had no resource requests/limits** — silently capped at the
Kyverno tier-defaults LimitRange (256Mi), no PDB, no probes. Pool
failures undetected until connection timeouts.
3. **Authentik 2026.2 has no Redis** (the cache moved to Postgres in
2025.10). Persistent Django connections + longer flow/policy cache TTLs
are the two knobs that move the needle most without DB tuning. Both are
safe because PgBouncer runs in session mode.
4. **Gunicorn defaults** (2 workers × 4 threads on server, 1 process × 2
threads on worker) don't use the pod's 1.5 Gi headroom. Each worker
preloads Django at ~500 MiB — bumping to 3 workers needs a memory bump
to 2 Gi first.
5. **AUTHENTIK_WORKER__CONCURRENCY was renamed AUTHENTIK_WORKER__THREADS**
in 2025.8 — the old name is aliased but the canonical config key changed.
## This change
### values.yaml
- server.replicas 2 → 3 (PDB minAvailable 1 → 2)
- worker.replicas 2 → 3
- server/worker limits.memory 1.5 Gi → 2 Gi (headroom for gunicorn workers)
- authentik.postgresql.conn_max_age = 60 (persistent connections; safe
with pgbouncer session mode, conn_max_age < server_idle_timeout=600s)
- authentik.postgresql.conn_health_checks = true
- authentik.cache.timeout_flows = 1800 (30 min; was 300)
- authentik.cache.timeout_policies = 900 (15 min; was 300)
- authentik.web.workers = 3, threads = 4
- authentik.worker.threads = 4 (was 2)
### pgbouncer.tf
- container resources: requests cpu=50m/mem=128Mi, limits mem=512Mi
(observed live usage is 1-3 m CPU, 2-4 MiB RSS — huge headroom,
safely above Kyverno 256Mi tier-default cap)
- readiness probe: TCP :6432, 10s period
- liveness probe: TCP :6432, 30s period, 30s delay
- kubernetes_pod_disruption_budget_v1.pgbouncer: minAvailable=2
(3 replicas; single drain rolls cleanly, two-node simultaneous
outage correctly blocked)
## What is NOT in this change (deferred as Phase 2 follow-ups)
- Codify outpost /dev/shm patch in Terraform (currently applied via
Authentik API, not in code). Needs authentik_outpost resource.
- Migrate embedded outpost → dedicated outpost Deployment with 2
replicas + sticky sessions. Only HA path per GH issue #18098; requires
flow design because outpost sessions are in-process memory only.
- PG max_connections 100 → 200 + shared_buffers 512MB → 768MB + CNPG
pod memory 2Gi → 3Gi. Needs coordinated DB restart.
- Enable pg_stat_statements on CNPG cluster for Authentik DB
observability (currently shared_preload_libraries is empty).
- PgBouncer pool_mode session → transaction + django_channels layer
split. Needs atomic change + psycopg3 prepared-statement support.
- authentik_tasks_tasklog 7-day retention (198k rows, unbounded).
- Traefik forward-auth plugin caching via
xabinapal/traefik-authentik-forward-plugin.
- Grafana dashboard 14837 import + recording rule for
authentik_flow_execution_duration (reported broken: values in ns
while default buckets are seconds — upstream discussion #7156).
## Test plan
### Automated
$ cd stacks/authentik && ../../scripts/tg plan
Plan: 1 to add, 3 to change, 0 to destroy.
$ ../../scripts/tg apply --non-interactive
module.authentik.kubernetes_pod_disruption_budget_v1.pgbouncer: Creation complete after 0s
module.authentik.kubernetes_deployment.pgbouncer: Modifications complete after 45s
module.authentik.helm_release.authentik: Modifications complete after 2m47s
Apply complete! Resources: 1 added, 3 changed, 0 destroyed.
### Manual Verification
1. **Pod topology and PDBs**:
$ kubectl -n authentik get pods,pdb
pod/goauthentik-server-5fc69b6cc6-ctvkp 1/1 Running 0 3m14s k8s-node2
pod/goauthentik-server-5fc69b6cc6-fkn8x 1/1 Running 0 3m45s k8s-node3
pod/goauthentik-server-5fc69b6cc6-jtjjd 1/1 Running 0 5m6s k8s-node1
pod/goauthentik-worker-5cfb7dc9bf-b2rlr 1/1 Running 0 3m44s k8s-node2
pod/goauthentik-worker-5cfb7dc9bf-fkfm4 1/1 Running 0 5m6s k8s-node1
pod/goauthentik-worker-5cfb7dc9bf-hxdg6 1/1 Running 0 3m3s k8s-node4
pod/pgbouncer-64746f955f-st567 1/1 Running 0 4m58s k8s-node4
pod/pgbouncer-64746f955f-xss9c 1/1 Running 0 5m11s k8s-node2
pod/pgbouncer-64746f955f-zvfkw 1/1 Running 0 4m45s k8s-node3
poddisruptionbudget/goauthentik-server 2 N/A 1
poddisruptionbudget/goauthentik-worker N/A 1 1
poddisruptionbudget/pgbouncer 2 N/A 1
All three workloads spread across 3+ nodes, PDBs allow 1 disruption.
2. **Authentik server health**:
$ curl -sS -o /dev/null -w "%{http_code}\n" \
https://authentik.viktorbarzin.me/-/health/ready/
200
3. **Forward-auth redirect on protected service**:
$ curl -sS -o /dev/null -w "%{http_code}\n" -L \
https://wealthfolio.viktorbarzin.me/
200
4. **Outpost /dev/shm still within sizeLimit** (patches from the
2026-04-18 post-mortem were not regressed):
$ kubectl -n authentik exec deploy/ak-outpost-authentik-embedded-outpost \
-c proxy -- df -h /dev/shm
tmpfs 2.0G 58M 2.0G 3% /dev/shm
5. **PgBouncer port reachable from other pods**:
$ kubectl -n authentik exec deploy/pgbouncer -- nc -zv 127.0.0.1 6432
127.0.0.1 (127.0.0.1:6432) open
## Reproduce locally
1. `cd stacks/authentik && ../../scripts/tg plan` — expect 0/0/0 (No changes).
2. `kubectl -n authentik get pdb pgbouncer` — expect MIN AVAILABLE 2.
3. `kubectl -n authentik get deploy goauthentik-server -o jsonpath='{.spec.replicas}'` — expect 3.
Closes: code-cwj
2026-04-19 11:52:41 +00:00
|
|
|
|
# --- 3b️⃣ PodDisruptionBudget ---
|
|
|
|
|
|
# Protects auth against simultaneous node drains. With 3 replicas and
|
|
|
|
|
|
# minAvailable=2, a single drain rolls cleanly; a simultaneous two-node
|
|
|
|
|
|
# outage is correctly blocked.
|
|
|
|
|
|
resource "kubernetes_pod_disruption_budget_v1" "pgbouncer" {
|
|
|
|
|
|
metadata {
|
|
|
|
|
|
name = "pgbouncer"
|
|
|
|
|
|
namespace = "authentik"
|
|
|
|
|
|
}
|
|
|
|
|
|
spec {
|
|
|
|
|
|
min_available = 2
|
|
|
|
|
|
selector {
|
|
|
|
|
|
match_labels = {
|
|
|
|
|
|
app = "pgbouncer"
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-03-17 18:11:53 +00:00
|
|
|
|
# --- 4️⃣ Service ---
|
|
|
|
|
|
resource "kubernetes_service" "pgbouncer" {
|
|
|
|
|
|
metadata {
|
|
|
|
|
|
name = "pgbouncer"
|
|
|
|
|
|
namespace = "authentik"
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
spec {
|
|
|
|
|
|
selector = {
|
|
|
|
|
|
app = "pgbouncer"
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
port {
|
|
|
|
|
|
port = 6432
|
|
|
|
|
|
target_port = 6432
|
|
|
|
|
|
protocol = "TCP"
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
type = "ClusterIP"
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|