From a1ba218cd2957e3bb5402444875f8db6dadd0388 Mon Sep 17 00:00:00 2001
From: Viktor Barzin <viktorbarzin@meta.com>
Date: Sat, 28 Feb 2026 19:08:06 +0000
Subject: [PATCH] [ci skip] Phase 1: PostgreSQL migrated to CNPG on local disk

Major milestone - shared PostgreSQL moved from NFS to CloudNativePG:
- CNPG cluster (pg-cluster) running in dbaas namespace on local-path storage
- PostGIS image (ghcr.io/cloudnative-pg/postgis:16) for dawarich compatibility
- All 20 databases and 19 roles restored from pg_dumpall backup
- postgresql.dbaas Service patched to point at CNPG primary
- Old PG deployment scaled to 0 (NFS data intact for rollback)
- All 12+ dependent services verified running:
  authentik, n8n, dawarich, tandoor, linkwarden, netbox, woodpecker,
  rybbit, affine, health, resume, trading-bot, atuin
- Authentik PgBouncer working through the switched endpoint

TODO: codify CNPG cluster in Terraform, add 2nd replica, update backup CronJob
---
 .woodpecker/build-cli.yml                     |   7 +-
 .../2026-02-23-mailserver-hardening-design.md |  62 ++++
 .../2026-02-23-mailserver-hardening-plan.md   | 315 ++++++++++++++++++
 secrets/nfs_directories.txt                   | Bin 1754 -> 1760 bytes
 stacks/actualbudget/tiers.tf                  |  10 +
 stacks/atuin/main.tf                          | 148 ++++++++
 stacks/atuin/secrets                          |   1 +
 stacks/atuin/terragrunt.hcl                   |   8 +
 stacks/atuin/tiers.tf                         |  10 +
 stacks/ebook2audiobook/tiers.tf               |  10 +
 stacks/nextcloud/tiers.tf                     |  10 +
 stacks/ollama/tiers.tf                        |  10 +
 stacks/osm_routing/tiers.tf                   |  10 +
 stacks/paperless-ngx/tiers.tf                 |  10 +
 stacks/platform/tiers.tf                      |  10 +
 stacks/plotting-book/tiers.tf                 |  10 +
 stacks/poison-fountain/tiers.tf               |  10 +
 stacks/rybbit/tiers.tf                        |  10 +
 stacks/servarr/tiers.tf                       |  10 +
 stacks/stirling-pdf/tiers.tf                  |  10 +
 stacks/trading-bot/main.tf                    |  29 +-
 stacks/trading-bot/tiers.tf                   |  10 +
 stacks/woodpecker/tiers.tf                    |  10 +
 stacks/ytdlp/tiers.tf                         |  10 +
 tiers.tf                                      |  10 +
 25 files changed, 733 insertions(+), 7 deletions(-)
 create mode 100644 docs/plans/2026-02-23-mailserver-hardening-design.md
 create mode 100644 docs/plans/2026-02-23-mailserver-hardening-plan.md
 create mode 100644 stacks/actualbudget/tiers.tf
 create mode 100644 stacks/atuin/main.tf
 create mode 120000 stacks/atuin/secrets
 create mode 100644 stacks/atuin/terragrunt.hcl
 create mode 100644 stacks/atuin/tiers.tf
 create mode 100644 stacks/ebook2audiobook/tiers.tf
 create mode 100644 stacks/nextcloud/tiers.tf
 create mode 100644 stacks/ollama/tiers.tf
 create mode 100644 stacks/osm_routing/tiers.tf
 create mode 100644 stacks/paperless-ngx/tiers.tf
 create mode 100644 stacks/platform/tiers.tf
 create mode 100644 stacks/plotting-book/tiers.tf
 create mode 100644 stacks/poison-fountain/tiers.tf
 create mode 100644 stacks/rybbit/tiers.tf
 create mode 100644 stacks/servarr/tiers.tf
 create mode 100644 stacks/stirling-pdf/tiers.tf
 create mode 100644 stacks/trading-bot/tiers.tf
 create mode 100644 stacks/woodpecker/tiers.tf
 create mode 100644 stacks/ytdlp/tiers.tf
 create mode 100644 tiers.tf

diff --git a/.woodpecker/build-cli.yml b/.woodpecker/build-cli.yml
index 44081d4c..1936d0d8 100644
--- a/.woodpecker/build-cli.yml
+++ b/.woodpecker/build-cli.yml
@@ -26,8 +26,5 @@ steps:
       dockerfile: cli/Dockerfile
       context: cli
       auto_tag: true
-      cache_images: registry.viktorbarzin.lan:5050/infra:buildcache
-      buildkit_config: |
-        [registry."registry.viktorbarzin.lan:5050"]
-          http = true
-          insecure = true
+      insecure: true
+      buildkit_config: "[registry.\"registry.viktorbarzin.lan:5050\"]\n  http = true\n  insecure = true\n"
diff --git a/docs/plans/2026-02-23-mailserver-hardening-design.md b/docs/plans/2026-02-23-mailserver-hardening-design.md
new file mode 100644
index 00000000..33657c20
--- /dev/null
+++ b/docs/plans/2026-02-23-mailserver-hardening-design.md
@@ -0,0 +1,62 @@
+# Mail Server Lightweight Hardening Design
+
+**Date**: 2026-02-23
+**Scope**: Security, reliability, and hygiene improvements to the docker-mailserver stack
+
+## Current State
+
+- docker-mailserver 15.0.0 on K8s (single replica, Recreate strategy)
+- Roundcubemail webmail (MySQL-backed, debug logging on, unpinned :latest tag)
+- Outbound relay via Mailgun, inbound MX via ForwardEmail
+- OpenDKIM for DKIM signing, no spam filtering (SpamAssassin/ClamAV/Amavis disabled)
+- DMARC policy `none` (monitoring only)
+- No brute-force protection, no mailserver-down alert
+- Dovecot exporter sidecar (unpinned), stale SendGrid DNS records
+
+## Changes
+
+### 1. Enable Rspamd (replace OpenDKIM as DKIM signer)
+
+Add to `mailserver_env_config`:
+- `ENABLE_RSPAMD = "1"` (spam filtering, DKIM verification, phishing detection, Oletools)
+- `ENABLE_OPENDKIM = "0"` (Rspamd handles DKIM signing natively)
+- `RSPAMD_LEARN = "1"` (learn from Junk folder movements)
+
+Existing OpenDKIM key mounts stay — Rspamd reads them from the same paths.
+Resource impact: ~150-200MB additional RAM.
+
+### 2. DMARC DNS enforcement
+
+Update `_dmarc` TXT record: `p=none` -> `p=quarantine`. Can tighten to `p=reject` after validation.
+
+### 3. Postfix rate limiting
+
+Add to `postfix_cf`:
+```
+smtpd_client_connection_rate_limit = 10
+smtpd_client_message_rate_limit = 30
+anvil_rate_time_unit = 60s
+```
+
+Service already uses `externalTrafficPolicy: Local`, so real client IPs are visible to Postfix.
+ForwardEmail IPs on port 25 are subject to same limits but 10 conn/min is generous.
+
+### 4. Prometheus alert
+
+Uncomment the existing mailserver-down alert in `prometheus_chart_values.tpl`.
+
+### 5. Roundcubemail cleanup
+
+- Pin image: `roundcube/roundcubemail:latest` -> `roundcube/roundcubemail:1.6-apache`
+- Disable debug: `ROUNDCUBEMAIL_SMTP_DEBUG = "false"`, `ROUNDCUBEMAIL_DEBUG_LEVEL = "1"`
+
+### 6. SendGrid DNS cleanup
+
+Remove stale CNAME records: `em7107`, `s1._domainkey`, `s2._domainkey`.
+
+## Not Changing
+
+- Roundcubemail stays (user preference)
+- ForwardEmail/Mailgun relay stays (practical dependency)
+- ClamAV stays disabled (Rspamd Oletools covers malicious attachments)
+- Single replica (HA email requires significant additional complexity)
diff --git a/docs/plans/2026-02-23-mailserver-hardening-plan.md b/docs/plans/2026-02-23-mailserver-hardening-plan.md
new file mode 100644
index 00000000..b555017f
--- /dev/null
+++ b/docs/plans/2026-02-23-mailserver-hardening-plan.md
@@ -0,0 +1,315 @@
+# Mail Server Lightweight Hardening Implementation Plan
+
+> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
+
+**Goal:** Harden the mail server with spam filtering (Rspamd), DMARC enforcement, rate limiting, monitoring alerts, and hygiene cleanup.
+
+**Architecture:** All changes are to the existing docker-mailserver 15.0.0 deployment managed by Terraform. Rspamd replaces OpenDKIM for DKIM signing and adds spam filtering. DMARC moves from `none` to `quarantine` in Cloudflare DNS. Postfix gets rate-limiting parameters. Prometheus gets a mailserver-down alert. Roundcubemail debug logging is disabled and image pinned.
+
+**Tech Stack:** Terraform/HCL, docker-mailserver, Rspamd, Cloudflare DNS, Prometheus
+
+---
+
+### Task 1: Enable Rspamd and disable OpenDKIM
+
+**Files:**
+- Modify: `stacks/platform/modules/mailserver/main.tf:39-62` (env ConfigMap)
+
+**Step 1: Add Rspamd env vars to the ConfigMap**
+
+In `stacks/platform/modules/mailserver/main.tf`, in the `kubernetes_config_map.mailserver_env_config` resource `data` block, add these entries and modify existing ones:
+
+```hcl
+  data = {
+    DMS_DEBUG = "0"
+    ENABLE_CLAMAV                          = "0"
+    ENABLE_AMAVIS                          = "0"
+    ENABLE_FAIL2BAN                        = "0"
+    ENABLE_FETCHMAIL                       = "0"
+    ENABLE_POSTGREY                        = "0"
+    ENABLE_SASLAUTHD                       = "0"
+    ENABLE_SPAMASSASSIN                    = "0"
+    ENABLE_SRS                             = "1"
+    ENABLE_RSPAMD                          = "1"
+    ENABLE_OPENDKIM                        = "0"
+    ENABLE_OPENDMARC                       = "0"
+    RSPAMD_LEARN                           = "1"
+    FETCHMAIL_POLL                         = "120"
+    ONE_DIR                                = "1"
+    OVERRIDE_HOSTNAME                      = "mail.viktorbarzin.me"
+    POSTFIX_MESSAGE_SIZE_LIMIT             = 1024 * 1024 * 200 # 200 MB
+    POSTFIX_REJECT_UNKNOWN_CLIENT_HOSTNAME = "1"
+    DEFAULT_RELAY_HOST = "[smtp.eu.mailgun.org]:587"
+    SPOOF_PROTECTION   = "1"
+    SSL_TYPE           = "manual"
+    SSL_CERT_PATH      = "/tmp/ssl/tls.crt"
+    SSL_KEY_PATH       = "/tmp/ssl/tls.key"
+  }
+```
+
+The key additions are: `ENABLE_RSPAMD = "1"`, `ENABLE_OPENDKIM = "0"`, `ENABLE_OPENDMARC = "0"`, `RSPAMD_LEARN = "1"`.
+
+**Note:** The existing OpenDKIM volume mounts (KeyTable, SigningTable, TrustedHosts, opendkim keys) should stay mounted. docker-mailserver's Rspamd integration reads the DKIM key from the same path (`/tmp/docker-mailserver/opendkim/keys/`) to configure Rspamd's DKIM signing module automatically.
+
+**Step 2: Commit**
+
+```bash
+git add stacks/platform/modules/mailserver/main.tf
+git commit -m "[ci skip] mailserver: enable Rspamd, disable OpenDKIM"
+```
+
+---
+
+### Task 2: Add Postfix rate limiting
+
+**Files:**
+- Modify: `stacks/platform/modules/mailserver/variables.tf:3-22` (postfix_cf variable)
+
+**Step 1: Add rate limiting parameters to postfix_cf**
+
+In `stacks/platform/modules/mailserver/variables.tf`, append these lines to the `postfix_cf` default value, before the `EOT`:
+
+```
+smtpd_client_connection_rate_limit = 10
+smtpd_client_message_rate_limit = 30
+anvil_rate_time_unit = 60s
+```
+
+The full `postfix_cf` variable should become:
+
+```hcl
+variable "postfix_cf" {
+  default = <<EOT
+#relayhost = [smtp.sendgrid.net]:587
+relayhost = [smtp.eu.mailgun.org]:587
+smtp_sasl_auth_enable = yes
+smtp_sasl_password_maps = hash:/etc/postfix/sasl/passwd
+smtp_sasl_security_options = noanonymous
+smtp_sasl_tls_security_options = noanonymous
+smtp_tls_security_level = encrypt
+smtpd_tls_cert_file=/tmp/ssl/tls.crt
+smtpd_tls_key_file=/tmp/ssl/tls.key
+smtpd_use_tls=yes
+header_size_limit = 4096000
+
+# Debug mail tls
+smtpd_tls_loglevel = 1
+
+# Rate limiting (brute-force protection)
+smtpd_client_connection_rate_limit = 10
+smtpd_client_message_rate_limit = 30
+anvil_rate_time_unit = 60s
+EOT
+}
+```
+
+**Step 2: Commit**
+
+```bash
+git add stacks/platform/modules/mailserver/variables.tf
+git commit -m "[ci skip] mailserver: add Postfix rate limiting"
+```
+
+---
+
+### Task 3: Update DMARC DNS record to quarantine
+
+**Files:**
+- Modify: `stacks/platform/modules/cloudflared/cloudflare.tf:132-138` (DMARC record)
+- Modify: `terraform.tfvars:85` (bind zone DMARC record)
+
+**Step 1: Update Cloudflare DMARC record**
+
+In `stacks/platform/modules/cloudflared/cloudflare.tf`, change the `cloudflare_record.mail_dmarc` content from `p=none` to `p=quarantine` and `sp=none` to `sp=quarantine`:
+
+```hcl
+resource "cloudflare_record" "mail_dmarc" {
+  content  = "\"v=DMARC1; p=quarantine; pct=100; fo=1; ri=3600; sp=quarantine; adkim=r; aspf=r; rua=mailto:e21c0ff8@dmarc.mailgun.org,mailto:adb84997@inbox.ondmarc.com; ruf=mailto:e21c0ff8@dmarc.mailgun.org,mailto:adb84997@inbox.ondmarc.com,mailto:postmaster@viktorbarzin.me;\""
+  name     = "_dmarc.viktorbarzin.me"
+  proxied  = false
+  ttl      = 1
+  type     = "TXT"
+  priority = 1
+  zone_id  = var.cloudflare_zone_id
+}
+```
+
+**Step 2: Update bind zone DMARC record**
+
+In `terraform.tfvars` line 85, update the DMARC record:
+
+```
+_dmarc IN TXT "v=DMARC1; p=quarantine; ruf=mailto:postmaster@viktorbarzin.me; adkim=r; aspf=r; pct=100; sp=quarantine"
+```
+
+**Step 3: Commit**
+
+```bash
+git add stacks/platform/modules/cloudflared/cloudflare.tf terraform.tfvars
+git commit -m "[ci skip] mailserver: tighten DMARC policy to quarantine"
+```
+
+---
+
+### Task 4: Enable Prometheus mailserver-down alert
+
+**Files:**
+- Modify: `stacks/platform/modules/monitoring/prometheus_chart_values.tpl:544-550`
+
+**Step 1: Uncomment the mailserver alert**
+
+In `stacks/platform/modules/monitoring/prometheus_chart_values.tpl`, replace lines 544-550:
+
+From:
+```yaml
+          # - alert: Mail server has no replicas available
+          #   expr: (kube_deployment_status_replicas_available{namespace="mailserver"} or on() vector(0)) < 1
+          #   for: 10m
+          #   labels:
+          #     severity: page
+          #   annotations:
+          #     summary: Mail server has no available replicas. This means mail may not be received.
+```
+
+To:
+```yaml
+          - alert: Mail server has no replicas available
+            expr: (kube_deployment_status_replicas_available{namespace="mailserver"} or on() vector(0)) < 1
+            for: 5m
+            labels:
+              severity: page
+            annotations:
+              summary: Mail server has no available replicas. This means mail may not be received.
+```
+
+Note: reduced `for` from 10m to 5m and fixed indentation to match the surrounding YAML (10 spaces).
+
+**Step 2: Commit**
+
+```bash
+git add stacks/platform/modules/monitoring/prometheus_chart_values.tpl
+git commit -m "[ci skip] monitoring: enable mailserver-down Prometheus alert"
+```
+
+---
+
+### Task 5: Pin Roundcubemail image and disable debug logging
+
+**Files:**
+- Modify: `stacks/platform/modules/mailserver/roundcubemail.tf:60,113-119`
+
+**Step 1: Pin the image tag**
+
+In `stacks/platform/modules/mailserver/roundcubemail.tf` line 60, change:
+
+```hcl
+          image = "roundcube/roundcubemail:latest"
+```
+
+To:
+
+```hcl
+          image = "roundcube/roundcubemail:1.6-apache"
+```
+
+**Step 2: Disable debug logging**
+
+In the same file, change the debug env vars:
+
+```hcl
+          env {
+            name  = "ROUNDCUBEMAIL_SMTP_DEBUG"
+            value = "false"
+          }
+          env {
+            name  = "ROUNDCUBEMAIL_DEBUG_LEVEL"
+            value = "1"
+          }
+```
+
+**Step 3: Commit**
+
+```bash
+git add stacks/platform/modules/mailserver/roundcubemail.tf
+git commit -m "[ci skip] roundcubemail: pin to 1.6-apache, disable debug logging"
+```
+
+---
+
+### Task 6: Clean up stale SendGrid DNS records
+
+**Files:**
+- Modify: `terraform.tfvars:88-90`
+
+**Step 1: Remove SendGrid CNAME records from bind zone**
+
+In `terraform.tfvars`, remove lines 88-90:
+
+```
+em7107 IN CNAME u31127144.wl145.sendgrid.net.
+s1._domainkey IN CNAME s1.domainkey.u31127144.wl145.sendgrid.net.
+s2._domainkey IN CNAME s2.domainkey.u31127144.wl145.sendgrid.net.
+```
+
+These are stale remnants from a previous SendGrid relay setup. They are not in the Cloudflare terraform config, so they may also need manual removal from Cloudflare if they were created outside Terraform.
+
+**Step 2: Commit**
+
+```bash
+git add terraform.tfvars
+git commit -m "[ci skip] dns: remove stale SendGrid CNAME records"
+```
+
+---
+
+### Task 7: Apply changes
+
+**Step 1: Apply the platform stack**
+
+```bash
+cd stacks/platform && terragrunt apply --non-interactive
+```
+
+This deploys: Rspamd enablement, Postfix rate limiting, DMARC DNS update, Prometheus alert, Roundcubemail changes.
+
+**Step 2: Verify the mailserver pod restarts with Rspamd**
+
+```bash
+kubectl --kubeconfig $(pwd)/config get pods -n mailserver
+```
+
+Wait for the pod to be Running. Then verify Rspamd is active:
+
+```bash
+kubectl --kubeconfig $(pwd)/config exec -n mailserver deploy/mailserver -c docker-mailserver -- pgrep -a rspamd
+```
+
+Should show rspamd processes running.
+
+**Step 3: Verify Postfix rate limiting is applied**
+
+```bash
+kubectl --kubeconfig $(pwd)/config exec -n mailserver deploy/mailserver -c docker-mailserver -- postconf smtpd_client_connection_rate_limit
+```
+
+Expected: `smtpd_client_connection_rate_limit = 10`
+
+**Step 4: Verify DKIM signing still works with Rspamd**
+
+Send a test email and check DKIM signature in the headers, or check Rspamd logs:
+
+```bash
+kubectl --kubeconfig $(pwd)/config logs -n mailserver deploy/mailserver -c docker-mailserver --tail=50 | grep -i dkim
+```
+
+**Step 5: Verify Roundcubemail is running with pinned image**
+
+```bash
+kubectl --kubeconfig $(pwd)/config get deploy -n mailserver roundcubemail -o jsonpath='{.spec.template.spec.containers[0].image}'
+```
+
+Expected: `roundcube/roundcubemail:1.6-apache`
+
+**Step 6: Verify Prometheus alert is active**
+
+Check in Grafana/Prometheus UI that the "Mail server has no replicas available" alert rule exists and is in `inactive` state (meaning the mailserver is healthy).
diff --git a/secrets/nfs_directories.txt b/secrets/nfs_directories.txt
index f8a4de4387a487434fe8e4d910de5fbefc5b153a..9b854c7c93723724a3bd217f07b94f05572d94b8 100644
GIT binary patch
literal 1760
zcmV<61|RtVM@dveQdv+`0I{_B*xa^GrUL+0yVd&s9mCOiKGoeAPXQK{oo={B<wXc`
zT-_d?h`&6z5QW^Xa>1EqA>xW;Q1KG-<-g<9;MTlrBdK$cxd<cybN8hH!a0GEL_~rC
z+-d3J>LH?W`XXheSTrMZrKKs6Uc0qmr=1o6>R|F8g0&t41)$qmqTOA|VHzccvj+nl
zs^0#$u%?jhDuM(VcK+$$Qqu7gZPtL6!oqg7Fa3F82Ibau3Cx$<MI3`|FhUfTsSIzK
zALuz&uV*UH{8d)^^K1rHVZw{Y@{hCGA516EiKv=G5;P?(`I^BYo@1}o1h!P<<eB3x
z<nsgBZmRlE%~5oN-nnSmL$lC5s0^PG$q1WMLDp<zx~d==Cw{I_Vmm(tZZIpH-l+a0
zFa^|3-ry9iwPVxuUJ6?tp?6mD0&lvxanbKLsGaIb`hABXGIdBKicwvi?Uhyc2yXJ;
z&ID6hwrOwTA%+-u?L_YugL@#p&m}(M0fE(c6wKF1iij+keo@Y{1|#eX&XaS&`BaF3
z`OKjDufz(q-g_ikXSub&X5%5r1@MwxZd`v_Nz!qnET0z&Z<_RHk}zrM<%F;Os{Rwj
zeW9&OmAsDd>G{a#PpurpgUhd1;9W6_m1Qj^illnps|3d!6q%vlNDg9y&3$$64MU3D
zZxz31P5Sf|H^!6X?LgLoNZ6inGq!j1SEdie$Pfe~V&nb@`>i*3p-6boVtt@r<?XP{
zOeRPg=msrmtRkR+$f*lnjcQOoBv07khCmBg9`((k<@r!g3Yye8_flO`OkcK#kH2$|
z8HeCf&qlUn3Wq}&g{|;H>8eTQ`f`03pOZd%pW2h~t-X)aI8sdhQbg6vDSkm*QzI9p
zY3PQ|2LL%hsJz0qJncbWRRnCqp;J8u-*!N*+0kQNJc|NFy6c4y9QNg3m-KPzA+w{o
ztTD7sIGpV>E=BRx`>SyRy57bjpe`s*Xba_kt5q~3d#a)L(9*A>>67^knbFMMcjVwe
zPyo5!jAY(Rsm2KZ3O~L)l&%1jO(Khp*#)(a6wd~xdZE}aLzcC3i+l2Fdm(&P5(nZS
z{y-?eZ$fr~z&PTJN>n_^{Yr+Wb2DC7^O!RIss;Bo&gAx`e-A&36>>HrvMD~ZF6UUQ
zz4Vq15V9rW@^QM&R#{3ZyKBdHEU3tLEH7$fL`IueO--AN<$1~kDvp}<E&%4cCe&S)
z%9U2f0L0G*D394VS<E9K$<a`;KS3Ws<u0^Tb4b3%Sjy}oV8C>PQ_X|(D;R0YPqDP}
zDEOVQu{kbd-FFeDvl31TQj&_66C%@4hiP^#TuSb!04T;dsG}o>uOD|(NxEw^mU)(d
z^$yGGizi#Izv*<XKGCl%k>H&kjF-JH3XXB;&A)lK<*D^;7ZyPC1D7TBl%IcTPpJ?z
zoOs!kuLY{zG+SHBWQ@q)`$dpo!ffE#S%#U)j7DO;ayX}B?)m6;J4U1wwQ|7N$sm3(
zNf4(F?_-)lR~_3{^j$p-o@iDmjU2dPtxFYgAOE(C2Jg;lj`eZ^>w1CyPax3q*W=G+
zksW!7Xg7>+*!E;J&!GXC6&xfvRl)G4+o-OSMd#(+3ChrtZk6d~B)+eHhO(#ak4DY3
zM!K2+mfzYsUe(@~pm;DwRn)<Vr*l=H+(vskPS8?1fo5zUwvDAOr?)<i_iTkg)Cs=v
z)2pqxpZn9L-KXoPpX6<?sHSzkqvOW3Pui1L|0Ru(M#*6w+W7PXWAsTyGo(<32hb8?
zs%6#vCE)v_fK;-~rc5<0@Oa;WX-Qu38nwKVgv#?5Gx6Z!3Ray(q+FjVAB}8#1C5VV
zqsavddfIL=?aaVxpYlZv7<T>ut#~%{o4HwsU$k_O&vH~T2Nztd+OyEr^mf!R*C75A
zUsv(&&MA@j5Nrb3Wfx^WpbXu5I`jna@D$BI!3gUt2AOp-Klb@Z6+JY>+Qm%RpK7Ej
zyA9FSXo(r8%nV)%wqm47&nl45@;Q9168Q0$DP$AU4LGsrFp%+HudXrL6+tfmbXESp
zm|4XPOCM)4Zn2bfmSisj=UH5Olha-MJ&&EvINVkEx1Jjc0#v2~=&zj$q1Op>)mtvt
zijC^QefHT%&fhC#i?u6y1Hcq^aX&Sa6|H!KRGLP<7?SQ$*Ib!7cCpl3PSe!lL2Fc#
zYK;{iC|-keZ)f~DO@9b7xck%X3Nrps(*=zG05Vk;nosD2)b0Qzw3x$3IA$T-BtMny
z6(4|5P8UJNcQ1dU)x^t(E0t?xbwNn%&|T4AYvN@i_DblHyxidDST6ABaBlVy6ziv8
CQG5;n

literal 1754
zcmV<01||6bM@dveQdv+`07h7--%tK(7`xtzLB5VK9&G%$AVj?NE|2&xwkxq-7#wZn
zhm5#SvoXVWv?PJE*>v-4A`KS=4_Y*#xG9Q4y*pvAh(4FbeA&9BTj3d4vA*TN_ZnH^
zv16`QM0y%yaC2Dx>c6hz2Joykbn@|T7Xv`rNgR|3uLF)fyUNRT4C5@VFRu1bNxROa
z-93%0Pe~xT<=V!qC+04wg-g9kJlItpAqGJhsr_%qcT|9~tIkb`E`l9h$K56a`(?}%
z25o38_@dhUg!OV4d-r`{(L{$;bMPxsScRs{nDi!yb;s_k5FGPz9Fa#xk0j6o7G&ka
z)!EH@cp>>==hgQX$Zt^AKEkX=v}bg#6O{tK@S^Ik=1^@XQebetHbr6~gLXXxge~yI
zei@m5Q&tZs>0vV8v1}(=y<zDi+0{xKEf->awfoz5ZdoyikI$*DGx(PtDRAw}iG)|X
zg3F`~3)9837R_d@j4s9t(?D%>Qe_b~^T;q4=kv=+gRKmKURYB(WcG$g0%o%A80n;f
zq5qE)f*5G^Q}d;liY{@N`5=C=XD(8zAarAL<tY7BQv;}-h*mu0#X9b%!eouyd?m-&
z*8B)?{+#ibI#3C(^VRX^MQ2#K3F!S*U^`8Bw$Ra4EsdBD7cik6gaVrg`@1TfImJN#
zRlr^8+?oVbuJp6m@nbUe+Ige)Ox}FG3DT`aW<EqFhvM(hRjKU~|0+4_A#8QLggI-N
zjM{*f;kF;IK}Es7O1bbKc{PGd>j%G6@60;a-vA%8!EL1!8TI7eZWOn`SYuWbuu>3R
zpBcocc<;@KC<O)gm5i!Wl&Ve*q`Az@V`l6G)OpqEh;y1E=5v2Xd*L@y$M`I<`VRNp
zsaxLnB{k$KgW~N+6uf`!2Qj=?&qxZcxWgS-*Hu?v0aNvHj{$PbRTI95shtU)%iz^j
z%U$<&eG(|kLfR1<+X04gC>tn}Pwj_*++ux@_Wou-Dv8r)P&wrjrWR~zlZ$iCdS!p1
ziHanYL?3AUHX1X0TtVYDMeMrNHD<sbFgI>$wmBsbi}dLB{y8ibW-*wiA2M`IiGv2J
z6}7czL1VWUd+3~898tgVdiM6a_jKHhPNk>bLQ8p{Q4nO1B|z~F$GwCVyj3$NiFnS`
z0})@z`oB$4R0HbIgi$-Ff>jytp}#Kz51~OodI57DKyXPn%GmTh|Hg+<D#<`^fq|i5
zAW|Zd`xA%;l^NL~dq|^;bDsS1L(L~jEsZm>jb0l#lNp+bU%9`Tp>6GPIuXFYUy5vR
z5#iTp?m^JZZX2w7CuKAld(aOc0-&YeqM5n9MPp%21d2|J@H|#KoMFcj*uvnzy=+a@
zdW_G?;ajaWI(o~YuyAW}<yNx(xBSD<CL^Vr>gKvl!xp{1)T3=#pWa-60b2~MJu2Zn
zFv}G}AwQXQ%LT#J#-slV|HI`VF<DIC==>!_q$_SpQe7HULyf~$Tdfzvt*78J0`FB!
zY6}#iBU4z<B3|AdQljj_^TU0#e3h?2o3JI}VAX~a^t!Z{p$M`rWLgGo*L(_N=wK-)
zQC^V!-#oK!p^i9MWV6X6kwH7&qxNF9iCC6i3)q(4o0=O-g+;W*rIp#_oRrgi2-PpR
z&`91+*<sy)HdVFA)uVh71;JYcXc6~wuw%WE;((9}&PHhW@m6<wG<d8qRNRs7kd~7@
zo&3DxGMRTm1i;KN9wL{J?NsMplS$4$L1#()t`U$7q*c#Z+3mvYR=o#DBP!9OuzY5X
z+Dz@R(p^-_=Gy_SWOYs@_*sh#P1=G`{L&^hebCSfyHQy=NiD1cHm0jIHH(l8IQ$+v
zkSS-L9lsge!lLOBX5>gz$Qcgn1zy9yaSa&F;7aGB$&p`k+?|K7f|IKKfNjCU?*8`d
z4@?RHKmf4cjDQs8gHpN?VJfXsQ@X3o6D$UY((HNhmz2a9>o3)6=kdTwrKqMpXGZL7
z=A-fg7QuiXyqcfrO|>ATo-)n@aj?Sw#UK>rjRL5(#gI{77t!^zWXacY;rZLwyrM~C
zwp4<ZVS0YrhC4>!&ozINlU7?Gl#J*IBp-pu6oDM4`_N0b4HEfz8=El5q>ZZR*^%gj
zc(|l^9mhH&)fe#e!x2+yS2gx*2m^iTt7y#-+uH%T!9P<;0@*j{v_dJ{JcVK8RfOWX
zn}P{XHa3s&JL+c(>D%d=eAui=Tm)B;21PASFmU7bofJl#nM3Oki>ctBJFRORBA63L
wjfimGu<x-`q7;!XU)ZgrRh|lGH&nzX+Wm!Gyt22W9RE+?WLbE6g_E2Bfq~(00ssI2

diff --git a/stacks/actualbudget/tiers.tf b/stacks/actualbudget/tiers.tf
new file mode 100644
index 00000000..eb0f8083
--- /dev/null
+++ b/stacks/actualbudget/tiers.tf
@@ -0,0 +1,10 @@
+# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
+locals {
+  tiers = {
+    core    = "0-core"
+    cluster = "1-cluster"
+    gpu     = "2-gpu"
+    edge    = "3-edge"
+    aux     = "4-aux"
+  }
+}
diff --git a/stacks/atuin/main.tf b/stacks/atuin/main.tf
new file mode 100644
index 00000000..0298b851
--- /dev/null
+++ b/stacks/atuin/main.tf
@@ -0,0 +1,148 @@
+variable "tls_secret_name" { type = string }
+variable "nfs_server" { type = string }
+variable "postgresql_host" { type = string }
+variable "atuin_postgresql_password" { type = string }
+
+resource "kubernetes_namespace" "atuin" {
+  metadata {
+    name = "atuin"
+    labels = {
+      tier = local.tiers.aux
+    }
+  }
+}
+
+module "tls_secret" {
+  source          = "../../modules/kubernetes/setup_tls_secret"
+  namespace       = kubernetes_namespace.atuin.metadata[0].name
+  tls_secret_name = var.tls_secret_name
+}
+
+resource "kubernetes_deployment" "atuin" {
+  wait_for_rollout = false
+  metadata {
+    name      = "atuin"
+    namespace = kubernetes_namespace.atuin.metadata[0].name
+    labels = {
+      app  = "atuin"
+      tier = local.tiers.aux
+    }
+  }
+  spec {
+    replicas = 1
+    selector {
+      match_labels = {
+        app = "atuin"
+      }
+    }
+    template {
+      metadata {
+        labels = {
+          app = "atuin"
+        }
+      }
+      spec {
+        container {
+          name  = "atuin"
+          image = "ghcr.io/atuinsh/atuin:3f775df"
+
+          args = ["start"]
+
+          port {
+            container_port = 8888
+          }
+
+          env {
+            name  = "ATUIN_HOST"
+            value = "0.0.0.0"
+          }
+          env {
+            name  = "ATUIN_PORT"
+            value = "8888"
+          }
+          env {
+            name  = "ATUIN_OPEN_REGISTRATION"
+            value = "false"
+          }
+          env {
+            name  = "ATUIN_DB_URI"
+            value = "postgres://atuin:${var.atuin_postgresql_password}@${var.postgresql_host}:5432/atuin"
+          }
+          env {
+            name  = "RUST_LOG"
+            value = "info"
+          }
+
+          volume_mount {
+            name       = "config"
+            mount_path = "/config"
+          }
+
+          resources {
+            requests = {
+              memory = "64Mi"
+              cpu    = "50m"
+            }
+            limits = {
+              memory = "256Mi"
+              cpu    = "500m"
+            }
+          }
+
+          liveness_probe {
+            http_get {
+              path = "/healthz"
+              port = 8888
+            }
+            initial_delay_seconds = 10
+            period_seconds        = 30
+          }
+
+          readiness_probe {
+            http_get {
+              path = "/healthz"
+              port = 8888
+            }
+            initial_delay_seconds = 5
+            period_seconds        = 10
+          }
+        }
+
+        volume {
+          name = "config"
+          nfs {
+            server = var.nfs_server
+            path   = "/mnt/main/atuin"
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_service" "atuin" {
+  metadata {
+    name      = "atuin"
+    namespace = kubernetes_namespace.atuin.metadata[0].name
+    labels = {
+      app = "atuin"
+    }
+  }
+  spec {
+    selector = {
+      app = "atuin"
+    }
+    port {
+      name        = "http"
+      port        = 80
+      target_port = 8888
+    }
+  }
+}
+
+module "ingress" {
+  source          = "../../modules/kubernetes/ingress_factory"
+  namespace       = kubernetes_namespace.atuin.metadata[0].name
+  name            = "atuin"
+  tls_secret_name = var.tls_secret_name
+}
diff --git a/stacks/atuin/secrets b/stacks/atuin/secrets
new file mode 120000
index 00000000..ca54a7cf
--- /dev/null
+++ b/stacks/atuin/secrets
@@ -0,0 +1 @@
+../../secrets
\ No newline at end of file
diff --git a/stacks/atuin/terragrunt.hcl b/stacks/atuin/terragrunt.hcl
new file mode 100644
index 00000000..0d1c8e53
--- /dev/null
+++ b/stacks/atuin/terragrunt.hcl
@@ -0,0 +1,8 @@
+include "root" {
+  path = find_in_parent_folders()
+}
+
+dependency "platform" {
+  config_path  = "../platform"
+  skip_outputs = true
+}
diff --git a/stacks/atuin/tiers.tf b/stacks/atuin/tiers.tf
new file mode 100644
index 00000000..eb0f8083
--- /dev/null
+++ b/stacks/atuin/tiers.tf
@@ -0,0 +1,10 @@
+# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
+locals {
+  tiers = {
+    core    = "0-core"
+    cluster = "1-cluster"
+    gpu     = "2-gpu"
+    edge    = "3-edge"
+    aux     = "4-aux"
+  }
+}
diff --git a/stacks/ebook2audiobook/tiers.tf b/stacks/ebook2audiobook/tiers.tf
new file mode 100644
index 00000000..eb0f8083
--- /dev/null
+++ b/stacks/ebook2audiobook/tiers.tf
@@ -0,0 +1,10 @@
+# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
+locals {
+  tiers = {
+    core    = "0-core"
+    cluster = "1-cluster"
+    gpu     = "2-gpu"
+    edge    = "3-edge"
+    aux     = "4-aux"
+  }
+}
diff --git a/stacks/nextcloud/tiers.tf b/stacks/nextcloud/tiers.tf
new file mode 100644
index 00000000..eb0f8083
--- /dev/null
+++ b/stacks/nextcloud/tiers.tf
@@ -0,0 +1,10 @@
+# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
+locals {
+  tiers = {
+    core    = "0-core"
+    cluster = "1-cluster"
+    gpu     = "2-gpu"
+    edge    = "3-edge"
+    aux     = "4-aux"
+  }
+}
diff --git a/stacks/ollama/tiers.tf b/stacks/ollama/tiers.tf
new file mode 100644
index 00000000..eb0f8083
--- /dev/null
+++ b/stacks/ollama/tiers.tf
@@ -0,0 +1,10 @@
+# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
+locals {
+  tiers = {
+    core    = "0-core"
+    cluster = "1-cluster"
+    gpu     = "2-gpu"
+    edge    = "3-edge"
+    aux     = "4-aux"
+  }
+}
diff --git a/stacks/osm_routing/tiers.tf b/stacks/osm_routing/tiers.tf
new file mode 100644
index 00000000..eb0f8083
--- /dev/null
+++ b/stacks/osm_routing/tiers.tf
@@ -0,0 +1,10 @@
+# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
+locals {
+  tiers = {
+    core    = "0-core"
+    cluster = "1-cluster"
+    gpu     = "2-gpu"
+    edge    = "3-edge"
+    aux     = "4-aux"
+  }
+}
diff --git a/stacks/paperless-ngx/tiers.tf b/stacks/paperless-ngx/tiers.tf
new file mode 100644
index 00000000..eb0f8083
--- /dev/null
+++ b/stacks/paperless-ngx/tiers.tf
@@ -0,0 +1,10 @@
+# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
+locals {
+  tiers = {
+    core    = "0-core"
+    cluster = "1-cluster"
+    gpu     = "2-gpu"
+    edge    = "3-edge"
+    aux     = "4-aux"
+  }
+}
diff --git a/stacks/platform/tiers.tf b/stacks/platform/tiers.tf
new file mode 100644
index 00000000..eb0f8083
--- /dev/null
+++ b/stacks/platform/tiers.tf
@@ -0,0 +1,10 @@
+# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
+locals {
+  tiers = {
+    core    = "0-core"
+    cluster = "1-cluster"
+    gpu     = "2-gpu"
+    edge    = "3-edge"
+    aux     = "4-aux"
+  }
+}
diff --git a/stacks/plotting-book/tiers.tf b/stacks/plotting-book/tiers.tf
new file mode 100644
index 00000000..eb0f8083
--- /dev/null
+++ b/stacks/plotting-book/tiers.tf
@@ -0,0 +1,10 @@
+# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
+locals {
+  tiers = {
+    core    = "0-core"
+    cluster = "1-cluster"
+    gpu     = "2-gpu"
+    edge    = "3-edge"
+    aux     = "4-aux"
+  }
+}
diff --git a/stacks/poison-fountain/tiers.tf b/stacks/poison-fountain/tiers.tf
new file mode 100644
index 00000000..eb0f8083
--- /dev/null
+++ b/stacks/poison-fountain/tiers.tf
@@ -0,0 +1,10 @@
+# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
+locals {
+  tiers = {
+    core    = "0-core"
+    cluster = "1-cluster"
+    gpu     = "2-gpu"
+    edge    = "3-edge"
+    aux     = "4-aux"
+  }
+}
diff --git a/stacks/rybbit/tiers.tf b/stacks/rybbit/tiers.tf
new file mode 100644
index 00000000..eb0f8083
--- /dev/null
+++ b/stacks/rybbit/tiers.tf
@@ -0,0 +1,10 @@
+# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
+locals {
+  tiers = {
+    core    = "0-core"
+    cluster = "1-cluster"
+    gpu     = "2-gpu"
+    edge    = "3-edge"
+    aux     = "4-aux"
+  }
+}
diff --git a/stacks/servarr/tiers.tf b/stacks/servarr/tiers.tf
new file mode 100644
index 00000000..eb0f8083
--- /dev/null
+++ b/stacks/servarr/tiers.tf
@@ -0,0 +1,10 @@
+# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
+locals {
+  tiers = {
+    core    = "0-core"
+    cluster = "1-cluster"
+    gpu     = "2-gpu"
+    edge    = "3-edge"
+    aux     = "4-aux"
+  }
+}
diff --git a/stacks/stirling-pdf/tiers.tf b/stacks/stirling-pdf/tiers.tf
new file mode 100644
index 00000000..eb0f8083
--- /dev/null
+++ b/stacks/stirling-pdf/tiers.tf
@@ -0,0 +1,10 @@
+# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
+locals {
+  tiers = {
+    core    = "0-core"
+    cluster = "1-cluster"
+    gpu     = "2-gpu"
+    edge    = "3-edge"
+    aux     = "4-aux"
+  }
+}
diff --git a/stacks/trading-bot/main.tf b/stacks/trading-bot/main.tf
index 53d8ad3d..c170bd59 100644
--- a/stacks/trading-bot/main.tf
+++ b/stacks/trading-bot/main.tf
@@ -47,8 +47,9 @@ resource "kubernetes_namespace" "trading-bot" {
   metadata {
     name = "trading-bot"
     labels = {
-      tier                               = local.tiers.edge
-      "resource-governance/custom-quota" = "true"
+      tier                                         = local.tiers.edge
+      "resource-governance/custom-quota"           = "true"
+      "goldilocks.fairwinds.com/vpa-update-mode"   = "off"
     }
   }
 }
@@ -265,6 +266,10 @@ resource "kubernetes_deployment" "trading-bot-workers" {
               value = env.value
             }
           }
+          env {
+            name  = "TRADING_OTEL_METRICS_PORT"
+            value = "9091"
+          }
           resources {
             requests = {
               cpu    = "10m"
@@ -288,6 +293,10 @@ resource "kubernetes_deployment" "trading-bot-workers" {
               value = env.value
             }
           }
+          env {
+            name  = "TRADING_OTEL_METRICS_PORT"
+            value = "9092"
+          }
           resources {
             requests = {
               cpu    = "100m"
@@ -311,6 +320,10 @@ resource "kubernetes_deployment" "trading-bot-workers" {
               value = env.value
             }
           }
+          env {
+            name  = "TRADING_OTEL_METRICS_PORT"
+            value = "9093"
+          }
           resources {
             requests = {
               cpu    = "10m"
@@ -334,6 +347,10 @@ resource "kubernetes_deployment" "trading-bot-workers" {
               value = env.value
             }
           }
+          env {
+            name  = "TRADING_OTEL_METRICS_PORT"
+            value = "9094"
+          }
           resources {
             requests = {
               cpu    = "10m"
@@ -357,6 +374,10 @@ resource "kubernetes_deployment" "trading-bot-workers" {
               value = env.value
             }
           }
+          env {
+            name  = "TRADING_OTEL_METRICS_PORT"
+            value = "9095"
+          }
           resources {
             requests = {
               cpu    = "10m"
@@ -380,6 +401,10 @@ resource "kubernetes_deployment" "trading-bot-workers" {
               value = env.value
             }
           }
+          env {
+            name  = "TRADING_OTEL_METRICS_PORT"
+            value = "9096"
+          }
           resources {
             requests = {
               cpu    = "10m"
diff --git a/stacks/trading-bot/tiers.tf b/stacks/trading-bot/tiers.tf
new file mode 100644
index 00000000..eb0f8083
--- /dev/null
+++ b/stacks/trading-bot/tiers.tf
@@ -0,0 +1,10 @@
+# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
+locals {
+  tiers = {
+    core    = "0-core"
+    cluster = "1-cluster"
+    gpu     = "2-gpu"
+    edge    = "3-edge"
+    aux     = "4-aux"
+  }
+}
diff --git a/stacks/woodpecker/tiers.tf b/stacks/woodpecker/tiers.tf
new file mode 100644
index 00000000..eb0f8083
--- /dev/null
+++ b/stacks/woodpecker/tiers.tf
@@ -0,0 +1,10 @@
+# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
+locals {
+  tiers = {
+    core    = "0-core"
+    cluster = "1-cluster"
+    gpu     = "2-gpu"
+    edge    = "3-edge"
+    aux     = "4-aux"
+  }
+}
diff --git a/stacks/ytdlp/tiers.tf b/stacks/ytdlp/tiers.tf
new file mode 100644
index 00000000..eb0f8083
--- /dev/null
+++ b/stacks/ytdlp/tiers.tf
@@ -0,0 +1,10 @@
+# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
+locals {
+  tiers = {
+    core    = "0-core"
+    cluster = "1-cluster"
+    gpu     = "2-gpu"
+    edge    = "3-edge"
+    aux     = "4-aux"
+  }
+}
diff --git a/tiers.tf b/tiers.tf
new file mode 100644
index 00000000..eb0f8083
--- /dev/null
+++ b/tiers.tf
@@ -0,0 +1,10 @@
+# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
+locals {
+  tiers = {
+    core    = "0-core"
+    cluster = "1-cluster"
+    gpu     = "2-gpu"
+    edge    = "3-edge"
+    aux     = "4-aux"
+  }
+}