From 0a2d8b2138f89195a9e102712ef9c3c49ae2616a Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Sat, 18 Apr 2026 23:39:06 +0000 Subject: [PATCH 1/2] [mailserver] Move probe secrets to ExternalSecret via ESO [ci skip] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Context The email-roundtrip-monitor CronJob injected `BREVO_API_KEY` and `EMAIL_MONITOR_IMAP_PASSWORD` as inline `env { value = var.xxx }` — Terraform read them from Vault at plan time and embedded them in the generated CronJob spec. Anyone with `kubectl describe cronjob` (or pod-event read) in the `mailserver` namespace could read both secrets verbatim. The two upstream Vault entries are not flat strings: - `secret/viktor` → `brevo_api_key` = base64(JSON({"api_key": "..."})) - `secret/platform` → `mailserver_accounts` = JSON({"spam@viktorbarzin.me": "", ...}) A plain ESO `remoteRef.property` can traverse one level of JSON but cannot base64-decode the wrapper or index a map key that contains `@`. So the ExternalSecret pulls the raw Vault values and the rendered K8s Secret is produced via ESO's `target.template` (engineVersion v2, sprig pipeline `b64dec | fromJson | dig`). `mergePolicy` defaults to Replace, so only the transformed `BREVO_API_KEY` / `EMAIL_MONITOR_IMAP_PASSWORD` keys land in the K8s Secret — the raw wrapped inputs never reach it. ## This change 1. New `kubernetes_manifest.email_roundtrip_monitor_secrets` rendering an `external-secrets.io/v1beta1` ExternalSecret into a K8s Secret named `mailserver-probe-secrets` via the `vault-kv` ClusterSecretStore. 2. CronJob's two `env { name=... value=var.xxx }` blocks replaced with a single `env_from { secret_ref { name = "mailserver-probe-secrets" } }`. 3. Unused `brevo_api_key` / `email_monitor_imap_password` module variables + their wiring in `stacks/mailserver/main.tf` removed. `data "vault_kv_secret_v2" "viktor"` dropped (last consumer gone). ``` Before: After: ┌────────────┐ ┌────────────┐ │ Vault KV │ │ Vault KV │ └────┬───────┘ └────┬───────┘ │ (plan-time read) │ (runtime pull) ▼ ▼ ┌────────────┐ ┌────────────┐ │ Terraform │ │ ESO ctrl │ │ state │ │ +template │ └────┬───────┘ └────┬───────┘ │ inline value= │ sprig b64dec | fromJson ▼ ▼ ┌────────────┐ ┌────────────┐ │ CronJob │ <-- kubectl describe leaks! │ K8s Secret │ │ env[].value│ │ probe-sec │ └────────────┘ └────┬───────┘ │ env_from.secret_ref ▼ ┌────────────┐ │ CronJob │ │ (no values │ │ in spec) │ └────────────┘ ``` ## Test Plan ### Automated `terragrunt plan -target=...ExternalSecret -target=...CronJob`: ``` Plan: 1 to add, 1 to change, 0 to destroy. + kubernetes_manifest.email_roundtrip_monitor_secrets (ExternalSecret) ~ kubernetes_cron_job_v1.email_roundtrip_monitor - env { name = "BREVO_API_KEY" ... } - env { name = "EMAIL_MONITOR_IMAP_PASSWORD" ... } + env_from { secret_ref { name = "mailserver-probe-secrets" } } ``` `terragrunt apply --non-interactive` same targets: ``` Apply complete! Resources: 1 added, 1 changed, 0 destroyed. ``` `kubectl get externalsecret -n mailserver mailserver-probe-secrets`: ``` NAME STORE REFRESH INTERVAL STATUS READY mailserver-probe-secrets vault-kv 15m SecretSynced True ``` `kubectl get secret -n mailserver mailserver-probe-secrets -o yaml` exposes exactly two data keys (`BREVO_API_KEY`, `EMAIL_MONITOR_IMAP_PASSWORD`) — both populated, 120 / 32 base64 chars, no raw `brevo_api_key_wrapped` / `mailserver_accounts` keys. `kubectl describe cronjob -n mailserver email-roundtrip-monitor`: ``` Environment Variables from: mailserver-probe-secrets Secret Optional: false Environment: ``` (Previously the `Environment:` block listed both secrets with their raw values.) ### Manual Verification 1. `kubectl create job --from=cronjob/email-roundtrip-monitor \ probe-test-$RANDOM -n mailserver` 2. `kubectl logs -n mailserver -l job-name=probe-test-... --tail=30` expected: ``` Sent test email via Brevo: 201 marker=e2e-probe-... Found test email after 1 attempts Deleted 1 e2e probe email(s) Round-trip SUCCESS in 20.3s Pushed metrics to Pushgateway Pushed to Uptime Kuma ``` 3. `kubectl exec -n monitoring deploy/prometheus-prometheus-pushgateway \ -- wget -q -O- http://localhost:9091/metrics | grep email_roundtrip` shows `email_roundtrip_success=1`, fresh timestamp, duration in range. 4. `kubectl delete job -n mailserver probe-test-...` to clean up. Closes: code-39v Co-Authored-By: Claude Opus 4.7 (1M context) --- stacks/mailserver/main.tf | 27 +++---- stacks/mailserver/modules/mailserver/main.tf | 75 +++++++++++++++----- 2 files changed, 69 insertions(+), 33 deletions(-) diff --git a/stacks/mailserver/main.tf b/stacks/mailserver/main.tf index da131f8d..bff786cc 100644 --- a/stacks/mailserver/main.tf +++ b/stacks/mailserver/main.tf @@ -11,11 +11,6 @@ data "vault_kv_secret_v2" "secrets" { name = "platform" } -data "vault_kv_secret_v2" "viktor" { - mount = "secret" - name = "viktor" -} - locals { mailserver_accounts = jsondecode(data.vault_kv_secret_v2.secrets.data["mailserver_accounts"]) mailserver_aliases = jsondecode(data.vault_kv_secret_v2.secrets.data["mailserver_aliases"]) @@ -24,16 +19,14 @@ locals { } module "mailserver" { - source = "./modules/mailserver" - tls_secret_name = var.tls_secret_name - nfs_server = var.nfs_server - mysql_host = var.mysql_host - mailserver_accounts = local.mailserver_accounts - postfix_account_aliases = local.mailserver_aliases - opendkim_key = local.mailserver_opendkim_key - sasl_passwd = local.mailserver_sasl_passwd - roundcube_db_password = data.vault_kv_secret_v2.secrets.data["mailserver_roundcubemail_db_password"] - tier = local.tiers.edge - brevo_api_key = jsondecode(base64decode(data.vault_kv_secret_v2.viktor.data["brevo_api_key"]))["api_key"] - email_monitor_imap_password = local.mailserver_accounts["spam@viktorbarzin.me"] + source = "./modules/mailserver" + tls_secret_name = var.tls_secret_name + nfs_server = var.nfs_server + mysql_host = var.mysql_host + mailserver_accounts = local.mailserver_accounts + postfix_account_aliases = local.mailserver_aliases + opendkim_key = local.mailserver_opendkim_key + sasl_passwd = local.mailserver_sasl_passwd + roundcube_db_password = data.vault_kv_secret_v2.secrets.data["mailserver_roundcubemail_db_password"] + tier = local.tiers.edge } diff --git a/stacks/mailserver/modules/mailserver/main.tf b/stacks/mailserver/modules/mailserver/main.tf index 06d8f815..c3b7bd91 100644 --- a/stacks/mailserver/modules/mailserver/main.tf +++ b/stacks/mailserver/modules/mailserver/main.tf @@ -5,15 +5,6 @@ variable "postfix_account_aliases" {} variable "opendkim_key" {} variable "sasl_passwd" {} # For sendgrid i.e relayhost variable "nfs_server" { type = string } -variable "brevo_api_key" { - type = string - sensitive = true -} -variable "email_monitor_imap_password" { - type = string - sensitive = true -} - # Build the virtual-alias map, dropping aliases where BOTH the source and # target are real mailboxes in var.mailserver_accounts (and are different). # Without this filter, docker-mailserver emits two passwd-file userdb lines @@ -564,6 +555,61 @@ resource "kubernetes_service" "mailserver" { # E2E Email Roundtrip Monitor # Sends test email via Brevo API, verifies delivery via IMAP, pushes metrics # ============================================================================= +# ExternalSecret syncing the probe's Vault inputs into a K8s Secret, so +# `kubectl describe cronjob email-roundtrip-monitor` no longer leaks the +# Brevo API key and IMAP password via `env[].value`. The two upstream Vault +# entries both wrap the effective secret: +# - secret/viktor → brevo_api_key = base64(JSON({"api_key": "..."})) +# - secret/platform → mailserver_accounts = JSON({"spam@viktorbarzin.me": "", ...}) +# ESO's `target.template` (engineVersion v2) runs sprig on the raw remote +# values so the rendered K8s Secret contains ONLY the two env vars the probe +# actually needs, under the exact keys `BREVO_API_KEY` and +# `EMAIL_MONITOR_IMAP_PASSWORD` so the CronJob can consume them via a single +# `env_from { secret_ref {} }` block. +resource "kubernetes_manifest" "email_roundtrip_monitor_secrets" { + manifest = { + apiVersion = "external-secrets.io/v1beta1" + kind = "ExternalSecret" + metadata = { + name = "mailserver-probe-secrets" + namespace = kubernetes_namespace.mailserver.metadata[0].name + } + spec = { + refreshInterval = "15m" + secretStoreRef = { + name = "vault-kv" + kind = "ClusterSecretStore" + } + target = { + name = "mailserver-probe-secrets" + template = { + engineVersion = "v2" + data = { + BREVO_API_KEY = "{{ .brevo_api_key_wrapped | b64dec | fromJson | dig \"api_key\" \"\" }}" + EMAIL_MONITOR_IMAP_PASSWORD = "{{ .mailserver_accounts | fromJson | dig \"spam@viktorbarzin.me\" \"\" }}" + } + } + } + data = [ + { + secretKey = "brevo_api_key_wrapped" + remoteRef = { + key = "viktor" + property = "brevo_api_key" + } + }, + { + secretKey = "mailserver_accounts" + remoteRef = { + key = "platform" + property = "mailserver_accounts" + } + }, + ] + } + } +} + resource "kubernetes_cron_job_v1" "email_roundtrip_monitor" { metadata { name = "email-roundtrip-monitor" @@ -695,13 +741,10 @@ sys.exit(0 if success else 1) ' EOT ] - env { - name = "BREVO_API_KEY" - value = var.brevo_api_key - } - env { - name = "EMAIL_MONITOR_IMAP_PASSWORD" - value = var.email_monitor_imap_password + env_from { + secret_ref { + name = "mailserver-probe-secrets" + } } resources { requests = { From ac604d4d1faa494986314fe3467b3d392f033416 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Sat, 18 Apr 2026 23:39:46 +0000 Subject: [PATCH 2/2] [monitoring] uk-payslip: cash-basis queries + RSU vest panel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Panels 1/2/4: compute on (gross_pay - rsu_vest) so numbers reflect actual UK cash pay, not the RSU-inflated figure the payslip shows. - Detailed table: add cash_gross / rsu_vest / rsu_offset columns. - New RSU panel at the bottom: bar chart of rsu_vest over time (only shows months with stock vests). Taxed at Schwab — included here for reporting/reconciliation, not for P&L. --- .../monitoring/dashboards/uk-payslip.json | 71 +++++++++++++++++-- 1 file changed, 64 insertions(+), 7 deletions(-) diff --git a/stacks/monitoring/modules/monitoring/dashboards/uk-payslip.json b/stacks/monitoring/modules/monitoring/dashboards/uk-payslip.json index 123ed959..d9e6cac7 100644 --- a/stacks/monitoring/modules/monitoring/dashboards/uk-payslip.json +++ b/stacks/monitoring/modules/monitoring/dashboards/uk-payslip.json @@ -24,7 +24,7 @@ "panels": [ { "id": 1, - "title": "Monthly gross / net / tax / NI", + "title": "Monthly cash gross / net / tax / NI (RSU stripped)", "type": "timeseries", "datasource": { "type": "grafana-postgresql-datasource", @@ -95,7 +95,7 @@ "type": "grafana-postgresql-datasource", "uid": "payslips-pg" }, - "rawSql": "SELECT pay_date AS \"time\", gross_pay, net_pay, income_tax, national_insurance FROM payslip_ingest.payslip WHERE $__timeFilter(pay_date) ORDER BY pay_date", + "rawSql": "SELECT pay_date AS \"time\", (gross_pay - rsu_vest) AS cash_gross, net_pay, income_tax, national_insurance FROM payslip_ingest.payslip WHERE $__timeFilter(pay_date) ORDER BY pay_date", "format": "time_series", "refId": "A", "rawQuery": true, @@ -105,7 +105,7 @@ }, { "id": 2, - "title": "YTD gross (this tax year) with UK band thresholds", + "title": "YTD cash gross (excl. RSU) with UK band thresholds", "type": "timeseries", "datasource": { "type": "grafana-postgresql-datasource", @@ -197,7 +197,7 @@ "type": "grafana-postgresql-datasource", "uid": "payslips-pg" }, - "rawSql": "SELECT pay_date AS \"time\", SUM(gross_pay) OVER (PARTITION BY tax_year ORDER BY pay_date) AS ytd_gross FROM payslip_ingest.payslip WHERE $__timeFilter(pay_date) ORDER BY pay_date", + "rawSql": "SELECT pay_date AS \"time\", SUM(gross_pay - rsu_vest) OVER (PARTITION BY tax_year ORDER BY pay_date) AS ytd_cash_gross FROM payslip_ingest.payslip WHERE $__timeFilter(pay_date) ORDER BY pay_date", "format": "time_series", "refId": "A", "rawQuery": true, @@ -288,7 +288,7 @@ }, { "id": 4, - "title": "Latest effective rate & take-home %", + "title": "Effective rate & take-home % (cash-basis)", "type": "timeseries", "datasource": { "type": "grafana-postgresql-datasource", @@ -361,7 +361,7 @@ "type": "grafana-postgresql-datasource", "uid": "payslips-pg" }, - "rawSql": "SELECT pay_date AS \"time\", ROUND(((income_tax + national_insurance)::numeric / NULLIF(gross_pay, 0)) * 100, 2) AS \"effective_rate_pct\", ROUND((net_pay::numeric / NULLIF(gross_pay, 0)) * 100, 2) AS \"take_home_pct\" FROM payslip_ingest.payslip WHERE $__timeFilter(pay_date) ORDER BY pay_date", + "rawSql": "SELECT pay_date AS \"time\", ROUND(((income_tax + national_insurance)::numeric / NULLIF(gross_pay - rsu_vest, 0)) * 100, 2) AS \"effective_rate_pct\", ROUND((net_pay::numeric / NULLIF(gross_pay - rsu_vest, 0)) * 100, 2) AS \"take_home_pct\" FROM payslip_ingest.payslip WHERE $__timeFilter(pay_date) ORDER BY pay_date", "format": "time_series", "refId": "A", "rawQuery": true, @@ -540,7 +540,7 @@ "rawQuery": true, "editorMode": "code", "format": "table", - "rawSql": "SELECT pay_date, employer, tax_year, gross_pay, income_tax, national_insurance, pension_employee, pension_employer, student_loan, COALESCE(other_deductions, '{}'::jsonb) AS other_deductions, net_pay, validated, paperless_doc_id FROM payslip_ingest.payslip WHERE $__timeFilter(pay_date) ORDER BY pay_date DESC" + "rawSql": "SELECT pay_date, employer, tax_year, gross_pay, (gross_pay - rsu_vest) AS cash_gross, rsu_vest, rsu_offset, income_tax, national_insurance, pension_employee, pension_employer, student_loan, COALESCE(other_deductions, '{}'::jsonb) AS other_deductions, net_pay, validated, paperless_doc_id FROM payslip_ingest.payslip WHERE $__timeFilter(pay_date) ORDER BY pay_date DESC" } ] }, @@ -605,6 +605,63 @@ "rawSql": "SELECT pay_date AS \"time\", income_tax, national_insurance, pension_employee, pension_employer, student_loan FROM payslip_ingest.payslip WHERE $__timeFilter(pay_date) ORDER BY pay_date" } ] + }, + { + "id": 7, + "title": "RSU vest history (notional, taxed at Schwab)", + "type": "timeseries", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "payslips-pg" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 41 + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "fillOpacity": 70, + "lineWidth": 1 + }, + "unit": "currencyGBP", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "payslips-pg" + }, + "rawQuery": true, + "editorMode": "code", + "format": "time_series", + "rawSql": "SELECT pay_date AS \"time\", rsu_vest FROM payslip_ingest.payslip WHERE $__timeFilter(pay_date) AND rsu_vest > 0 ORDER BY pay_date" + } + ] } ], "refresh": "5m",