diff --git a/scripts/pfsense-haproxy-bootstrap.php b/scripts/pfsense-haproxy-bootstrap.php index 0a0dc094..3834d852 100644 --- a/scripts/pfsense-haproxy-bootstrap.php +++ b/scripts/pfsense-haproxy-bootstrap.php @@ -1,6 +1,6 @@ []]; -if (!is_array($h['ha_pools']['item'])) $h['ha_pools']['item'] = []; -$h['ha_pools']['item'] = array_values(array_filter( - $h['ha_pools']['item'], - fn($p) => ($p['name'] ?? '') !== 'mailserver_nodes' -)); +// Our declared object names (anything starting with mailserver_ is ours) +$POOL_NAMES = [ + 'mailserver_nodes', // legacy (Phase 2/3 test) + 'mailserver_nodes_smtp', + 'mailserver_nodes_smtps', + 'mailserver_nodes_sub', + 'mailserver_nodes_imaps', +]; +$FRONTEND_NAMES = [ + 'mailserver_proxy_test', // legacy (Phase 2/3 test, :2525) + 'mailserver_proxy_25', + 'mailserver_proxy_465', + 'mailserver_proxy_587', + 'mailserver_proxy_993', +]; -$servers = []; -foreach ([ +// k8s workers. Not in the cluster: master (control-plane) and node5 +// (doesn't exist in this topology). +$NODES = [ ['k8s-node1', '10.0.20.101'], ['k8s-node2', '10.0.20.102'], ['k8s-node3', '10.0.20.103'], ['k8s-node4', '10.0.20.104'], -] as $n) { - $servers[] = [ - 'name' => $n[0], - 'address' => $n[1], - 'port' => '30125', - 'weight' => '10', - 'ssl' => '', - // check every 2 minutes to avoid flooding postscreen with - // send-proxy-v2 + immediate close connections (see bd code-yiu notes). - 'checkinter' => '120000', - 'advanced' => 'send-proxy-v2', - 'status' => 'active', +]; + +function build_pool(string $name, string $nodeport, array $nodes): array { + $servers = []; + foreach ($nodes as $n) { + $servers[] = [ + 'name' => $n[0], + 'address' => $n[1], + 'port' => $nodeport, + 'weight' => '10', + 'ssl' => '', + // check every 2 min — send-proxy-v2 check + close generates + // noise on postscreen, not worth doing more often. + 'checkinter' => '120000', + 'advanced' => 'send-proxy-v2', + 'status' => 'active', + ]; + } + return [ + 'name' => $name, + 'balance' => 'roundrobin', + 'check_type' => 'TCP', + 'checkinter' => '120000', + 'retries' => '3', + 'ha_servers' => ['item' => $servers], + 'advanced_bind' => '', + 'persist_cookie_enabled' => '', + 'transparent_clientip' => '', + 'advanced' => '', ]; } -$h['ha_pools']['item'][] = [ - 'name' => 'mailserver_nodes', - 'balance' => 'roundrobin', - 'check_type' => 'TCP', - 'checkinter' => '120000', - 'retries' => '3', - 'ha_servers' => ['item' => $servers], - 'advanced_bind' => '', - 'persist_cookie_enabled' => '', - 'transparent_clientip' => '', - 'advanced' => '', -]; +function build_frontend(string $name, string $descr, string $extaddr, string $port, string $pool): array { + return [ + 'name' => $name, + 'descr' => $descr, + 'status' => 'active', + 'secondary' => '', + 'type' => 'tcp', + 'a_extaddr' => ['item' => [[ + 'extaddr' => $extaddr, + 'extaddr_port' => $port, + 'extaddr_ssl' => '', + 'extaddr_advanced' => '', + ]]], + 'backend_serverpool' => $pool, + 'ha_acls' => '', + 'dontlognull'=> '', + 'httpclose' => '', + 'forwardfor' => '', + 'advanced' => '', + ]; +} -// ── Frontend (pfSense "ha_backends") ──────────────────────────────────── -if (!is_array($h['ha_backends'])) $h['ha_backends'] = ['item' => []]; +// ── Backend pools ─────────────────────────────────────────────────────── +if (!is_array($h['ha_pools'])) $h['ha_pools'] = ['item' => []]; +if (!is_array($h['ha_pools']['item'])) $h['ha_pools']['item'] = []; +$h['ha_pools']['item'] = array_values(array_filter( + $h['ha_pools']['item'], + fn($p) => !in_array($p['name'] ?? '', $POOL_NAMES, true) +)); + +// Legacy test pool (still used by the :2525 test frontend for manual SMTP roundtrip). +$h['ha_pools']['item'][] = build_pool('mailserver_nodes', '30125', $NODES); + +// Production pools — one per mail port. +$h['ha_pools']['item'][] = build_pool('mailserver_nodes_smtp', '30125', $NODES); +$h['ha_pools']['item'][] = build_pool('mailserver_nodes_smtps', '30126', $NODES); +$h['ha_pools']['item'][] = build_pool('mailserver_nodes_sub', '30127', $NODES); +$h['ha_pools']['item'][] = build_pool('mailserver_nodes_imaps', '30128', $NODES); + +// ── Frontends ─────────────────────────────────────────────────────────── +if (!is_array($h['ha_backends'])) $h['ha_backends'] = ['item' => []]; if (!is_array($h['ha_backends']['item'])) $h['ha_backends']['item'] = []; $h['ha_backends']['item'] = array_values(array_filter( $h['ha_backends']['item'], - fn($f) => ($f['name'] ?? '') !== 'mailserver_proxy_test' + fn($f) => !in_array($f['name'] ?? '', $FRONTEND_NAMES, true) )); -$h['ha_backends']['item'][] = [ - 'name' => 'mailserver_proxy_test', - 'descr' => 'code-yiu Phase 3 test — PROXY v2 to k8s mailserver NodePort 30125', - 'status' => 'active', - 'secondary' => '', - 'type' => 'tcp', - 'a_extaddr' => ['item' => [[ - 'extaddr' => '10.0.20.1', - 'extaddr_port' => '2525', - 'extaddr_ssl' => '', - 'extaddr_advanced' => '', - ]]], - 'backend_serverpool' => 'mailserver_nodes', - 'ha_acls' => '', - 'dontlognull'=> '', - 'httpclose' => '', - 'forwardfor' => '', - 'advanced' => '', -]; +// Legacy test frontend — :2525 — retained so SMTP roundtrip tests keep working +// without touching the real :25. Safe to remove once fully validated. +$h['ha_backends']['item'][] = build_frontend( + 'mailserver_proxy_test', + 'code-yiu Phase 2/3 test — PROXY v2 to k8s mailserver NodePort 30125 (alt port :2525)', + '10.0.20.1', '2525', + 'mailserver_nodes' +); -write_config('code-yiu: mailserver_proxy HAProxy frontend + backend (bootstrap)'); +// Production frontends — 4 ports listening on pfSense VLAN20 IP 10.0.20.1. +$h['ha_backends']['item'][] = build_frontend( + 'mailserver_proxy_25', + 'code-yiu Phase 4/5 — external SMTP (:25) via PROXY v2 → pod :2525 postscreen', + '10.0.20.1', '25', + 'mailserver_nodes_smtp' +); +$h['ha_backends']['item'][] = build_frontend( + 'mailserver_proxy_465', + 'code-yiu Phase 4/5 — external SMTPS (:465) via PROXY v2 → pod :4465 smtpd', + '10.0.20.1', '465', + 'mailserver_nodes_smtps' +); +$h['ha_backends']['item'][] = build_frontend( + 'mailserver_proxy_587', + 'code-yiu Phase 4/5 — external submission (:587) via PROXY v2 → pod :5587 smtpd', + '10.0.20.1', '587', + 'mailserver_nodes_sub' +); +$h['ha_backends']['item'][] = build_frontend( + 'mailserver_proxy_993', + 'code-yiu Phase 4/5 — external IMAPS (:993) via PROXY v2 → pod :10993 Dovecot', + '10.0.20.1', '993', + 'mailserver_nodes_imaps' +); + +write_config('code-yiu: mailserver HAProxy — 4 production frontends + legacy :2525 test'); $messages = ''; $rc = haproxy_check_and_run($messages, true); diff --git a/scripts/pfsense-nat-mailserver-haproxy-flip.php b/scripts/pfsense-nat-mailserver-haproxy-flip.php new file mode 100644 index 00000000..da41bce9 --- /dev/null +++ b/scripts/pfsense-nat-mailserver-haproxy-flip.php @@ -0,0 +1,68 @@ + alias (10.0.20.202 MetalLB LB) to pfSense's own HAProxy +// listener (10.0.20.1). bd code-yiu. +// +// THIS IS THE CUTOVER. After this script: +// Internet → pfSense WAN:{25,465,587,993} → rdr → 10.0.20.1:{...} +// (pfSense HAProxy) → send-proxy-v2 → k8s-node:{30125..30128} NodePort +// → kube-proxy → mailserver pod alt listeners (2525/4465/5587/10993) +// → Postfix/Dovecot parse PROXY v2 → real client IP recovered. +// +// Internal clients (Roundcube, email-roundtrip-monitor CronJob) continue +// using the existing mailserver ClusterIP Service on the stock ports +// (25/465/587/993) which hit container stock listeners WITHOUT PROXY. +// No change to internal traffic paths. +// +// USAGE +// scp infra/scripts/pfsense-nat-mailserver-haproxy-flip.php admin@10.0.20.1:/tmp/ +// ssh admin@10.0.20.1 'php /tmp/pfsense-nat-mailserver-haproxy-flip.php' +// +// REVERT — run pfsense-nat-mailserver-haproxy-unflip.php (companion script). +// +// IDEMPOTENT — re-runs converge. Flips nothing if already pointed at 10.0.20.1. + +require_once('/etc/inc/config.inc'); +require_once('/etc/inc/filter.inc'); + +global $config; +parse_config(true); + +$PORTS_TO_FLIP = ['25', '465', '587', '993']; +$OLD_TARGET = 'mailserver'; +$NEW_TARGET = '10.0.20.1'; + +$changed = 0; +foreach ($config['nat']['rule'] as $i => &$r) { + $iface = $r['interface'] ?? ''; + $lport = $r['local-port'] ?? ''; + $tgt = $r['target'] ?? ''; + + if ($iface !== 'wan') continue; + if (!in_array($lport, $PORTS_TO_FLIP, true)) continue; + if ($tgt !== $OLD_TARGET) { + printf("rule %d (dport=%s) target=%s — not flipping (already %s or unexpected)\n", + $i, $lport, $tgt, $NEW_TARGET); + continue; + } + + $r['target'] = $NEW_TARGET; + // Also unset the 'associated-rule-id' linked filter rule target if any — + // actually pfSense regenerates the associated rule from NAT rule on apply, + // so leaving associated-rule-id intact is fine. + $changed++; + printf("rule %d (dport=%s): target %s → %s\n", $i, $lport, $OLD_TARGET, $NEW_TARGET); +} +unset($r); + +if ($changed === 0) { + echo "No changes. (Already flipped? Run unflip script to revert.)\n"; + exit(0); +} + +write_config("code-yiu: NAT rdr — mail ports {$changed} flipped to HAProxy (10.0.20.1)"); + +// Rebuild pf rules & reload. +$rc = filter_configure(); +printf("filter_configure rc=%s\n", var_export($rc, true)); +echo "done.\n"; diff --git a/scripts/pfsense-nat-mailserver-haproxy-unflip.php b/scripts/pfsense-nat-mailserver-haproxy-unflip.php new file mode 100644 index 00000000..f35870c5 --- /dev/null +++ b/scripts/pfsense-nat-mailserver-haproxy-unflip.php @@ -0,0 +1,48 @@ + alias (10.0.20.202 MetalLB LB IP). bd code-yiu rollback. +// +// USE THIS IF: external mail breaks after the flip, any postscreen +// PROXY timeouts show up in logs, or you need to back out before Phase 6. + +require_once('/etc/inc/config.inc'); +require_once('/etc/inc/filter.inc'); + +global $config; +parse_config(true); + +$PORTS_TO_REVERT = ['25', '465', '587', '993']; +$OLD_TARGET = '10.0.20.1'; +$NEW_TARGET = 'mailserver'; + +$changed = 0; +foreach ($config['nat']['rule'] as $i => &$r) { + $iface = $r['interface'] ?? ''; + $lport = $r['local-port'] ?? ''; + $tgt = $r['target'] ?? ''; + + if ($iface !== 'wan') continue; + if (!in_array($lport, $PORTS_TO_REVERT, true)) continue; + if ($tgt !== $OLD_TARGET) { + printf("rule %d (dport=%s) target=%s — not reverting (already %s or unexpected)\n", + $i, $lport, $tgt, $NEW_TARGET); + continue; + } + + $r['target'] = $NEW_TARGET; + $changed++; + printf("rule %d (dport=%s): target %s → %s\n", $i, $lport, $OLD_TARGET, $NEW_TARGET); +} +unset($r); + +if ($changed === 0) { + echo "No changes. (Already reverted.)\n"; + exit(0); +} + +write_config("code-yiu: NAT rdr — mail ports {$changed} reverted to alias"); + +$rc = filter_configure(); +printf("filter_configure rc=%s\n", var_export($rc, true)); +echo "done.\n"; diff --git a/stacks/mailserver/modules/mailserver/main.tf b/stacks/mailserver/modules/mailserver/main.tf index 3ada09c5..7063fedf 100644 --- a/stacks/mailserver/modules/mailserver/main.tf +++ b/stacks/mailserver/modules/mailserver/main.tf @@ -139,6 +139,24 @@ resource "kubernetes_config_map" "mailserver_config" { # attempt waits 5s before responding, stretching a 1000-password # dictionary attack from <1s to ~85min. Addresses code-9mi. auth_failure_delay = 5s + + # code-yiu Phase 5: alt IMAPS listener on :10993 that REQUIRES the + # HAProxy PROXY v2 wire format. pfSense HAProxy injects the header + # on backend connects via k8s-node:30128 → kube-proxy → pod :10993. + # Real client IP recovered from header despite kube-proxy SNAT. + # The stock :993 listener stays PROXY-free for internal clients + # (Roundcube, email-roundtrip-monitor) on the mailserver ClusterIP. + # haproxy_trusted_networks = source IPs allowed to *send* PROXY v2. + # Post kube-proxy SNAT the source is the k8s node IP (10.0.20.101-104); + # allow-list the whole VLAN 20 node subnet. + haproxy_trusted_networks = 10.0.20.0/24 + service imap-login { + inet_listener imaps_proxy { + port = 10993 + ssl = yes + haproxy = yes + } + } EOF fail2ban_conf = <<-EOF [DEFAULT] @@ -192,22 +210,60 @@ resource "kubernetes_config_map" "mailserver_user_patches" { data = { "user-patches.sh" = <<-EOT #!/bin/bash - # code-yiu: append PROXY-speaking alt SMTP listener on :2525 to master.cf. - # Runs in parallel to stock :25 postscreen (which stays PROXY-free for - # internal clients). pfSense HAProxy injects PROXY v2 on connections to - # k8s-node:NodePort → kube-proxy → pod :2525. Real client IP recovered - # from PROXY header despite kube-proxy SNAT. + # code-yiu Phase 5: append PROXY-speaking alt listeners to Postfix master.cf: + # :2525 postscreen (alt :25) — injected with PROXY v2 by pfSense HAProxy + # :4465 smtpd (alt :465 SMTPS) — ditto, wrappermode TLS + # :5587 smtpd (alt :587 submission) — ditto + # Stock :25/:465/:587 stay in parallel (no PROXY required) so internal + # Roundcube/probe traffic on mailserver.svc ClusterIP keeps working. + # Dovecot alt IMAPS listener on :10993 is configured via dovecot.cf + # (not here) because that's a Dovecot config, not a Postfix master.cf. set -euxo pipefail MASTER_CF=/etc/postfix/master.cf - SENTINEL='# code-yiu:2525' + SENTINEL='# code-yiu:alt-proxy' if ! grep -qF "$SENTINEL" "$MASTER_CF"; then cat >> "$MASTER_CF" <<'PFXEOF' - # code-yiu:2525 — PROXY-speaking postscreen listener for pfSense HAProxy backend. - 2525 inet n - y - 1 postscreen - -o syslog_name=postfix/smtpd-proxy + # code-yiu:alt-proxy — PROXY-speaking alt listeners for pfSense HAProxy backend pool. + # Mirrors stock docker-mailserver submission/submissions options (incl. SASL via + # Dovecot's /dev/shm/sasl-auth.sock) but with PROXY v2 upstream. chroot=n so the + # SASL path is readable from the smtpd process (sockets live outside /var/spool). + 2525 inet n - n - 1 postscreen + -o syslog_name=postfix/smtpd-proxy25 -o postscreen_upstream_proxy_protocol=haproxy -o postscreen_upstream_proxy_timeout=5s + 4465 inet n - n - - smtpd + -o syslog_name=postfix/smtpd-proxy465 + -o smtpd_tls_wrappermode=yes + -o smtpd_sasl_auth_enable=yes + -o smtpd_sasl_type=dovecot + -o smtpd_tls_auth_only=yes + -o smtpd_reject_unlisted_recipient=no + -o smtpd_sasl_authenticated_header=yes + -o smtpd_client_restrictions=permit_sasl_authenticated,reject + -o smtpd_relay_restrictions=permit_sasl_authenticated,reject + -o smtpd_sender_restrictions=$mua_sender_restrictions + -o smtpd_discard_ehlo_keywords= + -o milter_macro_daemon_name=ORIGINATING + -o cleanup_service_name=sender-cleanup + -o smtpd_upstream_proxy_protocol=haproxy + -o smtpd_upstream_proxy_timeout=5s + 5587 inet n - n - - smtpd + -o syslog_name=postfix/smtpd-proxy587 + -o smtpd_tls_security_level=encrypt + -o smtpd_sasl_auth_enable=yes + -o smtpd_sasl_type=dovecot + -o smtpd_tls_auth_only=yes + -o smtpd_reject_unlisted_recipient=no + -o smtpd_sasl_authenticated_header=yes + -o smtpd_client_restrictions=permit_sasl_authenticated,reject + -o smtpd_relay_restrictions=permit_sasl_authenticated,reject + -o smtpd_sender_restrictions=$mua_sender_restrictions + -o smtpd_discard_ehlo_keywords= + -o milter_macro_daemon_name=ORIGINATING + -o cleanup_service_name=sender-cleanup + -o smtpd_upstream_proxy_protocol=haproxy + -o smtpd_upstream_proxy_timeout=5s PFXEOF fi EOT @@ -455,12 +511,29 @@ resource "kubernetes_deployment" "mailserver" { container_port = 993 protocol = "TCP" } - # code-yiu Phase 1a: alt PROXY-speaking SMTP listener. + # code-yiu Phase 5: alt PROXY-speaking listeners. + # Postfix: 2525 (postscreen), 4465 (smtps), 5587 (submission). + # Dovecot: 10993 (imaps). All require PROXY v2 from pfSense HAProxy. port { name = "smtp-proxy" container_port = 2525 protocol = "TCP" } + port { + name = "smtps-proxy" + container_port = 4465 + protocol = "TCP" + } + port { + name = "sub-proxy" + container_port = 5587 + protocol = "TCP" + } + port { + name = "imaps-proxy" + container_port = 10993 + protocol = "TCP" + } env_from { config_map_ref { name = "mailserver.env.config" @@ -637,6 +710,27 @@ resource "kubernetes_service" "mailserver_proxy" { target_port = 2525 node_port = 30125 } + port { + name = "smtps-proxy" + protocol = "TCP" + port = 465 + target_port = 4465 + node_port = 30126 + } + port { + name = "sub-proxy" + protocol = "TCP" + port = 587 + target_port = 5587 + node_port = 30127 + } + port { + name = "imaps-proxy" + protocol = "TCP" + port = 993 + target_port = 10993 + node_port = 30128 + } } }