[ci skip] poison-fountain: fix single point of failure causing transient service outages

- Scale to 2 replicas with RollingUpdate (maxUnavailable=0)
- Add topology spread constraint to place pods on different nodes
- Switch from single-threaded to ThreadingMixIn HTTP server so tarpit
  slow-drip requests no longer block /auth and /healthz endpoints

stacks/poison-fountain/app/server.py

@@ -14,6 +14,7 @@ import random
 import time
 import hashlib
 import sys
+import socketserver
 
 LISTEN_PORT = int(os.environ.get("PORT", "8080"))
 CACHE_DIR = os.environ.get("CACHE_DIR", "/data/cache")
@@ -163,8 +164,12 @@ class PoisonHandler(http.server.BaseHTTPRequestHandler):
             pass
 
 
+class ThreadedHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
+    daemon_threads = True
+
+
 if __name__ == "__main__":
     os.makedirs(CACHE_DIR, exist_ok=True)
-    server = http.server.HTTPServer(("0.0.0.0", LISTEN_PORT), PoisonHandler)
+    server = ThreadedHTTPServer(("0.0.0.0", LISTEN_PORT), PoisonHandler)
     print(f"Poison Fountain service listening on :{LISTEN_PORT}", flush=True)
     server.serve_forever()

stacks/poison-fountain/main.tf

@@ -54,9 +54,13 @@ resource "kubernetes_deployment" "poison_fountain" {
   }
 
   spec {
-    replicas = 1
+    replicas = 2
     strategy {
-      type = "Recreate"
+      type = "RollingUpdate"
+      rolling_update {
+        max_unavailable = 0
+        max_surge       = 1
+      }
     }
     selector {
       match_labels = {
@@ -70,6 +74,16 @@ resource "kubernetes_deployment" "poison_fountain" {
         }
       }
       spec {
+        topology_spread_constraint {
+          max_skew           = 1
+          topology_key       = "kubernetes.io/hostname"
+          when_unsatisfiable = "DoNotSchedule"
+          label_selector {
+            match_labels = {
+              app = "poison-fountain"
+            }
+          }
+        }
         container {
           name    = "poison-fountain"
           image   = "python:3.12-slim"