From 5fe288a4e4aa0cd7612adcbb85a6196453f5171a Mon Sep 17 00:00:00 2001
From: Viktor Barzin <viktorbarzin@meta.com>
Date: Sat, 21 Feb 2026 17:32:45 +0000
Subject: [PATCH] [ci skip] Real estate crawler: 2 replicas for UI/API, rolling
 update for celery
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- UI and API: 1 → 2 replicas for zero-downtime during restarts/crashes
- Celery worker: Recreate → RollingUpdate strategy
- Celery beat: unchanged (Recreate, singleton scheduler)
- Move f1 from Cloudflare proxied to non-proxied DNS
---
 .../kubernetes/real-estate-crawler/main.tf    | 91 +++++++------------
 1 file changed, 32 insertions(+), 59 deletions(-)

diff --git a/modules/kubernetes/real-estate-crawler/main.tf b/modules/kubernetes/real-estate-crawler/main.tf
index 2758fbae..0db7b77c 100644
--- a/modules/kubernetes/real-estate-crawler/main.tf
+++ b/modules/kubernetes/real-estate-crawler/main.tf
@@ -32,7 +32,7 @@ resource "kubernetes_deployment" "realestate-crawler-ui" {
     }
   }
   spec {
-    replicas = 1
+    replicas = 2
     strategy {
       type = "RollingUpdate"
       rolling_update {
@@ -104,7 +104,7 @@ resource "kubernetes_deployment" "realestate-crawler-api" {
     }
   }
   spec {
-    replicas = 1
+    replicas = 2
     strategy {
       type = "RollingUpdate"
       rolling_update {
@@ -262,7 +262,11 @@ resource "kubernetes_deployment" "realestate-crawler-celery" {
   spec {
     replicas = 1
     strategy {
-      type = "Recreate"
+      type = "RollingUpdate"
+      rolling_update {
+        max_unavailable = 0
+        max_surge       = 1
+      }
     }
     selector {
       match_labels = {
@@ -281,6 +285,11 @@ resource "kubernetes_deployment" "realestate-crawler-celery" {
           image             = "viktorbarzin/realestatecrawler:latest"
           image_pull_policy = "Always"
           command           = ["python", "-m", "celery", "-A", "celery_app", "worker", "--loglevel=info"]
+          port {
+            name           = "metrics"
+            container_port = 9090
+            protocol       = "TCP"
+          }
           env {
             name  = "ENV"
             value = "prod"
@@ -330,6 +339,26 @@ resource "kubernetes_deployment" "realestate-crawler-celery" {
   }
 }
 
+resource "kubernetes_service" "realestate-crawler-celery-metrics" {
+  metadata {
+    name      = "realestate-crawler-celery-metrics"
+    namespace = kubernetes_namespace.realestate-crawler.metadata[0].name
+    labels = {
+      "app" = "realestate-crawler-celery"
+    }
+  }
+
+  spec {
+    selector = {
+      app = "realestate-crawler-celery"
+    }
+    port {
+      port        = 9090
+      target_port = 9090
+    }
+  }
+}
+
 # Celery beat for scheduled task management
 resource "kubernetes_deployment" "realestate-crawler-celery-beat" {
   metadata {
@@ -398,59 +427,3 @@ resource "kubernetes_deployment" "realestate-crawler-celery-beat" {
   }
 }
 
-resource "kubernetes_cron_job_v1" "scrape-rightmove" {
-  metadata {
-    name      = "scrape-rightmove"
-    namespace = kubernetes_namespace.realestate-crawler.metadata[0].name
-  }
-  spec {
-    concurrency_policy            = "Replace"
-    failed_jobs_history_limit     = 5
-    schedule                      = "0 0 1 * *"
-    starting_deadline_seconds     = 10
-    successful_jobs_history_limit = 10
-    job_template {
-      metadata {}
-      spec {
-        backoff_limit              = 3
-        ttl_seconds_after_finished = 10
-        template {
-          metadata {}
-          spec {
-            container {
-              name  = "scrape-rightmove"
-              image = "viktorbarzin/realestatecrawler:latest"
-              command = ["/bin/sh", "-c", <<-EOT
-                /app/runall.sh # Run the scrape script
-              EOT
-              ]
-              env {
-                name  = "DB_CONNECTION_STRING"
-                value = "mysql://wrongmove:wrongmove@mysql.dbaas.svc.cluster.local:3306/wrongmove"
-              }
-              # env {
-              #   name  = "HTTP_PROXY"
-              #   value = "http://tor-proxy.tor-proxy:8118"
-              # }
-              # env {
-              #   name  = "HTTPS_PROXY"
-              #   value = "http://tor-proxy.tor-proxy:8118"
-              # }
-              volume_mount {
-                name       = "data"
-                mount_path = "/app/data"
-              }
-            }
-            volume {
-              name = "data"
-              nfs {
-                path   = "/mnt/main/real-estate-crawler"
-                server = "10.0.10.15"
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-}