Add throttling detection and circuit breaker for Rightmove scraper

crawler/services/query_splitter.py

@@ -16,6 +16,7 @@ import aiohttp
 from config.scraper_config import ScraperConfig
 from models.listing import ListingType, QueryParameters
 from rec.districts import get_districts
+from rec.exceptions import CircuitBreakerOpenError, ThrottlingError
 
 logger = logging.getLogger("uvicorn.error")
 
@@ -113,6 +114,9 @@ class QuerySplitter:
 
         Returns:
             Total available results for this subquery.
+
+        Raises:
+            CircuitBreakerOpenError: If the circuit breaker is open.
         """
         from rec.query import probe_query
 
@@ -128,8 +132,17 @@ class QuerySplitter:
                 district=subquery.district,
                 max_days_since_added=parameters.max_days_since_added,
                 furnish_types=parameters.furnish_types or [],
+                config=self.config,
             )
             return result.get("totalAvailableResults", 0)
+        except CircuitBreakerOpenError:
+            logger.error("Circuit breaker is open, stopping probe operations")
+            raise
+        except ThrottlingError as e:
+            logger.warning(
+                f"Throttling detected during probe for {subquery.district}: {e}"
+            )
+            return 0
         except Exception as e:
             logger.warning(f"Failed to probe subquery {subquery}: {e}")
             return 0