Add throttling detection and circuit breaker for Rightmove scraper

crawler/rec/exceptions.py

@@ -0,0 +1,74 @@
+"""Custom exceptions for Rightmove API errors."""
+
+
+class RightmoveAPIError(Exception):
+    """Base exception for all Rightmove API errors."""
+
+    pass
+
+
+class ThrottlingError(RightmoveAPIError):
+    """Base exception for throttling-related errors.
+
+    Indicates that Rightmove is limiting our requests and we should back off.
+    """
+
+    pass
+
+
+class RateLimitError(ThrottlingError):
+    """HTTP 429 - Too Many Requests.
+
+    Rightmove is explicitly rate limiting our requests.
+    """
+
+    pass
+
+
+class ServiceUnavailableError(ThrottlingError):
+    """HTTP 503 - Service Unavailable.
+
+    Rightmove's service is temporarily unavailable, possibly due to overload.
+    """
+
+    pass
+
+
+class IPBlockedError(ThrottlingError):
+    """HTTP 403 - Forbidden (IP blocked).
+
+    Our IP may be blocked or blacklisted by Rightmove.
+    """
+
+    pass
+
+
+class SlowResponseError(ThrottlingError):
+    """Response time exceeded threshold.
+
+    API is responding very slowly, indicating potential throttling or overload.
+    """
+
+    pass
+
+
+class UnexpectedEmptyResponseError(RightmoveAPIError):
+    """Empty response received when data was expected."""
+
+    pass
+
+
+class InvalidResponseError(RightmoveAPIError):
+    """Response contains error messages or invalid data."""
+
+    pass
+
+
+class CircuitBreakerOpenError(RightmoveAPIError):
+    """Circuit breaker is open, requests are being blocked.
+
+    The circuit breaker has detected too many failures and is preventing
+    further requests to allow the service to recover.
+    """
+
+    pass