"""Circuit breaker pattern for protecting against cascading failures.""" from __future__ import annotations import enum import logging import time from dataclasses import dataclass from rec.exceptions import CircuitBreakerOpenError logger = logging.getLogger("uvicorn.error") class CircuitState(enum.Enum): """Circuit breaker states.""" CLOSED = "closed" # Normal operation OPEN = "open" # Too many failures, blocking requests HALF_OPEN = "half_open" # Testing if service recovered @dataclass class CircuitBreaker: """Circuit breaker for protecting against cascading failures. Implements the circuit breaker pattern: - CLOSED: Requests pass through normally, failures are counted - OPEN: After N consecutive failures, circuit opens and blocks all requests - HALF_OPEN: After recovery timeout, allow one request to test if service recovered Attributes: failure_threshold: Number of consecutive failures before opening. recovery_timeout: Seconds to wait before attempting half-open state. state: Current circuit state. failure_count: Count of consecutive failures. last_failure_time: Timestamp of last failure. last_state_change: Timestamp of last state change. """ failure_threshold: int recovery_timeout: float state: CircuitState = CircuitState.CLOSED failure_count: int = 0 last_failure_time: float = 0.0 last_state_change: float = 0.0 def __post_init__(self) -> None: """Initialize state change timestamp.""" self.last_state_change = time.time() def call(self) -> None: """Check if a request should be allowed. Raises: CircuitBreakerOpenError: If circuit is open and blocking requests. """ current_time = time.time() if self.state == CircuitState.OPEN: # Check if we should transition to half-open if current_time - self.last_failure_time >= self.recovery_timeout: self._transition_to_half_open() else: raise CircuitBreakerOpenError( f"Circuit breaker is open. " f"Waiting {self.recovery_timeout - (current_time - self.last_failure_time):.1f}s " f"before retry." ) # Allow request to proceed (CLOSED or HALF_OPEN) def record_success(self) -> None: """Record a successful request.""" if self.state == CircuitState.HALF_OPEN: # Service has recovered, close the circuit self._transition_to_closed() # Reset failure count on success self.failure_count = 0 def record_failure(self) -> None: """Record a failed request.""" self.failure_count += 1 self.last_failure_time = time.time() if self.state == CircuitState.HALF_OPEN: # Test request failed, reopen circuit self._transition_to_open() elif self.state == CircuitState.CLOSED: # Check if we should open the circuit if self.failure_count >= self.failure_threshold: self._transition_to_open() def _transition_to_open(self) -> None: """Transition to OPEN state.""" self.state = CircuitState.OPEN self.last_state_change = time.time() logger.warning( f"Circuit breaker OPENED after {self.failure_count} consecutive failures. " f"Will retry in {self.recovery_timeout}s" ) def _transition_to_half_open(self) -> None: """Transition to HALF_OPEN state.""" self.state = CircuitState.HALF_OPEN self.last_state_change = time.time() logger.info("Circuit breaker entering HALF_OPEN state, testing service recovery") def _transition_to_closed(self) -> None: """Transition to CLOSED state.""" self.state = CircuitState.CLOSED self.last_state_change = time.time() self.failure_count = 0 logger.info("Circuit breaker CLOSED, service recovered") def reset(self) -> None: """Manually reset the circuit breaker to CLOSED state.""" self.state = CircuitState.CLOSED self.failure_count = 0 self.last_failure_time = 0.0 self.last_state_change = time.time() logger.info("Circuit breaker manually reset to CLOSED state") @property def is_open(self) -> bool: """Check if circuit is currently open.""" return self.state == CircuitState.OPEN @property def is_closed(self) -> bool: """Check if circuit is currently closed.""" return self.state == CircuitState.CLOSED @property def is_half_open(self) -> bool: """Check if circuit is currently half-open.""" return self.state == CircuitState.HALF_OPEN