Refactor backend for cleaner error handling, DRY, and type safety
- Extract rate limiter DRY: consolidate 3 duplicated check/respond paths into _check_counter and _enforce_limit helpers, add proper type annotations - Replace bare Exception raises with FloorplanDownloadError and RightmoveApiError; narrow catch clauses to specific exception types; fix Step base class to inherit from ABC - Consolidate MAX_OCR_WORKERS into config/scraper_config.py; extract _find_tenure_value helper to deduplicate tenure parsing - Extract _build_poi_distances_lookup from stream endpoint to reduce nesting - Fix csv_exporter: optional decisions.json, NaN instead of -1 sentinels, guard against division by zero on missing square meters - Fix notifications.py broken list[Surface]() constructor, database.py stale comments and missing type annotation, auth.py type:ignore, ui_exporter.py stale TODO - Fix 3 pre-existing test failures: mock cache layer in streaming tests, bypass rate limiter for test isolation, fix cache invalidation test to account for two-pattern scan loop
This commit is contained in:
parent
6897820cc7
commit
f833309297
20 changed files with 199 additions and 178 deletions
|
|
@ -10,6 +10,7 @@ from models.listing import FurnishType, ListingType
|
|||
from rec import districts
|
||||
from rec.exceptions import (
|
||||
CircuitBreakerOpenError,
|
||||
RightmoveAPIError,
|
||||
ThrottlingError,
|
||||
)
|
||||
from rec.throttle_detector import get_throttle_metrics, validate_response
|
||||
|
|
@ -205,9 +206,9 @@ def _build_listing_params(
|
|||
7,
|
||||
14,
|
||||
]:
|
||||
raise Exception(
|
||||
f"Invalid max days - {max_days_since_added} Can only be got",
|
||||
[1, 3, 7, 14],
|
||||
raise ValueError(
|
||||
f"Invalid max_days_since_added={max_days_since_added}, "
|
||||
f"must be one of [1, 3, 7, 14]"
|
||||
)
|
||||
params["maxDaysSinceAdded"] = str(max_days_since_added)
|
||||
|
||||
|
|
@ -287,7 +288,7 @@ async def _execute_api_request(
|
|||
)
|
||||
|
||||
if response.status != 200:
|
||||
raise Exception(
|
||||
raise RightmoveAPIError(
|
||||
f"{error_context}Failed due to: {await response.text()}"
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue