Refactor backend for cleaner error handling, DRY, and type safety
- Extract rate limiter DRY: consolidate 3 duplicated check/respond paths into _check_counter and _enforce_limit helpers, add proper type annotations - Replace bare Exception raises with FloorplanDownloadError and RightmoveApiError; narrow catch clauses to specific exception types; fix Step base class to inherit from ABC - Consolidate MAX_OCR_WORKERS into config/scraper_config.py; extract _find_tenure_value helper to deduplicate tenure parsing - Extract _build_poi_distances_lookup from stream endpoint to reduce nesting - Fix csv_exporter: optional decisions.json, NaN instead of -1 sentinels, guard against division by zero on missing square meters - Fix notifications.py broken list[Surface]() constructor, database.py stale comments and missing type annotation, auth.py type:ignore, ui_exporter.py stale TODO - Fix 3 pre-existing test failures: mock cache layer in streaming tests, bypass rate limiter for test isolation, fix cache invalidation test to account for two-pattern scan loop
This commit is contained in:
parent
6897820cc7
commit
f833309297
20 changed files with 199 additions and 178 deletions
|
|
@ -204,8 +204,12 @@ class TestInvalidateCache:
|
|||
mock_client = mock.MagicMock()
|
||||
mock_pipeline = mock.MagicMock()
|
||||
mock_client.pipeline.return_value = mock_pipeline
|
||||
# Simulate one scan iteration that returns keys, then done
|
||||
mock_client.scan.return_value = (0, ["listings:geojson:abc", "listings:geojson:def"])
|
||||
# invalidate_cache scans two patterns (CACHE_PREFIX*, STAGING_PREFIX*)
|
||||
# First scan returns matching keys, second returns none
|
||||
mock_client.scan.side_effect = [
|
||||
(0, ["listings:geojson:abc", "listings:geojson:def"]),
|
||||
(0, []),
|
||||
]
|
||||
mock_get_client.return_value = mock_client
|
||||
|
||||
invalidate_cache()
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
|||
import pytest
|
||||
|
||||
from models.listing import ListingType, QueryParameters
|
||||
from rec.exceptions import CircuitBreakerOpenError, ThrottlingError
|
||||
from rec.exceptions import CircuitBreakerOpenError, InvalidResponseError, ThrottlingError
|
||||
from services.listing_fetcher import (
|
||||
NUM_WORKERS,
|
||||
_fetch_subquery,
|
||||
|
|
@ -227,7 +227,7 @@ class TestFetchSubquery:
|
|||
with patch(
|
||||
"services.listing_fetcher.listing_query",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=Exception("GENERIC_ERROR: no more results"),
|
||||
side_effect=InvalidResponseError("GENERIC_ERROR: no more results"),
|
||||
):
|
||||
ids_found = await _fetch_subquery(
|
||||
sq=sq,
|
||||
|
|
|
|||
|
|
@ -3,12 +3,12 @@ from datetime import datetime
|
|||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
import pytest
|
||||
from models.listing import FurnishType, ListingType
|
||||
from config.scraper_config import MAX_OCR_WORKERS
|
||||
from listing_processor import (
|
||||
_parse_furnish_type,
|
||||
_parse_available_from,
|
||||
ListingProcessor,
|
||||
FetchListingDetailsStep,
|
||||
MAX_OCR_WORKERS,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -77,7 +77,7 @@ class TestListingProcessor:
|
|||
processor = ListingProcessor(mock_repo)
|
||||
for step in processor.process_steps:
|
||||
step.needs_processing = AsyncMock(return_value=True)
|
||||
step.process = AsyncMock(side_effect=Exception("fail"))
|
||||
step.process = AsyncMock(side_effect=ValueError("fail"))
|
||||
result = await processor.process_listing(123)
|
||||
assert result is None
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue