"""Unit tests for rec/query.py.""" import pytest from unittest.mock import AsyncMock, MagicMock, patch import aiohttp from rec.query import ( detail_query, listing_query, probe_query, PropertyType, create_session, _build_base_params, _build_listing_params, _build_probe_params, ANDROID_APP_VERSION, ANDROID_APP_VERSION_LISTING, RIGHTMOVE_API_BASE, PROPERTY_LISTING_ENDPOINT, DEFAULT_HEADERS, LISTING_HEADERS, check_circuit_breaker, reset_circuit_breaker, get_circuit_breaker, ) from models.listing import ListingType, FurnishType from config.scraper_config import ScraperConfig from rec.exceptions import CircuitBreakerOpenError from rec.throttle_detector import reset_throttle_metrics @pytest.fixture def config() -> ScraperConfig: return ScraperConfig( max_concurrent_requests=5, request_delay_ms=10, slow_response_threshold=10.0, enable_circuit_breaker=True, circuit_breaker_failure_threshold=3, circuit_breaker_recovery_timeout=0.5, ) @pytest.fixture def config_no_cb() -> ScraperConfig: return ScraperConfig(enable_circuit_breaker=False) @pytest.fixture(autouse=True) def reset_globals() -> None: reset_throttle_metrics() reset_circuit_breaker() class MockResponse: def __init__( self, status: int = 200, json_data: dict | None = None, text: str = "", ): self.status = status self._json_data = json_data or {} self._text = text async def json(self) -> dict: return self._json_data async def text(self) -> str: return self._text async def __aenter__(self) -> "MockResponse": return self async def __aexit__(self, *args: object) -> None: pass def make_mock_session(response: MockResponse) -> MagicMock: """Create a mock session whose .get() returns an async context manager.""" mock_session = MagicMock() mock_session.get = MagicMock(return_value=response) return mock_session def make_mock_session_fn(get_fn: object) -> MagicMock: """Create a mock session whose .get() calls a function to produce responses.""" mock_session = MagicMock() mock_session.get = MagicMock(side_effect=get_fn) return mock_session class TestBuildBaseParams: def test_constructs_correct_params(self) -> None: with patch("rec.query.districts.get_districts", return_value={"TestDistrict": "REGION^123"}): params = _build_base_params( channel=ListingType.RENT, page=2, page_size=25, radius=1.5, min_price=1000, max_price=3000, min_bedrooms=1, max_bedrooms=3, district="TestDistrict", ) assert params["locationIdentifier"] == "REGION^123" assert params["channel"] == "RENT" assert params["page"] == "2" assert params["numberOfPropertiesPerPage"] == "25" assert params["radius"] == "1.5" assert params["sortBy"] == "distance" assert params["includeUnavailableProperties"] == "false" assert params["minPrice"] == "1000" assert params["maxPrice"] == "3000" assert params["minBedrooms"] == "1" assert params["maxBedrooms"] == "3" assert params["apiApplication"] == "ANDROID" assert params["appVersion"] == ANDROID_APP_VERSION_LISTING def test_buy_channel_includes_dont_show_and_max_days(self) -> None: with patch("rec.query.districts.get_districts", return_value={"D": "LOC1"}): params = _build_listing_params( page=1, channel=ListingType.BUY, min_bedrooms=1, max_bedrooms=2, radius=1.0, min_price=100000, max_price=500000, district="D", mustNewHome=False, max_days_since_added=7, property_type=[], page_size=25, furnish_types=[], ) assert params["dontShow"] == "sharedOwnership,retirement" assert params["maxDaysSinceAdded"] == "7" def test_rent_channel_includes_furnish_types(self) -> None: with patch("rec.query.districts.get_districts", return_value={"D": "LOC1"}): params = _build_listing_params( page=1, channel=ListingType.RENT, min_bedrooms=1, max_bedrooms=2, radius=1.0, min_price=1000, max_price=3000, district="D", mustNewHome=False, max_days_since_added=30, property_type=[], page_size=25, furnish_types=[FurnishType.FURNISHED, FurnishType.UNFURNISHED], ) assert params["furnishTypes"] == "furnished,unfurnished" assert "dontShow" not in params assert "maxDaysSinceAdded" not in params def test_buy_channel_probe_includes_dont_show(self) -> None: with patch("rec.query.districts.get_districts", return_value={"D": "LOC1"}): params = _build_probe_params( channel=ListingType.BUY, min_bedrooms=1, max_bedrooms=2, radius=1.0, min_price=100000, max_price=500000, district="D", max_days_since_added=7, furnish_types=[], ) assert params["dontShow"] == "sharedOwnership,retirement" assert params["maxDaysSinceAdded"] == "7" assert params["numberOfPropertiesPerPage"] == "1" def test_probe_buy_skips_max_days_if_not_valid(self) -> None: with patch("rec.query.districts.get_districts", return_value={"D": "LOC1"}): params = _build_probe_params( channel=ListingType.BUY, min_bedrooms=1, max_bedrooms=2, radius=1.0, min_price=100000, max_price=500000, district="D", max_days_since_added=30, furnish_types=[], ) # 30 is not in [1, 3, 7, 14], so maxDaysSinceAdded is not added for probe assert "maxDaysSinceAdded" not in params class TestMutableDefaultArgFix: @pytest.mark.asyncio async def test_property_type_default_not_shared(self, config: ScraperConfig) -> None: """Calling listing_query with no property_type should not share state between calls.""" response = MockResponse( status=200, json_data={"totalAvailableResults": 0, "properties": []}, ) mock_session = make_mock_session(response) with patch("rec.query.districts.get_districts", return_value={"D": "LOC1"}): # Call twice without explicit property_type await listing_query( page=1, channel=ListingType.RENT, min_bedrooms=1, max_bedrooms=2, radius=1.0, min_price=1000, max_price=2000, district="D", session=mock_session, config=config, ) await listing_query( page=1, channel=ListingType.RENT, min_bedrooms=1, max_bedrooms=2, radius=1.0, min_price=1000, max_price=2000, district="D", session=mock_session, config=config, ) # If mutable default was shared, this test would detect mutations. # The fact that it completes without error proves defaults are independent. @pytest.mark.asyncio async def test_furnish_types_default_not_shared(self, config: ScraperConfig) -> None: """Calling probe_query with no furnish_types should not share state between calls.""" response = MockResponse( status=200, json_data={"totalAvailableResults": 0, "properties": []}, ) mock_session = make_mock_session(response) with patch("rec.query.districts.get_districts", return_value={"D": "LOC1"}): await probe_query( session=mock_session, channel=ListingType.RENT, min_bedrooms=1, max_bedrooms=2, radius=1.0, min_price=1000, max_price=2000, district="D", config=config, ) await probe_query( session=mock_session, channel=ListingType.RENT, min_bedrooms=1, max_bedrooms=2, radius=1.0, min_price=1000, max_price=2000, district="D", config=config, ) class TestPropertyTypeEnum: def test_enum_values(self) -> None: assert PropertyType.BUNGALOW == "bungalow" assert PropertyType.DETACHED == "detached" assert PropertyType.FLAT == "flat" assert PropertyType.LAND == "land" assert PropertyType.PARK_HOME == "park-home" assert PropertyType.SEMI_DETACHED == "semi-detached" assert PropertyType.TERRACED == "terraced" def test_enum_is_str(self) -> None: assert isinstance(PropertyType.FLAT, str) assert ",".join([PropertyType.FLAT, PropertyType.DETACHED]) == "flat,detached" class TestDetailQuery: @pytest.mark.asyncio async def test_success_200(self, config: ScraperConfig) -> None: expected_body = {"id": 12345, "address": "123 Test St"} response = MockResponse(status=200, json_data=expected_body) mock_session = make_mock_session(response) result = await detail_query(12345, session=mock_session, config=config) assert result == expected_body @pytest.mark.asyncio async def test_raises_on_non_200(self, config: ScraperConfig) -> None: response = MockResponse(status=404, text="Not Found") mock_session = make_mock_session(response) with pytest.raises(Exception, match="Failed due to"): await detail_query(99999, session=mock_session, config=config) class TestCircuitBreakerBlocksRequests: @pytest.mark.asyncio async def test_circuit_breaker_blocks_when_open(self, config: ScraperConfig) -> None: cb = get_circuit_breaker(config) assert cb is not None for _ in range(config.circuit_breaker_failure_threshold): cb.record_failure() assert cb.is_open mock_session = MagicMock() with pytest.raises(CircuitBreakerOpenError): await detail_query(1, session=mock_session, config=config) @pytest.mark.asyncio async def test_circuit_breaker_blocks_listing_query(self, config: ScraperConfig) -> None: cb = get_circuit_breaker(config) assert cb is not None for _ in range(config.circuit_breaker_failure_threshold): cb.record_failure() mock_session = MagicMock() with patch("rec.query.districts.get_districts", return_value={"D": "LOC1"}): with pytest.raises(CircuitBreakerOpenError): await listing_query( page=1, channel=ListingType.RENT, min_bedrooms=1, max_bedrooms=2, radius=1.0, min_price=1000, max_price=2000, district="D", session=mock_session, config=config, ) @pytest.mark.asyncio async def test_circuit_breaker_blocks_probe_query(self, config: ScraperConfig) -> None: cb = get_circuit_breaker(config) assert cb is not None for _ in range(config.circuit_breaker_failure_threshold): cb.record_failure() mock_session = MagicMock() with patch("rec.query.districts.get_districts", return_value={"D": "LOC1"}): with pytest.raises(CircuitBreakerOpenError): await probe_query( session=mock_session, channel=ListingType.RENT, min_bedrooms=1, max_bedrooms=2, radius=1.0, min_price=1000, max_price=2000, district="D", config=config, ) class TestConstants: def test_android_app_version(self) -> None: assert ANDROID_APP_VERSION == "3.70.0" def test_android_app_version_listing(self) -> None: assert ANDROID_APP_VERSION_LISTING == "4.28.0" def test_rightmove_api_base(self) -> None: assert RIGHTMOVE_API_BASE == "https://api.rightmove.co.uk/api" def test_property_listing_endpoint(self) -> None: assert PROPERTY_LISTING_ENDPOINT == "https://api.rightmove.co.uk/api/property-listing" def test_listing_headers_extends_default(self) -> None: for key, value in DEFAULT_HEADERS.items(): assert LISTING_HEADERS[key] == value assert LISTING_HEADERS["Accept-Encoding"] == "gzip, deflate, br"