468 lines
14 KiB
Python
468 lines
14 KiB
Python
import enum
|
|
import logging
|
|
import time
|
|
from typing import Any
|
|
from contextlib import asynccontextmanager
|
|
from collections.abc import AsyncIterator
|
|
|
|
import aiohttp
|
|
from models.listing import FurnishType, ListingType
|
|
from rec import districts
|
|
from rec.exceptions import (
|
|
CircuitBreakerOpenError,
|
|
ThrottlingError,
|
|
)
|
|
from rec.throttle_detector import get_throttle_metrics, validate_response
|
|
from rec.circuit_breaker import CircuitBreaker
|
|
from tenacity import (
|
|
retry,
|
|
retry_if_exception_type,
|
|
stop_after_attempt,
|
|
wait_exponential,
|
|
wait_random,
|
|
)
|
|
from config.scraper_config import ScraperConfig
|
|
|
|
logger = logging.getLogger("uvicorn.error")
|
|
|
|
# Global circuit breaker instance
|
|
_circuit_breaker: CircuitBreaker | None = None
|
|
|
|
|
|
DEFAULT_HEADERS = {
|
|
"Host": "api.rightmove.co.uk",
|
|
"User-Agent": "okhttp/4.12.0",
|
|
"Connection": "keep-alive",
|
|
}
|
|
|
|
|
|
class PropertyType(enum.StrEnum):
|
|
BUNGALOW = "bungalow"
|
|
DETACHED = "detached"
|
|
FLAT = "flat"
|
|
LAND = "land"
|
|
PARK_HOME = "park-home"
|
|
SEMI_DETACHED = "semi-detached"
|
|
TERRACED = "terraced"
|
|
|
|
|
|
@asynccontextmanager
|
|
async def create_session(
|
|
config: ScraperConfig | None = None,
|
|
) -> AsyncIterator[aiohttp.ClientSession]:
|
|
"""Create an aiohttp session with optional proxy support.
|
|
|
|
Args:
|
|
config: Scraper configuration. Loads from environment if not provided.
|
|
|
|
Yields:
|
|
Configured aiohttp ClientSession.
|
|
"""
|
|
if config is None:
|
|
config = ScraperConfig.from_env()
|
|
|
|
connector = None
|
|
if config.proxy_url:
|
|
try:
|
|
from aiohttp_socks import ProxyConnector
|
|
|
|
connector = ProxyConnector.from_url(config.proxy_url)
|
|
except ImportError:
|
|
raise ImportError(
|
|
"aiohttp-socks is required for proxy support. "
|
|
"Install with: pip install aiohttp-socks"
|
|
)
|
|
|
|
session = aiohttp.ClientSession(
|
|
trust_env=True,
|
|
connector=connector,
|
|
headers=DEFAULT_HEADERS,
|
|
)
|
|
try:
|
|
yield session
|
|
finally:
|
|
await session.close()
|
|
|
|
|
|
def get_circuit_breaker(config: ScraperConfig | None = None) -> CircuitBreaker | None:
|
|
"""Get the global circuit breaker instance.
|
|
|
|
Args:
|
|
config: Configuration for initializing the circuit breaker.
|
|
|
|
Returns:
|
|
CircuitBreaker instance if enabled, None otherwise.
|
|
"""
|
|
global _circuit_breaker
|
|
if config is None:
|
|
config = ScraperConfig.from_env()
|
|
|
|
if not config.enable_circuit_breaker:
|
|
return None
|
|
|
|
if _circuit_breaker is None:
|
|
_circuit_breaker = CircuitBreaker(
|
|
failure_threshold=config.circuit_breaker_failure_threshold,
|
|
recovery_timeout=config.circuit_breaker_recovery_timeout,
|
|
)
|
|
return _circuit_breaker
|
|
|
|
|
|
def reset_circuit_breaker() -> None:
|
|
"""Reset the global circuit breaker."""
|
|
global _circuit_breaker
|
|
if _circuit_breaker is not None:
|
|
_circuit_breaker.reset()
|
|
|
|
|
|
def check_circuit_breaker(config: ScraperConfig | None = None) -> None:
|
|
"""Check if the circuit breaker allows requests.
|
|
|
|
Args:
|
|
config: Configuration for the circuit breaker.
|
|
|
|
Raises:
|
|
CircuitBreakerOpenError: If the circuit is open.
|
|
"""
|
|
cb = get_circuit_breaker(config)
|
|
if cb is not None:
|
|
cb.call()
|
|
|
|
|
|
@retry(
|
|
retry=retry_if_exception_type(ThrottlingError),
|
|
wait=wait_exponential(multiplier=2, min=2, max=120),
|
|
stop=stop_after_attempt(5),
|
|
)
|
|
async def detail_query(
|
|
detail_id: int,
|
|
session: aiohttp.ClientSession | None = None,
|
|
config: ScraperConfig | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Fetch detailed property information.
|
|
|
|
Args:
|
|
detail_id: The property identifier.
|
|
session: Optional aiohttp session. Creates new one if not provided.
|
|
config: Scraper configuration. Loads from environment if not provided.
|
|
|
|
Returns:
|
|
Property details as a dictionary.
|
|
|
|
Raises:
|
|
CircuitBreakerOpenError: If the circuit breaker is open.
|
|
ThrottlingError: If the request is throttled.
|
|
"""
|
|
if config is None:
|
|
config = ScraperConfig.from_env()
|
|
|
|
check_circuit_breaker(config)
|
|
cb = get_circuit_breaker(config)
|
|
|
|
params = {
|
|
"apiApplication": "ANDROID",
|
|
"appVersion": "3.70.0",
|
|
}
|
|
url = f"https://api.rightmove.co.uk/api/property/{detail_id}"
|
|
|
|
async def do_request(s: aiohttp.ClientSession) -> dict[str, Any]:
|
|
start_time = time.time()
|
|
try:
|
|
async with s.get(url, params=params, headers=DEFAULT_HEADERS) as response:
|
|
response_time = time.time() - start_time
|
|
body = await response.json() if response.status == 200 else None
|
|
|
|
# Validate response for throttling
|
|
validate_response(
|
|
response,
|
|
response_time,
|
|
body,
|
|
config.slow_response_threshold,
|
|
expect_data=True,
|
|
)
|
|
|
|
if response.status != 200:
|
|
raise Exception(
|
|
f"""id: {detail_id}. Status Code: {response.status}."""
|
|
f"""Failed due to: {await response.text()}"""
|
|
)
|
|
|
|
if cb is not None:
|
|
cb.record_success()
|
|
return body # type: ignore
|
|
except ThrottlingError:
|
|
if cb is not None:
|
|
cb.record_failure()
|
|
raise
|
|
except Exception as e:
|
|
if cb is not None:
|
|
cb.record_failure()
|
|
raise e
|
|
|
|
if session:
|
|
return await do_request(session)
|
|
else:
|
|
async with aiohttp.ClientSession(trust_env=True) as new_session:
|
|
return await do_request(new_session)
|
|
|
|
|
|
@retry(
|
|
retry=retry_if_exception_type(ThrottlingError),
|
|
wait=wait_exponential(multiplier=2, min=2, max=120),
|
|
stop=stop_after_attempt(5),
|
|
)
|
|
async def listing_query(
|
|
*,
|
|
page: int,
|
|
channel: ListingType,
|
|
min_bedrooms: int,
|
|
max_bedrooms: int,
|
|
radius: float,
|
|
min_price: int,
|
|
max_price: int,
|
|
district: str, # = "STATION^5168", # kings cross station
|
|
mustNewHome: bool = False,
|
|
max_days_since_added: int = 30,
|
|
property_type: list[PropertyType] = [],
|
|
page_size: int = 25,
|
|
furnish_types: list[FurnishType] = [],
|
|
session: aiohttp.ClientSession | None = None,
|
|
config: ScraperConfig | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Execute a listing search query.
|
|
|
|
Args:
|
|
page: Page number to fetch (1-indexed).
|
|
channel: Listing type (BUY or RENT).
|
|
min_bedrooms: Minimum number of bedrooms.
|
|
max_bedrooms: Maximum number of bedrooms.
|
|
radius: Search radius.
|
|
min_price: Minimum price.
|
|
max_price: Maximum price.
|
|
district: District identifier string.
|
|
mustNewHome: Filter for new homes only (BUY only).
|
|
max_days_since_added: Maximum days since listing was added (BUY only).
|
|
property_type: List of property types to filter (BUY only).
|
|
page_size: Number of results per page (default 25).
|
|
furnish_types: List of furnish types to filter (RENT only).
|
|
session: Optional aiohttp session. Creates new one if not provided.
|
|
config: Scraper configuration. Loads from environment if not provided.
|
|
|
|
Returns:
|
|
API response as a dictionary.
|
|
|
|
Raises:
|
|
CircuitBreakerOpenError: If the circuit breaker is open.
|
|
ThrottlingError: If the request is throttled.
|
|
"""
|
|
if config is None:
|
|
config = ScraperConfig.from_env()
|
|
|
|
check_circuit_breaker(config)
|
|
cb = get_circuit_breaker(config)
|
|
|
|
params: dict[str, str] = {
|
|
"locationIdentifier": districts.get_districts()[district],
|
|
"channel": str(channel).upper(),
|
|
"page": str(page),
|
|
"numberOfPropertiesPerPage": str(page_size),
|
|
"radius": str(radius),
|
|
"sortBy": "distance",
|
|
"includeUnavailableProperties": "false",
|
|
"minPrice": str(min_price),
|
|
"maxPrice": str(max_price),
|
|
"minBedrooms": str(min_bedrooms),
|
|
"maxBedrooms": str(max_bedrooms),
|
|
"apiApplication": "ANDROID",
|
|
"appVersion": "4.28.0",
|
|
}
|
|
if channel is ListingType.BUY:
|
|
params["dontShow"] = "sharedOwnership,retirement"
|
|
if len(property_type) > 0:
|
|
params["propertyTypes"] = ",".join(property_type)
|
|
if max_days_since_added is not None and max_days_since_added not in [
|
|
1,
|
|
3,
|
|
7,
|
|
14,
|
|
]:
|
|
raise Exception(
|
|
f"Invalid max days - {max_days_since_added} Can only be got",
|
|
[1, 3, 7, 14],
|
|
)
|
|
params["maxDaysSinceAdded"] = str(max_days_since_added)
|
|
|
|
if mustNewHome:
|
|
params["mustHave"] = "newHome"
|
|
if channel is ListingType.RENT:
|
|
if furnish_types:
|
|
params["furnishTypes"] = ",".join(furnish_types)
|
|
|
|
request_headers = {
|
|
"Host": "api.rightmove.co.uk",
|
|
"Accept-Encoding": "gzip, deflate, br",
|
|
"User-Agent": "okhttp/4.12.0",
|
|
"Connection": "keep-alive",
|
|
}
|
|
|
|
async def do_request(s: aiohttp.ClientSession) -> dict[str, Any]:
|
|
start_time = time.time()
|
|
try:
|
|
async with s.get(
|
|
"https://api.rightmove.co.uk/api/property-listing",
|
|
params=params,
|
|
headers=request_headers,
|
|
) as response:
|
|
response_time = time.time() - start_time
|
|
body = await response.json() if response.status == 200 else None
|
|
|
|
# Validate response for throttling
|
|
validate_response(
|
|
response,
|
|
response_time,
|
|
body,
|
|
config.slow_response_threshold,
|
|
expect_data=(page == 1), # Only expect data on first page
|
|
)
|
|
|
|
if response.status != 200:
|
|
raise Exception(f"Failed due to: {await response.text()}")
|
|
|
|
if cb is not None:
|
|
cb.record_success()
|
|
return body # type: ignore
|
|
except ThrottlingError:
|
|
if cb is not None:
|
|
cb.record_failure()
|
|
raise
|
|
except Exception as e:
|
|
if cb is not None:
|
|
cb.record_failure()
|
|
raise e
|
|
|
|
if session:
|
|
return await do_request(session)
|
|
else:
|
|
async with aiohttp.ClientSession(trust_env=True) as new_session:
|
|
return await do_request(new_session)
|
|
|
|
|
|
@retry(
|
|
retry=retry_if_exception_type(ThrottlingError),
|
|
wait=wait_exponential(multiplier=2, min=2, max=60),
|
|
stop=stop_after_attempt(5),
|
|
)
|
|
async def probe_query(
|
|
*,
|
|
session: aiohttp.ClientSession,
|
|
channel: ListingType,
|
|
min_bedrooms: int,
|
|
max_bedrooms: int,
|
|
radius: float,
|
|
min_price: int,
|
|
max_price: int,
|
|
district: str,
|
|
max_days_since_added: int = 30,
|
|
furnish_types: list[FurnishType] = [],
|
|
config: ScraperConfig | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Probe the API to get result count without fetching full results.
|
|
|
|
Makes a minimal request (page_size=1) to efficiently get totalAvailableResults.
|
|
|
|
Args:
|
|
session: aiohttp session for making requests.
|
|
channel: Listing type (BUY or RENT).
|
|
min_bedrooms: Minimum number of bedrooms.
|
|
max_bedrooms: Maximum number of bedrooms.
|
|
radius: Search radius.
|
|
min_price: Minimum price.
|
|
max_price: Maximum price.
|
|
district: District identifier string.
|
|
max_days_since_added: Maximum days since listing was added (BUY only).
|
|
furnish_types: List of furnish types to filter (RENT only).
|
|
config: Scraper configuration. Loads from environment if not provided.
|
|
|
|
Returns:
|
|
API response containing totalAvailableResults.
|
|
|
|
Raises:
|
|
CircuitBreakerOpenError: If the circuit breaker is open.
|
|
ThrottlingError: If the request is throttled.
|
|
"""
|
|
if config is None:
|
|
config = ScraperConfig.from_env()
|
|
|
|
check_circuit_breaker(config)
|
|
cb = get_circuit_breaker(config)
|
|
|
|
params: dict[str, str] = {
|
|
"locationIdentifier": districts.get_districts()[district],
|
|
"channel": str(channel).upper(),
|
|
"page": "1",
|
|
"numberOfPropertiesPerPage": "1", # Minimal page size for probing
|
|
"radius": str(radius),
|
|
"sortBy": "distance",
|
|
"includeUnavailableProperties": "false",
|
|
"minPrice": str(min_price),
|
|
"maxPrice": str(max_price),
|
|
"minBedrooms": str(min_bedrooms),
|
|
"maxBedrooms": str(max_bedrooms),
|
|
"apiApplication": "ANDROID",
|
|
"appVersion": "4.28.0",
|
|
}
|
|
|
|
if channel is ListingType.BUY:
|
|
params["dontShow"] = "sharedOwnership,retirement"
|
|
if max_days_since_added is not None and max_days_since_added in [
|
|
1,
|
|
3,
|
|
7,
|
|
14,
|
|
]:
|
|
params["maxDaysSinceAdded"] = str(max_days_since_added)
|
|
|
|
if channel is ListingType.RENT:
|
|
if furnish_types:
|
|
params["furnishTypes"] = ",".join(furnish_types)
|
|
|
|
request_headers = {
|
|
"Host": "api.rightmove.co.uk",
|
|
"Accept-Encoding": "gzip, deflate, br",
|
|
"User-Agent": "okhttp/4.12.0",
|
|
"Connection": "keep-alive",
|
|
}
|
|
|
|
start_time = time.time()
|
|
try:
|
|
async with session.get(
|
|
"https://api.rightmove.co.uk/api/property-listing",
|
|
params=params,
|
|
headers=request_headers,
|
|
) as response:
|
|
response_time = time.time() - start_time
|
|
body = await response.json() if response.status == 200 else None
|
|
|
|
# Validate response for throttling
|
|
validate_response(
|
|
response,
|
|
response_time,
|
|
body,
|
|
config.slow_response_threshold,
|
|
expect_data=False, # Probe doesn't need data, just count
|
|
)
|
|
|
|
if response.status != 200:
|
|
raise Exception(f"Probe failed: {await response.text()}")
|
|
|
|
if cb is not None:
|
|
cb.record_success()
|
|
return body # type: ignore
|
|
except ThrottlingError:
|
|
if cb is not None:
|
|
cb.record_failure()
|
|
raise
|
|
except Exception as e:
|
|
if cb is not None:
|
|
cb.record_failure()
|
|
raise e
|