viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add configurable request timeout and retry on TimeoutError

Browse source 
Requests to Rightmove API previously had no explicit timeout, causing
hung connections to block workers indefinitely. Add a configurable
request_timeout (default 30s) to ScraperConfig and apply it to all
aiohttp sessions. Also retry on TimeoutError in addition to
ThrottlingError for all API query functions.
This commit is contained in:
Viktor Barzin 2026-02-21 17:50:25 +00:00
parent 7a1042741e
commit 578b97b0c5
No known key found for this signature in database
GPG key ID: 0EB088298288D958
2 changed files with 14 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

12
rec/query.py
View file

@ -88,6 +88,7 @@ async def create_session(
trust_env=True,
connector=connector,
headers=DEFAULT_HEADERS,
timeout=aiohttp.ClientTimeout(total=config.request_timeout),
)
try:
yield session
@ -307,12 +308,15 @@ async def _execute_api_request(
if session:
return await do_request(session)
else:
async with aiohttp.ClientSession(trust_env=True) as new_session:
async with aiohttp.ClientSession(
trust_env=True,
timeout=aiohttp.ClientTimeout(total=config.request_timeout),
) as new_session:
return await do_request(new_session)
@retry(
retry=retry_if_exception_type(ThrottlingError),
retry=retry_if_exception_type((ThrottlingError, TimeoutError)),
wait=wait_exponential(multiplier=2, min=2, max=120),
stop=stop_after_attempt(5),
)
@ -356,7 +360,7 @@ async def detail_query(
@retry(
retry=retry_if_exception_type(ThrottlingError),
retry=retry_if_exception_type((ThrottlingError, TimeoutError)),
wait=wait_exponential(multiplier=2, min=2, max=120),
stop=stop_after_attempt(5),
)
@ -438,7 +442,7 @@ async def listing_query(
@retry(
retry=retry_if_exception_type(ThrottlingError),
retry=retry_if_exception_type((ThrottlingError, TimeoutError)),
wait=wait_exponential(multiplier=2, min=2, max=60),
stop=stop_after_attempt(5),
)