viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add timeout when fetching details and use new entrypoint for task processing

Browse source
This commit is contained in:
Viktor Barzin 2025-08-23 22:20:42 +00:00
parent e5c68f6bb7
commit 480957dc72
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
2 changed files with 23 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

9
crawler/listing_processor.py
View file

@ -33,7 +33,11 @@ class ListingProcessor:
for step in self.process_steps:
if await step.needs_processing(listing_id):
async with self.semaphore:
listing = await step.process(listing_id)
try:
listing = await step.process(listing_id)
except Exception as e:
logger.error(f"Failed to process {listing_id=}: {e}")
return None
return listing
async def listing_exists(self, listing_id: int) -> bool: ...
@ -145,6 +149,7 @@ class FetchImagesStep(Step):
all_floorplans = listing.additional_info.get("property", {}).get(
"floorplans", []
)
client_timeout = aiohttp.ClientTimeout(total=30)
for floorplan in all_floorplans:
url = floorplan["url"]
picname = url.split("/")[-1]
@ -152,7 +157,7 @@ class FetchImagesStep(Step):
if floorplan_path.exists():
continue
async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
async with session.get(url, timeout=client_timeout) as response:
if response.status == 404:
return listing
if response.status != 200: