viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

retry transient errors from rightmove when fetching images

Browse source
This commit is contained in:
Viktor Barzin 2025-06-21 12:04:48 +00:00
parent ec2e0dded8
commit 4e13dbdb7f
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
1 changed files with 4 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

4
crawler/3_dump_images.py
View file

@ -3,6 +3,7 @@ import json
from pathlib import Path from pathlib import Path
import aiohttp import aiohttp
from repositories import ListingRepository from repositories import ListingRepository
from tenacity import retry, wait_random
from tqdm.asyncio import tqdm from tqdm.asyncio import tqdm
from models import Listing from models import Listing
@ -21,6 +22,7 @@ async def dump_images(repository: ListingRepository, image_base_path: Path):
) )
@retry(wait=wait_random(min=1, max=2))
async def dump_images_for_listing(listing: Listing, base_path: Path) -> Listing | None: async def dump_images_for_listing(listing: Listing, base_path: Path) -> Listing | None:
all_floorplans = listing.additional_info["property"]["floorplans"] all_floorplans = listing.additional_info["property"]["floorplans"]
for floorplan in all_floorplans: for floorplan in all_floorplans:
@ -33,6 +35,8 @@ async def dump_images_for_listing(listing: Listing, base_path: Path) -> Listing
async with semaphore: async with semaphore:
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
async with session.get(url) as response: async with session.get(url) as response:
if response.status == 404:
return None
if response.status != 200: if response.status != 200:
raise Exception(f"Error for {url}: {response.status}") raise Exception(f"Error for {url}: {response.status}")
floorplan_path.parent.mkdir(parents=True, exist_ok=True) floorplan_path.parent.mkdir(parents=True, exist_ok=True)