viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

reduce concurrency when fetching images + add retries

Browse source
This commit is contained in:
Viktor Barzin 2025-07-01 16:12:06 +00:00
parent 59c33428c2
commit 20ff91d663
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
3 changed files with 8 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

6
crawler/3_dump_images.py
View file

@ -3,13 +3,13 @@ import json
from pathlib import Path
import aiohttp
from repositories import ListingRepository
from tenacity import retry, wait_random
from tenacity import retry, stop_after_attempt, wait_random
from tqdm.asyncio import tqdm
from models import Listing
# Setting this too high either crashes rightmove or gets us blocked
semaphore = asyncio.Semaphore(10)
semaphore = asyncio.Semaphore(5)
async def dump_images(
@ -25,7 +25,7 @@ async def dump_images(
)
@retry(wait=wait_random(min=1, max=2))
@retry(wait=wait_random(min=1, max=2), stop=stop_after_attempt(3))
async def dump_images_for_listing(listing: Listing, base_path: Path) -> Listing | None:
all_floorplans = listing.additional_info["property"]["floorplans"]
for floorplan in all_floorplans: