viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

limit the number of concurrenct requests when dumping listings as right move blocks us

Browse source
This commit is contained in:
Viktor Barzin 2025-06-01 00:27:12 +00:00
parent 24bf44caf9
commit 9735db72a0
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
2 changed files with 17 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crawler/1_dump_listings.py
View file

@ -39,6 +39,7 @@ async def dump_listings(
print("Valid districts to scrape:", districts.keys())
listings = []
semaphore = asyncio.Semaphore(5) # if too high, rightmove drops connections
json_responses = await asyncio.gather(
*[
listing_query(
@ -53,6 +54,7 @@ async def dump_listings(
page_size=parameters.page_size,
max_days_since_added=parameters.max_days_since_added,
furnish_types=parameters.furnish_types or [],
semaphore=semaphore,
)
for locid in districts.values()
for i in [1, 2]