viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor the semaphore when dumping listings

Browse source
This commit is contained in:
Viktor Barzin 2025-06-06 20:08:38 +00:00
parent b7a2ea75aa
commit 29213f3d26
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
2 changed files with 34 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

40
crawler/1_dump_listings.py
View file

@ -1,6 +1,6 @@
import asyncio
from dataclasses import dataclass
import pathlib
from typing import Any
from rec.query import listing_query, QueryParameters
from rec.districts import get_districts
from data_access import Listing
@ -24,20 +24,7 @@ async def dump_listings(
semaphore = asyncio.Semaphore(5) # if too high, rightmove drops connections
json_responses = await asyncio.gather(
*[
listing_query(
page=i,
channel=parameters.listing_type,
min_bedrooms=parameters.min_bedrooms,
max_bedrooms=parameters.max_bedrooms,
radius=parameters.radius,
min_price=parameters.min_price,
max_price=parameters.max_price,
location_id=locid,
page_size=parameters.page_size,
max_days_since_added=parameters.max_days_since_added,
furnish_types=parameters.furnish_types or [],
semaphore=semaphore,
)
_dump_listings_with_semaphore(semaphore, i, parameters, locid)
for locid in districts.values()
for i in [1, 2]
]
@ -57,3 +44,26 @@ async def dump_listings(
listings.append(listing)
return listings
async def _dump_listings_with_semaphore(
semaphore: asyncio.Semaphore,
page_id: int,
parameters: QueryParameters,
location_id: str,
) -> dict[str, Any]:
async with semaphore:
listing = await listing_query(
page=page_id,
channel=parameters.listing_type,
min_bedrooms=parameters.min_bedrooms,
max_bedrooms=parameters.max_bedrooms,
radius=parameters.radius,
min_price=parameters.min_price,
max_price=parameters.max_price,
location_id=location_id,
page_size=parameters.page_size,
max_days_since_added=parameters.max_days_since_added,
furnish_types=parameters.furnish_types or [],
)
return listing