viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix types and format

Browse source
This commit is contained in:
Viktor Barzin 2025-05-18 12:27:26 +00:00
parent 91d3237516
commit b873eaf203
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
8 changed files with 117 additions and 172 deletions
Download patch file Download diff file Expand all files Collapse all files

48
crawler/1_dump_listings.py
View file

@ -16,12 +16,13 @@ class QueryParameters:
district_names: set[str]
radius: float = 0
page_size: int = 500 # items per page
max_days_since_added: int | None = None
max_days_since_added: int = 30
# available from; furnished/unfurnished; council tax
async def dump_listings(
parameters: QueryParameters,
data_dir: pathlib.Path = pathlib.Path("data/rs/"),
parameters: QueryParameters,
data_dir: pathlib.Path = pathlib.Path("data/rs/"),
) -> list[Listing]:
districts = {
district: locid
@ -31,29 +32,28 @@ async def dump_listings(
print("Valid districts to scrape:", districts.keys())
listings = []
json_responses = await asyncio.gather(
*[
listing_query(
page=i,
channel=parameters.listing_type,
min_bedrooms=parameters.min_bedrooms,
max_bedrooms=parameters.max_bedrooms,
radius=parameters.radius,
min_price=parameters.min_price,
max_price=parameters.max_price,
location_id=locid,
page_size=parameters.page_size,
max_days_since_added=parameters.max_days_since_added,
) for locid in districts.values() for i in [1, 2]
]
)
json_responses = await asyncio.gather(*[
listing_query(
page=i,
channel=parameters.listing_type,
min_bedrooms=parameters.min_bedrooms,
max_bedrooms=parameters.max_bedrooms,
radius=parameters.radius,
min_price=parameters.min_price,
max_price=parameters.max_price,
location_id=locid,
page_size=parameters.page_size,
max_days_since_added=parameters.max_days_since_added,
) for locid in districts.values() for i in [1, 2]
])
listings = []
for response_json in json_responses:
if response_json["totalAvailableResults"] == 0:
print("No results found")
continue
if response_json["totalAvailableResults"] > 0:
print("totalAvailableResults: ", response_json["totalAvailableResults"])
print("totalAvailableResults: ",
response_json["totalAvailableResults"])
for property in response_json["properties"]:
identifier = property["identifier"]
@ -62,11 +62,3 @@ async def dump_listings(
listings.append(listing)
return listings
def main():
dump_listings()
if __name__ == "__main__":
main()