viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

changing to 500 listings per query

Browse source
This commit is contained in:
Kadir 2024-04-01 20:28:15 +02:00
parent 7ce54a01bd
commit b16fee0648
2 changed files with 6 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

5
crawler/1_dump_listings.py
View file

@ -9,7 +9,7 @@ districts = get_districts()
for district, locid in districts.items(): for district, locid in districts.items():
print("#### District:", district) print("#### District:", district)
for i in range(1, 41): for i in [1, 2]:
try: try:
d = listing_query( d = listing_query(
page=i, page=i,
@ -19,6 +19,7 @@ for district, locid in districts.items():
min_price=0, min_price=0,
max_price=800000, max_price=800000,
location_id=locid, location_id=locid,
page_size=500,
) )
except Exception as e: except Exception as e:
print(e) print(e)
@ -35,4 +36,4 @@ for district, locid in districts.items():
listing = Listing(identifier) listing = Listing(identifier)
with open(listing.path_listing_json(), "w") as f: with open(listing.path_listing_json(), "w") as f:
json.dump(property, f) json.dump(property, f)
print() # break line as we used end=, above. print() # break line as we used end=, above.

6
crawler/rec/query.py
View file

@ -49,16 +49,17 @@ def listing_query(
radius: float, radius: float,
min_price: int, min_price: int,
max_price: int, max_price: int,
location_id: str = "STATION^5168", # kings cross station location_id: str = "STATION^5168", # kings cross station
mustNewHome: bool = False, mustNewHome: bool = False,
max_days_since_added: int = None, max_days_since_added: int = None,
property_type: List["PropertyType"] = [], property_type: List["PropertyType"] = [],
page_size=25,
) -> dict: ) -> dict:
params = { params = {
"locationIdentifier": location_id, "locationIdentifier": location_id,
"channel": "BUY", "channel": "BUY",
"page": str(page), "page": str(page),
"numberOfPropertiesPerPage": "100", "numberOfPropertiesPerPage": str(page_size),
"radius": str(radius), "radius": str(radius),
"sortBy": "distance", "sortBy": "distance",
"includeUnavailableProperties": "false", "includeUnavailableProperties": "false",
@ -111,4 +112,3 @@ if __name__ == "__main__":
lon=d["longitude"], lon=d["longitude"],
) )
rl.save() rl.save()