diff --git a/crawler/1_dump_listings.py b/crawler/1_dump_listings.py index 8be24ac..0d05d6b 100644 --- a/crawler/1_dump_listings.py +++ b/crawler/1_dump_listings.py @@ -9,7 +9,7 @@ districts = get_districts() for district, locid in districts.items(): print("#### District:", district) - for i in range(1, 41): + for i in [1, 2]: try: d = listing_query( page=i, @@ -19,6 +19,7 @@ for district, locid in districts.items(): min_price=0, max_price=800000, location_id=locid, + page_size=500, ) except Exception as e: print(e) @@ -35,4 +36,4 @@ for district, locid in districts.items(): listing = Listing(identifier) with open(listing.path_listing_json(), "w") as f: json.dump(property, f) - print() # break line as we used end=, above. + print() # break line as we used end=, above. diff --git a/crawler/rec/query.py b/crawler/rec/query.py index 1c9896f..4e5c4e3 100644 --- a/crawler/rec/query.py +++ b/crawler/rec/query.py @@ -49,16 +49,17 @@ def listing_query( radius: float, min_price: int, max_price: int, - location_id: str = "STATION^5168", # kings cross station + location_id: str = "STATION^5168", # kings cross station mustNewHome: bool = False, max_days_since_added: int = None, property_type: List["PropertyType"] = [], + page_size=25, ) -> dict: params = { "locationIdentifier": location_id, "channel": "BUY", "page": str(page), - "numberOfPropertiesPerPage": "100", + "numberOfPropertiesPerPage": str(page_size), "radius": str(radius), "sortBy": "distance", "includeUnavailableProperties": "false", @@ -111,4 +112,3 @@ if __name__ == "__main__": lon=d["longitude"], ) rl.save() -