changing to 500 listings per query

crawler/1_dump_listings.py

@@ -9,7 +9,7 @@ districts = get_districts()
 
 for district, locid in districts.items():
     print("#### District:", district)
-    for i in range(1, 41):
+    for i in [1, 2]:
         try:
             d = listing_query(
                 page=i,
@@ -19,6 +19,7 @@ for district, locid in districts.items():
                 min_price=0,
                 max_price=800000,
                 location_id=locid,
+                page_size=500,
             )
         except Exception as e:
             print(e)
@@ -35,4 +36,4 @@ for district, locid in districts.items():
             listing = Listing(identifier)
             with open(listing.path_listing_json(), "w") as f:
                 json.dump(property, f)
-    print() # break line as we used end=, above.
+    print()  # break line as we used end=, above.

crawler/rec/query.py

@@ -49,16 +49,17 @@ def listing_query(
     radius: float,
     min_price: int,
     max_price: int,
-    location_id: str = "STATION^5168", # kings cross station
+    location_id: str = "STATION^5168",  # kings cross station
     mustNewHome: bool = False,
     max_days_since_added: int = None,
     property_type: List["PropertyType"] = [],
+    page_size=25,
 ) -> dict:
     params = {
         "locationIdentifier": location_id,
         "channel": "BUY",
         "page": str(page),
-        "numberOfPropertiesPerPage": "100",
+        "numberOfPropertiesPerPage": str(page_size),
         "radius": str(radius),
         "sortBy": "distance",
         "includeUnavailableProperties": "false",
@@ -111,4 +112,3 @@ if __name__ == "__main__":
             lon=d["longitude"],
         )
         rl.save()
-