changing to 500 listings per query

crawler/1_dump_listings.py

@@ -9,7 +9,7 @@ districts = get_districts()
 
 for district, locid in districts.items():
     print("#### District:", district)
-    for i in range(1, 41):
+    for i in [1, 2]:
         try:
             d = listing_query(
                 page=i,
@@ -19,6 +19,7 @@ for district, locid in districts.items():
                 min_price=0,
                 max_price=800000,
                 location_id=locid,
+                page_size=500,
             )
         except Exception as e:
             print(e)

crawler/rec/query.py

@@ -53,12 +53,13 @@ def listing_query(
     mustNewHome: bool = False,
     max_days_since_added: int = None,
     property_type: List["PropertyType"] = [],
+    page_size=25,
 ) -> dict:
     params = {
         "locationIdentifier": location_id,
         "channel": "BUY",
         "page": str(page),
-        "numberOfPropertiesPerPage": "100",
+        "numberOfPropertiesPerPage": str(page_size),
         "radius": str(radius),
         "sortBy": "distance",
         "includeUnavailableProperties": "false",
@@ -111,4 +112,3 @@ if __name__ == "__main__":
             lon=d["longitude"],
         )
         rl.save()
-