adding better exception messages and interrupt free crawling

crawler/2_dump_detail.py

@@ -15,6 +15,5 @@ for listing in tqdm(filtered_listings):
         d = detail_query(listing.identifier)
         with open(listing.path_detail_json(), "w") as f:
             json.dump(d, f)
-    except:
-        print("Failed at: ", listing.identifier)
-        raise
+    except Exception as e:
+        print(e)

crawler/data_access.py

@@ -161,6 +161,10 @@ class Listing:
     @property
     def price(self) -> float:
         return self.detailobject["property"]["price"]
+    
+    @property
+    def tenure_type(self) -> str:
+        return self.detailobject["property"]["tenureType"]
 
     @property
     def price_per_sqm(self) -> float:
@@ -228,6 +232,7 @@ class Listing:
             "lease_left": self.leaseLeft,
             "service_charge": self.serviceCharge,
             "development": self.development,
+            "tenure_type": self.tenure_type,
         }
 
 

crawler/rec/query.py

@@ -36,7 +36,7 @@ def detail_query(detail_id: int):
     url = f"https://api.rightmove.co.uk/api/property/{detail_id}"
     response = requests.get(url, params=params, headers=headers, verify=False)
     if response.status_code != 200:
-        raise Exception("Failed due to: ", response.text)
+        raise Exception(f"id: {detail_id}. Status Code: {response.status_code}. Failed due to: {response.text}")
 
     return response.json()