fixing full detail dumping

crawler/2_dump_detail.py

@@ -10,7 +10,13 @@ incremental = True
 listings = Listing.get_all_listings()
 filtered_listings = []
 for listing in listings:
-    if not incremental and not listing.isRemoved:
+    # We introduced last_seen later, so not all entries have it. 
+    # If it doesnt exist then its on the platform anymore. So skip
+    last_seen = listing.last_seen 
+    if last_seen is None:
+        continue
+
+    if not incremental and last_seen <= 1:
         filtered_listings.append(listing)
 
     if incremental and not listing.path_detail_json().exists():

crawler/data_access.py

@@ -163,6 +163,12 @@ class Listing:
     def url(self):
         return f"https://www.rightmove.co.uk/properties/{self.identifier}"
 
+    @property
+    def listingobject(self):
+        if self._cached is None:
+            with open(self.path_listing_json()) as f:
+                return json.load(f)
+    
     @property
     def detailobject(self):
         if self._cached is None:
@@ -214,7 +220,7 @@ class Listing:
         return (now - ds).days
 
     @property
-    def last_seen(self) -> datetime.datetime:
+    def last_seen(self) -> int:
         if not self.path_last_seen_listing().exists():
             return None
         
@@ -246,7 +252,7 @@ class Listing:
     
     @property
     def isRemoved(self) -> bool:
-        return not self.detailobject["property"]["visible"]
+        return not self.listingobject["visible"]
     
     @property
     def status(self) -> str: