add csv exporter command

crawler/data_access.py

@@ -12,9 +12,29 @@ class Listing:
     identifier: int
     _cached: Dict = None
     data_dir: pathlib.Path = pathlib.Path("data/rs/")
+    ALL_COLUMNS = [
+        "identifier",
+        "sqm_ocr",
+        "price",
+        "price_per_sqm",
+        "url",
+        "bedrooms",
+        "travel_time_fastest",
+        "travel_time_second",
+        "lease_left",
+        "service_charge",
+        "development",
+        "tenure_type",
+        "updated_days",
+        "status",
+        "last_seen",
+    ]
 
     @staticmethod
-    def get_all_listings(listing_paths: list[str]) -> List["Listing"]:
+    def get_all_listings(
+        listing_paths: list[str],
+        seen_in_the_last_n_days: int = 30,
+    ) -> List["Listing"]:
         identifiers = []
         for listing_path in listing_paths:
             with open(listing_path) as f:
@@ -24,7 +44,12 @@ class Listing:
             data_dir = pathlib.Path(listing_path)
             while str(d['identifier']) in str(data_dir.resolve().absolute()):
                 data_dir = data_dir.parent
-            identifiers.append(Listing(d["identifier"], data_dir=data_dir))
+            listing = Listing(d["identifier"], data_dir=data_dir)
+            if (
+                listing.last_seen is not None
+                and listing.last_seen < seen_in_the_last_n_days
+            ):
+                identifiers.append(listing)
 
         return identifiers