add filter for last seen days

crawler/main.py

@@ -82,6 +82,12 @@ def listing_filter_options(func):
         default=None,
         type=click.DateTime(),
     )
+    @click.option(
+        "--last-seen-days",
+        help="Last seen (days). If set, only listings that were seen in the last N days will be included.",
+        default=14,
+        type=int,
+    )
     @wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -121,6 +127,7 @@ def dump_listings(
     type: str,
     furnish_types: list[str],
     available_from: datetime | None,
+    last_seen_days: int,
 ):
     data_dir: str = ctx.obj["data_dir"]
     query_parameters = QueryParameters(
@@ -132,6 +139,7 @@ def dump_listings(
         max_price=max_price,
         furnish_types=[FurnishType[furnish_type] for furnish_type in furnish_types],
         let_date_available_from=available_from,
+        last_seen_days=last_seen_days,
     )
     click.echo(
         f"Running dump_listings for districts {district}, data dir {data_dir} and parameters: "
@@ -279,6 +287,7 @@ def export_immoweb(
     type: str,
     furnish_types: list[str],
     available_from: datetime | None,
+    last_seen_days: int,
 ):
     query_parameters = QueryParameters(
         listing_type=ListingType[type],
@@ -289,6 +298,7 @@ def export_immoweb(
         max_price=max_price,
         furnish_types=[FurnishType[furnish_type] for furnish_type in furnish_types],
         let_date_available_from=available_from,
+        last_seen_days=last_seen_days,
     )
     click.echo(
         f"Exporting data to {output_file} that matches the query parameters: {query_parameters}"

crawler/rec/query.py

@@ -35,7 +35,7 @@ class QueryParameters:
     # hence we apply them after fetching
     # available from; council tax
     let_date_available_from: datetime | None = None
-    last_seen_days: int = 14
+    last_seen_days: int | None = None
 
 
 async def filter_listings(
@@ -57,7 +57,10 @@ async def filter_listings(
             or listing.price > query_parameters.max_price
         ):
             continue
-        if listing.last_seen > query_parameters.last_seen_days:
+        if (
+            query_parameters.last_seen_days is not None
+            and listing.last_seen > query_parameters.last_seen_days
+        ):
             continue
         if (
             listing.letDateAvailable is not None