use query params to filter out models; also make csv exporter work with models

crawler/main.py

@@ -6,10 +6,10 @@ import pathlib
 import click
 import importlib
 
+from models.listing import FurnishType, ListingType, QueryParameters
 from rec.districts import get_districts
 from data_access import Listing
 import csv_exporter
-from rec.query import ListingType, FurnishType, QueryParameters
 from rec.routing import API_KEY_ENVIRONMENT_VARIABLE, TravelMode
 from repositories.listing_repository import ListingRepository
 from ui_exporter import export_immoweb as export_immoweb_ui
@@ -230,17 +230,18 @@ def routing(
 
 
 @cli.command()
-@click.option(
-    "--columns",
-    "-C",
-    help="Columns to include in the CSV file",
-    type=click.Choice(
-        Listing.ALL_COLUMNS,
-        case_sensitive=False,
-    ),
-    multiple=True,
-    default=Listing.ALL_COLUMNS,
-)
+# @click.option(
+#     "--columns",
+#     "-C",
+#     help="Columns to include in the CSV file",
+#     type=click.Choice(
+#         # csv_exporter.get_columns_from_listings(),
+#         [1],
+#         case_sensitive=False,
+#     ),
+#     multiple=True,
+#     default=Listing.ALL_COLUMNS,
+# )
 @click.option(
     "--output-file",
     "-O",
@@ -258,7 +259,7 @@ def routing(
 def export_csv(
     ctx: click.core.Context,
     output_file: str,
-    columns: tuple[str],
+    # columns: tuple[str],
     district: list[str],
     min_bedrooms: int,
     max_bedrooms: int,
@@ -270,6 +271,7 @@ def export_csv(
     last_seen_days: int,
     min_sqm: int | None = None,
 ):
+    # use model
     data_dir = ctx.obj["data_dir"]
     query_parameters = QueryParameters(
         listing_type=ListingType[type],
@@ -287,13 +289,12 @@ def export_csv(
         f"Exporting data to {output_file} using {data_dir=} and query parameters: {query_parameters}"
     )
     output_file_path = pathlib.Path(output_file)
-    listing_paths = sorted(list(pathlib.Path(data_dir).glob("*/listing.json")))
-    listings = Listing.get_all_listings([path for path in listing_paths])
+    repository = ListingRepository(engine=engine)
     asyncio.run(
         csv_exporter.export_to_csv(
-            listings,
+            repository,
             output_file_path,
-            list(columns),
+            # list(columns),
             query_parameters=query_parameters,
         ),
     )
@@ -328,6 +329,7 @@ def export_immoweb(
     last_seen_days: int,
     min_sqm: int | None = None,
 ):
+    # use model
     query_parameters = QueryParameters(
         listing_type=ListingType[type],
         district_names=set(district),