add command to dump existing listing from fs to db

crawler/main.py

@@ -11,8 +11,10 @@ from data_access import Listing
 import csv_exporter
 from rec.query import ListingType, FurnishType, QueryParameters
 from rec.routing import API_KEY_ENVIRONMENT_VARIABLE, TravelMode
+from repositories.listing_repositorty import ListingRepository
 from ui_exporter import export_immoweb as export_immoweb_ui
 from functools import wraps
+from database import engine
 
 
 dump_listings_module = importlib.import_module("1_dump_listings")
@@ -319,7 +321,7 @@ def export_csv(
 @listing_filter_options
 @click.pass_context
 def export_immoweb(
-    ctx,
+    ctx: click.core.Context,
     output_file: str,
     district: list[str],
     min_bedrooms: int,
@@ -350,5 +352,19 @@ def export_immoweb(
     asyncio.run(export_immoweb_ui(ctx, output_file, query_parameters))
 
 
+@cli.command()
+@click.pass_context
+def populate_db(
+    ctx: click.core.Context,
+):
+    data_dir = ctx.obj["data_dir"]
+    click.echo(f"Populating the database with data from {data_dir}")
+    repository = ListingRepository(engine=engine)
+    listings = Listing.get_all_listings(
+        [str(path) for path in pathlib.Path(data_dir).glob("*/listing.json")]
+    )
+    asyncio.run(repository.upsert_listings(listings))
+
+
 if __name__ == "__main__":
     cli()