add worker api to refresh data in the background

crawler/main.py

@@ -122,9 +122,11 @@ def cli(ctx, data_dir: str):
 
 @cli.command()
 @listing_filter_options
+@click.option("--full", is_flag=True)
 @click.pass_context
 def dump_listings(
     ctx: click.core.Context,
+    full: bool,
     district: list[str],
     min_bedrooms: int,
     max_bedrooms: int,
@@ -148,6 +150,9 @@ def dump_listings(
         let_date_available_from=available_from,
         last_seen_days=last_seen_days,
         min_sqm=min_sqm,
+        radius=0,
+        page_size=500,
+        max_days_since_added=14,
     )
     click.echo(
         f"Running dump_listings for districts {district}, data dir {data_dir} and parameters: "
@@ -155,9 +160,18 @@ def dump_listings(
     )
     data_dir_path = pathlib.Path(data_dir)
     repository = ListingRepository(engine=engine)
-    asyncio.run(
-        dump_listings_module.dump_listings(query_parameters, repository, data_dir_path)
-    )
+    if not full:  # only listings
+        asyncio.run(
+            dump_listings_module.dump_listings(
+                query_parameters, repository, data_dir_path
+            )
+        )
+    else:  # include images, floorplan detection etc.
+        asyncio.run(
+            dump_listings_module.dump_listings_full(
+                query_parameters, repository, data_dir_path
+            )
+        )
 
 
 @cli.command()