add worker api to refresh data in the background

crawler/api/app.py

@@ -1,7 +1,12 @@
+from pathlib import Path
+import queue
+from threading import Thread
 from typing import Annotated
+import uuid
 from api.auth import get_current_user
 from api.config import DEV_TIER_ORIGINS, PROD_TIER_ORIGINS
-from fastapi import Depends, FastAPI, Query
+from api.worker import TaskStatus, dump_listings_worker, task_queue, task_results
+from fastapi import Depends, FastAPI, HTTPException, Query
 from api.auth import User
 from models.listing import QueryParameters
 from repositories.listing_repository import ListingRepository
@@ -10,9 +15,11 @@ from database import engine
 from fastapi.middleware.cors import CORSMiddleware
 from ui_exporter import export_immoweb
 
-
 app = FastAPI()
 
+# Start worker thread
+Thread(target=dump_listings_worker, daemon=True).start()
+
 # Allow CORS (for React frontend)
 app.add_middleware(
     CORSMiddleware,
@@ -39,3 +46,33 @@ async def get_listing_geojson(
         repository, query_parameters=query_parameters, limit=None
     )
     return geojson_data
+
+
+@app.post("/api/refresh_listings")
+async def refresh_listings(
+    user: Annotated[User, Depends(get_current_user)],
+    query_parameters: Annotated[QueryParameters, Query()],
+) -> dict[str, str]:
+    # Submit processing task
+    task_id = str(uuid.uuid4())
+    task_results[task_id] = {"status": TaskStatus.QUEUED}
+    try:
+        task_queue.put_nowait(
+            (task_id, query_parameters),
+        )
+    except queue.Full:
+        raise HTTPException(
+            status_code=429,
+            detail="Already processing at maximum capacity. Please try again later",
+        )
+    return {"task_id": task_id}
+
+
+@app.get("/api/task_status")
+async def get_task_status(
+    user: Annotated[User, Depends(get_current_user)],
+    task_id: str,
+) -> dict[str, str]:
+    if task_id not in task_results:
+        return {"status": "not_found"}
+    return task_results[task_id]