migrate background tasks to celery

crawler/api/app.py

@@ -1,4 +1,6 @@
 import asyncio
+import dataclasses
+import json
 import logging
 import logging.config
 from pathlib import Path
@@ -15,6 +17,7 @@ from api.worker import (
     task_queue,
     task_results,
 )
+from dotenv import load_dotenv
 from fastapi import Depends, FastAPI, HTTPException, Query
 from api.auth import User
 from models.listing import QueryParameters
@@ -22,11 +25,14 @@ from repositories.listing_repository import ListingRepository
 from repositories.listing_repository import ListingRepository
 from database import engine
 from fastapi.middleware.cors import CORSMiddleware
+
+from tasks import listing_tasks
 from ui_exporter import export_immoweb
 from alembic import command
 from alembic.config import Config
 from contextlib import asynccontextmanager
 
+load_dotenv()
 logger = logging.getLogger("uvicorn")
 
 
@@ -43,9 +49,6 @@ logger = logging.getLogger("uvicorn")
 # app = FastAPI(lifespan=lifespan)
 app = FastAPI()
 
-# Start worker thread
-WorkerManager(DumpListingsWorker()).start()
-
 
 # Allow CORS (for React frontend)
 app.add_middleware(
@@ -81,19 +84,9 @@ async def refresh_listings(
     user: Annotated[User, Depends(get_current_user)],
     query_parameters: Annotated[QueryParameters, Query()],
 ) -> dict[str, str]:
-    # Submit processing task
-    task_id = str(uuid.uuid4())
-    task_results[task_id] = {"status": TaskStatus.QUEUED}
-    try:
-        task_queue.put_nowait(
-            (task_id, query_parameters),
-        )
-    except queue.Full:
-        raise HTTPException(
-            status_code=429,
-            detail="Already processing at maximum capacity. Please try again later",
-        )
-    return {"task_id": task_id}
+    # TODO: rate limit
+    task = listing_tasks.dump_listings_task.delay(query_parameters.json())
+    return {"task_id": task.id}
 
 
 @app.get("/api/task_status")
@@ -101,6 +94,9 @@ async def get_task_status(
     user: Annotated[User, Depends(get_current_user)],
     task_id: str,
 ) -> dict[str, str]:
-    if task_id not in task_results:
-        return {"status": "not_found"}
-    return task_results[task_id]
+    task_result = listing_tasks.dump_listings_task.AsyncResult(task_id)
+    return {
+        "task_id": task_id,
+        "status": task_result.status,
+        "result": json.dumps(task_result.result),
+    }