Add real-time WebSocket task progress with multi-job drawer
Replace 5s HTTP polling with WebSocket-based real-time updates for task progress. Celery workers publish progress to Redis pub/sub channels; a FastAPI WebSocket endpoint subscribes and forwards to the browser. Polling is kept as a 30s fallback when WebSocket is unavailable. The task progress drawer now supports multiple concurrent jobs with a tab bar for switching between scrape and POI distance tasks. Backend: - Add services/task_progress_publisher.py (Redis pub/sub bridge) - Add api/ws_routes.py (WebSocket endpoint with JWT auth) - Publish progress from listing_tasks and poi_tasks - Publish REVOKED via pub/sub on cancel/clear to fix stuck UI Frontend: - Add useTaskWebSocket hook with reconnection and keepalive - Add TaskState and WS message types - TaskIndicator: WS-driven updates with polling fallback - TaskProgressDrawer: multi-job tabs, POI phase timeline - Guard against WS overwriting local cancel state
This commit is contained in:
parent
73d19e29d5
commit
8559c4b461
11 changed files with 774 additions and 72 deletions
|
|
@ -19,6 +19,7 @@ from database import engine
|
|||
from services.query_splitter import QuerySplitter, SubQuery
|
||||
from utils.redis_lock import redis_lock
|
||||
from services.listing_cache import invalidate_cache
|
||||
from services.task_progress_publisher import publish_task_progress
|
||||
|
||||
logger = logging.getLogger("uvicorn.error")
|
||||
|
||||
|
|
@ -86,6 +87,8 @@ def _update_task_state(task: Task, state: str, meta: dict[str, Any]) -> None:
|
|||
if _active_log_buffer is not None:
|
||||
meta["logs"] = list(_active_log_buffer)
|
||||
task.update_state(state=state, meta=meta)
|
||||
if hasattr(task, 'request') and task.request.id:
|
||||
publish_task_progress(task.request.id, state, meta)
|
||||
|
||||
|
||||
async def _fetch_subquery(
|
||||
|
|
@ -266,7 +269,9 @@ def dump_listings_task(self: Task, parameters_json: str) -> dict[str, Any]:
|
|||
if not acquired:
|
||||
msg = "Another scrape job is already running, skipping this execution"
|
||||
celery_logger.warning(msg)
|
||||
self.update_state(state="SKIPPED", meta={"reason": "Another scrape job is running"})
|
||||
meta = {"reason": "Another scrape job is running"}
|
||||
self.update_state(state="SKIPPED", meta=meta)
|
||||
publish_task_progress(self.request.id, "SKIPPED", meta)
|
||||
return {"status": "skipped", "reason": "another_job_running"}
|
||||
|
||||
celery_logger.info(f"Acquired lock: {SCRAPE_LOCK_NAME}")
|
||||
|
|
@ -275,8 +280,11 @@ def dump_listings_task(self: Task, parameters_json: str) -> dict[str, Any]:
|
|||
celery_logger.info(f"Starting scrape with parameters: {parsed_parameters}")
|
||||
|
||||
self.update_state(state="Starting...", meta={"phase": PHASE_SPLITTING, "progress": 0})
|
||||
publish_task_progress(self.request.id, "Starting...", {"phase": PHASE_SPLITTING, "progress": 0})
|
||||
asyncio.run(dump_listings_full(task=self, parameters=parsed_parameters))
|
||||
return {"phase": PHASE_COMPLETED, "progress": 1}
|
||||
result = {"phase": PHASE_COMPLETED, "progress": 1}
|
||||
publish_task_progress(self.request.id, "SUCCESS", result)
|
||||
return result
|
||||
|
||||
|
||||
async def async_dump_listings_task(parameters_json: str) -> dict[str, Any]:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue