Add real-time WebSocket task progress with multi-job drawer
Replace 5s HTTP polling with WebSocket-based real-time updates for task progress. Celery workers publish progress to Redis pub/sub channels; a FastAPI WebSocket endpoint subscribes and forwards to the browser. Polling is kept as a 30s fallback when WebSocket is unavailable. The task progress drawer now supports multiple concurrent jobs with a tab bar for switching between scrape and POI distance tasks. Backend: - Add services/task_progress_publisher.py (Redis pub/sub bridge) - Add api/ws_routes.py (WebSocket endpoint with JWT auth) - Publish progress from listing_tasks and poi_tasks - Publish REVOKED via pub/sub on cancel/clear to fix stuck UI Frontend: - Add useTaskWebSocket hook with reconnection and keepalive - Add TaskState and WS message types - TaskIndicator: WS-driven updates with polling fallback - TaskProgressDrawer: multi-job tabs, POI phase timeline - Guard against WS overwriting local cancel state
This commit is contained in:
parent
73d19e29d5
commit
8559c4b461
11 changed files with 774 additions and 72 deletions
|
|
@ -19,6 +19,7 @@ from database import engine
|
|||
from services.query_splitter import QuerySplitter, SubQuery
|
||||
from utils.redis_lock import redis_lock
|
||||
from services.listing_cache import invalidate_cache
|
||||
from services.task_progress_publisher import publish_task_progress
|
||||
|
||||
logger = logging.getLogger("uvicorn.error")
|
||||
|
||||
|
|
@ -86,6 +87,8 @@ def _update_task_state(task: Task, state: str, meta: dict[str, Any]) -> None:
|
|||
if _active_log_buffer is not None:
|
||||
meta["logs"] = list(_active_log_buffer)
|
||||
task.update_state(state=state, meta=meta)
|
||||
if hasattr(task, 'request') and task.request.id:
|
||||
publish_task_progress(task.request.id, state, meta)
|
||||
|
||||
|
||||
async def _fetch_subquery(
|
||||
|
|
@ -266,7 +269,9 @@ def dump_listings_task(self: Task, parameters_json: str) -> dict[str, Any]:
|
|||
if not acquired:
|
||||
msg = "Another scrape job is already running, skipping this execution"
|
||||
celery_logger.warning(msg)
|
||||
self.update_state(state="SKIPPED", meta={"reason": "Another scrape job is running"})
|
||||
meta = {"reason": "Another scrape job is running"}
|
||||
self.update_state(state="SKIPPED", meta=meta)
|
||||
publish_task_progress(self.request.id, "SKIPPED", meta)
|
||||
return {"status": "skipped", "reason": "another_job_running"}
|
||||
|
||||
celery_logger.info(f"Acquired lock: {SCRAPE_LOCK_NAME}")
|
||||
|
|
@ -275,8 +280,11 @@ def dump_listings_task(self: Task, parameters_json: str) -> dict[str, Any]:
|
|||
celery_logger.info(f"Starting scrape with parameters: {parsed_parameters}")
|
||||
|
||||
self.update_state(state="Starting...", meta={"phase": PHASE_SPLITTING, "progress": 0})
|
||||
publish_task_progress(self.request.id, "Starting...", {"phase": PHASE_SPLITTING, "progress": 0})
|
||||
asyncio.run(dump_listings_full(task=self, parameters=parsed_parameters))
|
||||
return {"phase": PHASE_COMPLETED, "progress": 1}
|
||||
result = {"phase": PHASE_COMPLETED, "progress": 1}
|
||||
publish_task_progress(self.request.id, "SUCCESS", result)
|
||||
return result
|
||||
|
||||
|
||||
async def async_dump_listings_task(parameters_json: str) -> dict[str, Any]:
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from models.listing import ListingType
|
|||
from repositories.listing_repository import ListingRepository
|
||||
from repositories.poi_repository import POIRepository
|
||||
from services.poi_distance_calculator import calculate_poi_distances
|
||||
from services.task_progress_publisher import publish_task_progress
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -55,6 +56,11 @@ def calculate_poi_distances_task(
|
|||
"progress": 0,
|
||||
"message": "Starting distance calculation...",
|
||||
})
|
||||
publish_task_progress(self.request.id, "PROGRESS", {
|
||||
"phase": "starting",
|
||||
"progress": 0,
|
||||
"message": "Starting distance calculation...",
|
||||
})
|
||||
|
||||
listing_repo = ListingRepository(engine)
|
||||
poi_repo = POIRepository(engine)
|
||||
|
|
@ -62,19 +68,23 @@ def calculate_poi_distances_task(
|
|||
poi = poi_repo.get_poi_by_id(poi_id)
|
||||
if poi is None:
|
||||
celery_logger.error(f"POI {poi_id} not found")
|
||||
return {"error": f"POI {poi_id} not found", "distances_computed": 0}
|
||||
error_result = {"error": f"POI {poi_id} not found", "distances_computed": 0}
|
||||
publish_task_progress(self.request.id, "FAILURE", error_result)
|
||||
return error_result
|
||||
|
||||
lt = ListingType(listing_type)
|
||||
|
||||
def on_progress(completed: int, total: int, message: str) -> None:
|
||||
progress = round(completed / total, 2) if total > 0 else 0
|
||||
self.update_state(state="PROGRESS", meta={
|
||||
meta = {
|
||||
"phase": "computing",
|
||||
"progress": progress,
|
||||
"processed": completed,
|
||||
"total": total,
|
||||
"message": message,
|
||||
})
|
||||
}
|
||||
self.update_state(state="PROGRESS", meta=meta)
|
||||
publish_task_progress(self.request.id, "PROGRESS", meta)
|
||||
|
||||
try:
|
||||
total = asyncio.run(
|
||||
|
|
@ -96,9 +106,12 @@ def calculate_poi_distances_task(
|
|||
|
||||
celery_logger.info(f"POI distance calculation complete: {total} distances computed")
|
||||
|
||||
return {
|
||||
result = {
|
||||
"phase": "completed",
|
||||
"progress": 1,
|
||||
"distances_computed": total,
|
||||
"message": f"Computed {total} distances for POI '{poi.name}'",
|
||||
}
|
||||
publish_task_progress(self.request.id, "SUCCESS", result)
|
||||
|
||||
return result
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue