Fix stuck Celery tasks and add purge all tasks functionality

crawler/frontend/src/components/TaskIndicator.tsx

@@ -1,12 +1,12 @@
 import { getUser } from '@/auth/authService';
 import { POLLING_INTERVALS } from '@/constants';
-import { fetchTaskStatus, cancelTask } from '@/services';
+import { fetchTaskStatus, cancelTask, clearAllTasks } from '@/services';
 import { TaskStatus, type TaskResult } from '@/types';
 import type { User } from 'oidc-client-ts';
 import { useEffect, useState } from 'react';
 import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from './ui/tooltip';
 import { Button } from './ui/button';
-import { Loader2, CheckCircle2, XCircle, X } from 'lucide-react';
+import { Loader2, CheckCircle2, XCircle, X, Trash2 } from 'lucide-react';
 
 interface TaskIndicatorProps {
     taskID: string | null;
@@ -20,6 +20,7 @@ export function TaskIndicator({ taskID, onTaskCancelled }: TaskIndicatorProps) {
     const [total, setTotal] = useState<number | null>(null);
     const [taskStatus, setTaskStatus] = useState<TaskStatus | null>(null);
     const [isCancelling, setIsCancelling] = useState(false);
+    const [isClearing, setIsClearing] = useState(false);
 
     useEffect(() => {
         getUser().then(setUser);
@@ -104,6 +105,23 @@ export function TaskIndicator({ taskID, onTaskCancelled }: TaskIndicatorProps) {
         }
     };
 
+    const handleClearAll = async () => {
+        if (!user || isClearing) return;
+
+        setIsClearing(true);
+        try {
+            const result = await clearAllTasks(user);
+            if (result.success) {
+                setTaskStatus(null);
+                onTaskCancelled?.();
+            }
+        } catch {
+            // Ignore clear errors
+        } finally {
+            setIsClearing(false);
+        }
+    };
+
     if (!taskID || !taskStatus) {
         return null;
     }
@@ -195,6 +213,22 @@ export function TaskIndicator({ taskID, onTaskCancelled }: TaskIndicatorProps) {
                         </TooltipContent>
                     </Tooltip>
                 )}
+                <Tooltip>
+                    <TooltipTrigger asChild>
+                        <Button
+                            variant="ghost"
+                            size="icon"
+                            onClick={handleClearAll}
+                            disabled={isClearing}
+                            className="h-6 w-6 text-muted-foreground hover:text-destructive"
+                        >
+                            <Trash2 className="h-3 w-3" />
+                        </Button>
+                    </TooltipTrigger>
+                    <TooltipContent side="bottom">
+                        <p>Clear all tasks</p>
+                    </TooltipContent>
+                </Tooltip>
             </div>
         </TooltipProvider>
     );

crawler/frontend/src/constants/colorSchemes.ts

@@ -0,0 +1,83 @@
+// Color schemes for map visualization
+// Different color schemes for different metrics to improve clarity
+
+import { Metric } from "@/components/Parameters";
+
+// For metrics where LOW is GOOD (price, price per sqm): Green → Yellow → Red
+export const LOW_IS_GOOD_COLOR_STOPS: [number, string][] = [
+  [0, 'rgba(34, 197, 94, 0.7)'],   // Green - good deal
+  [25, 'rgba(132, 204, 22, 0.7)'], // Lime
+  [50, 'rgba(250, 204, 21, 0.7)'], // Yellow - neutral
+  [75, 'rgba(249, 115, 22, 0.7)'], // Orange
+  [100, 'rgba(239, 68, 68, 0.7)'], // Red - expensive
+];
+
+// For metrics where HIGH is GOOD (size, rooms): Red → Yellow → Green
+export const HIGH_IS_GOOD_COLOR_STOPS: [number, string][] = [
+  [0, 'rgba(239, 68, 68, 0.7)'],   // Red - small
+  [25, 'rgba(249, 115, 22, 0.7)'], // Orange
+  [50, 'rgba(250, 204, 21, 0.7)'], // Yellow - medium
+  [75, 'rgba(132, 204, 22, 0.7)'], // Lime
+  [100, 'rgba(34, 197, 94, 0.7)'], // Green - large
+];
+
+// Legacy color stops (for backwards compatibility)
+export const LEGACY_COLOR_STOPS: [number, string][] = [
+  [0, 'rgba(0,185,243,0)'],
+  [25, 'rgba(0,185,243,0.24)'],
+  [60, 'rgba(255,223,0,0.3)'],
+  [100, 'rgba(255,105,0,0.3)'],
+];
+
+// Get the appropriate color scheme based on metric type
+export function getColorSchemeForMetric(metric: Metric | string): [number, string][] {
+  switch (metric) {
+    case Metric.qmprice:
+    case Metric.price:
+    case 'qmprice':
+    case 'total_price':
+      // Lower price is better → Green for low, Red for high
+      return LOW_IS_GOOD_COLOR_STOPS;
+
+    case Metric.qm:
+    case Metric.rooms:
+    case 'qm':
+    case 'rooms':
+      // Higher value is better → Green for high, Red for low
+      return HIGH_IS_GOOD_COLOR_STOPS;
+
+    default:
+      return LOW_IS_GOOD_COLOR_STOPS;
+  }
+}
+
+// Get interpretation text for legend
+export function getMetricInterpretation(metric: Metric | string): { low: string; high: string; name: string } {
+  switch (metric) {
+    case Metric.qmprice:
+    case 'qmprice':
+      return { low: 'Good deal', high: 'Expensive', name: 'Price per m²' };
+
+    case Metric.price:
+    case 'total_price':
+      return { low: 'Good deal', high: 'Expensive', name: 'Total Price' };
+
+    case Metric.qm:
+    case 'qm':
+      return { low: 'Small', high: 'Large', name: 'Size (m²)' };
+
+    case Metric.rooms:
+    case 'rooms':
+      return { low: 'Few rooms', high: 'Many rooms', name: 'Bedrooms' };
+
+    default:
+      return { low: 'Low', high: 'High', name: 'Value' };
+  }
+}
+
+// Color scheme names for display
+export const COLOR_SCHEME_NAMES = {
+  LOW_IS_GOOD: 'Green → Red (low is good)',
+  HIGH_IS_GOOD: 'Red → Green (high is good)',
+  LEGACY: 'Classic (blue → orange)',
+} as const;

crawler/frontend/src/constants/index.ts

@@ -0,0 +1,63 @@
+// Application constants and configuration
+
+// Re-export color schemes
+export * from './colorSchemes';
+
+// API endpoints
+export const API_ENDPOINTS = {
+  LISTING_GEOJSON: '/api/listing_geojson',
+  LISTING_GEOJSON_STREAM: '/api/listing_geojson/stream',
+  REFRESH_LISTINGS: '/api/refresh_listings',
+  TASK_STATUS: '/api/task_status',
+  TASKS_FOR_USER: '/api/tasks_for_user',
+  CANCEL_TASK: '/api/cancel_task',
+  CLEAR_ALL_TASKS: '/api/clear_all_tasks',
+  GET_DISTRICTS: '/api/get_districts',
+} as const;
+
+// Map configuration
+export const MAP_CONFIG = {
+  MAPBOX_TOKEN: import.meta.env.VITE_MAPBOX_TOKEN || 'pk.eyJ1IjoiZGktdG8iLCJhIjoiY2o0bnBoYXcxMW1mNzJ3bDhmc2xiNWttaiJ9.ZccatVk_4shzoAsEUXXecA',
+  DEFAULT_CENTER: [13.38032, 49.994210] as [number, number],
+  DEFAULT_ZOOM: 5,
+  STYLE: 'mapbox://styles/mapbox/light-v9',
+} as const;
+
+// Heatmap configuration
+export const HEATMAP_CONFIG = {
+  INTENSITY: 9,
+  SPREAD: 0.05,
+  CELL_DENSITY: 0.5, // Smaller value = bigger hexagons
+  SEARCH_BUFFER: 0.001, // ~100m for click detection
+} as const;
+
+// Percentile configuration for data visualization
+export const PERCENTILE_CONFIG = {
+  MIN_BOUND: 0.05, // 5th percentile for color scale minimum
+  MAX_BOUND: 0.95, // 95th percentile for color scale maximum
+  BOUNDS_CLIP_MIN: 0.01, // 1st percentile for bounding box
+  BOUNDS_CLIP_MAX: 0.99, // 99th percentile for bounding box
+} as const;
+
+// Heatmap color gradient stops
+export const HEATMAP_COLOR_STOPS: [number, string][] = [
+  [0, 'rgba(0,185,243,0)'],
+  [25, 'rgba(0,185,243,0.24)'],
+  [60, 'rgba(255,223,0,0.3)'],
+  [100, 'rgba(255,105,0,0.3)'],
+];
+
+// Default form values
+export const DEFAULT_FORM_VALUES = {
+  min_bedrooms: 1,
+  max_bedrooms: 3,
+  max_price: 3000,
+  min_price: 2000,
+  min_sqm: 50,
+  last_seen_days: 28,
+} as const;
+
+// Polling intervals
+export const POLLING_INTERVALS = {
+  TASK_STATUS_MS: 5000, // 5 seconds
+} as const;

crawler/frontend/src/services/index.ts

@@ -0,0 +1,6 @@
+// Re-export all services
+export { apiRequest } from './apiClient';
+export { fetchListingGeoJSON, refreshListings } from './listingService';
+export { streamListingGeoJSON, type StreamingProgress } from './streamingService';
+export { fetchTasksForUser, fetchTaskStatus, cancelTask, clearAllTasks, type CancelTaskResponse, type ClearAllTasksResponse } from './taskService';
+export { checkBackendHealth, type HealthStatus, type HealthCheckResult } from './healthService';

crawler/frontend/src/services/taskService.ts

@@ -10,6 +10,12 @@ export interface CancelTaskResponse {
   message: string;
 }
 
+export interface ClearAllTasksResponse {
+  success: boolean;
+  count: number;
+  message: string;
+}
+
 /**
  * Fetch all active tasks for the current user
  */
@@ -41,3 +47,12 @@ export async function cancelTask(
     params: { task_id: taskId },
   });
 }
+
+/**
+ * Clear all tasks for the current user
+ */
+export async function clearAllTasks(user: User): Promise<ClearAllTasksResponse> {
+  return apiRequest<ClearAllTasksResponse>(user, API_ENDPOINTS.CLEAR_ALL_TASKS, {
+    method: 'POST',
+  });
+}