add worker api to refresh data in the background

crawler/frontend/src/components/ActiveQuery.tsx

@@ -0,0 +1,138 @@
+import { getUser } from '@/auth/authService';
+import type { User } from 'oidc-client-ts';
+import React, { useEffect, useState } from 'react';
+import { HoverCard, HoverCardContent, HoverCardTrigger } from './ui/hover-card';
+import { Progress } from './ui/progress';
+
+interface ModalProps {
+    taskID: string | null;
+}
+
+const fetchTaskStatus = async (user: User, taskID: string) => {
+    const accessToken = user?.access_token;
+    const response = await fetch(`/api/task_status?task_id=${taskID}`, {
+        method: 'GET',
+        headers: {
+            'Authorization': `Bearer ${accessToken}`, // Pass the token
+            'Content-Type': 'application/json',
+        },
+    });
+
+    if (!response.ok) {
+        throw new Error(`Failed to fetch task status: ${response.status}`);
+    }
+
+    const data =
+        await response.json();
+    return data;
+};
+
+enum TaskStatus {
+    QUEUED = 'queued',
+    PROCESSING = 'processing',
+    COMPLETED = 'completed',
+    FAILED = 'failed',
+}
+
+const taskStatusToProgress = (taskStatus: TaskStatus): number => {
+    switch (taskStatus) {
+        case TaskStatus.QUEUED:
+            return 0.33; // Queued status
+        case TaskStatus.PROCESSING:
+            return 0.66; // Processing status
+        case TaskStatus.COMPLETED:
+            return 1.0; // Completed status
+        default:
+            throw new Error('Unknown task status: ' + status);
+    }
+}
+
+const getTaskStatus = (status: string): TaskStatus => {
+    switch (status.toLowerCase()) {
+        case 'queued':
+            return TaskStatus.QUEUED;
+        case 'processing':
+            return TaskStatus.PROCESSING;
+        case 'completed':
+            return TaskStatus.COMPLETED;
+        case 'failed':
+            return TaskStatus.FAILED;
+        default:
+            throw new Error('Unknown task status: ' + status);
+    }
+};
+
+const ActiveQuery: React.FC<ModalProps> = ({
+    taskID
+}) => {
+    const [user, setUser] = useState<User | null>(null);
+    useEffect(() => {
+        getUser().then(setUser);
+    }, []);
+
+    const [progressPercentage, setProgressPercentage] = useState<number>(0);
+    const [taskStatus, setTaskStatus] = useState<TaskStatus | null>(TaskStatus.QUEUED);
+    const [lastUpdateTime, setLastUpdateTime] = useState<Date>(new Date());
+
+    // fetch status periodically
+    // maybe move to ws one day
+    useEffect(() => {
+        const interval = setInterval
+            (async () => {
+                if (!user || !taskID) {
+                    return;
+                }
+                let data = null
+                try {
+                    data = await fetchTaskStatus(user, taskID);
+                } catch (error: any) {
+                    clearInterval(interval);
+                    setTaskStatus(TaskStatus.FAILED)
+                    alert(error)
+                }
+                if (!data) {
+                    clearInterval(interval);
+                    return;
+                }
+                setLastUpdateTime(new Date());
+                const taskStatus = getTaskStatus(data.status);
+                if (taskStatus === TaskStatus.FAILED) {
+                    clearInterval(interval);
+                    throw new Error('Task failed');
+                }
+                setTaskStatus(taskStatus);
+                const progress = taskStatusToProgress(taskStatus);
+                setProgressPercentage(progress * 100);
+                if (taskStatus === TaskStatus.COMPLETED) {
+                    clearInterval(interval);
+                    return;
+                }
+            }, 5000); // every 5 seconds
+        return () => clearInterval(interval);
+    }, [taskID]);
+
+    if (!taskID) {
+        return null;
+    }
+
+    return (
+        <>
+            <div>
+                <HoverCard>
+                    <HoverCardTrigger>
+                        {taskStatus && <p>Task status: {taskStatus} </p>}
+                        <Progress value={progressPercentage} />
+                    </HoverCardTrigger>
+                    <HoverCardContent>
+                        Task ID: {taskID}
+                        <br />
+                        Last updated: {lastUpdateTime.toLocaleString()}
+                    </HoverCardContent>
+                </HoverCard>
+            </div>
+
+        </>
+    )
+};
+
+export default ActiveQuery;

crawler/frontend/src/components/Map.tsx

@@ -1,4 +1,4 @@
-// // @ts-nocheck
+// @ts-nocheck
 import crossfilter from "crossfilter2";
 import * as d3 from "d3";
 import mapboxgl from "mapbox-gl";
@@ -195,7 +195,7 @@ export function Map(
             .call(xAxis);
     }
 
-    function openListingsDialog(longtitude, latitude) {
+    function openListingsDialog(longtitude: number, latitude: number) {
         const searchBuffer = 0.001 // ~100m
         const properties = heatmap._tree.search({
             minX: longtitude - searchBuffer,

crawler/frontend/src/components/Parameters.tsx

@@ -1,5 +1,6 @@
 import { zodResolver } from "@hookform/resolvers/zod";
 import { DialogTitle } from "@radix-ui/react-dialog";
+import { useState } from "react";
 import { useForm } from "react-hook-form";
 import { z } from "zod";
 import { Button } from "./ui/button";
@@ -34,13 +35,13 @@ export interface ParameterValues {
 export function Parameters(
     props: {
         isOpen: boolean,
-        onSubmit: (fromValues: ParameterValues) => void,
+        onSubmit: (action: 'fetch-data' | 'visualize', fromValues: ParameterValues) => void,
     }
 ) {
     const {
         register,
     } = useForm<ParameterValues>()
-    // const onSubmit: SubmitHandler<ParameterValues> = (data) => console.log(data)
+    const [action, setAction] = useState<'fetch-data' | 'visualize' | null>(null)
 
     const formSchema = z.object({
         metric: z.nativeEnum(Metric, { required_error: "Metric is required" }),
@@ -48,7 +49,7 @@ export function Parameters(
         min_bedrooms: z.number().min(1).max(10).optional(),
         max_bedrooms: z.number().min(1).max(10).optional(),
         max_price: z.number().optional(),
-        min_price: z.number().optional(),
+        min_price: z.number().min(0).optional(),
         min_sqm: z.number().optional(),
     })
     const form = useForm<z.infer<typeof formSchema>>({
@@ -58,7 +59,7 @@ export function Parameters(
             min_bedrooms: 1,
             max_bedrooms: 3,
             max_price: 3000,
-            min_price: 0,
+            min_price: 2000,
             min_sqm: 0,
         },
     })
@@ -67,11 +68,15 @@ export function Parameters(
         // Do something with the form values.
         // ✅ This will be type-safe and validated.
         console.log(values)
-        props.onSubmit(values)
+        if (action) {
+            props.onSubmit(action, values)
+        }
     }
 
 
+
     return <>
+        {/* <Dialog open={props.isOpen} > */}
         <Dialog >
             <DialogTrigger asChild>
                 <Button variant="outline">Open Parameters</Button>
@@ -195,7 +200,8 @@ export function Parameters(
                             )}
                         />
 
-                        <Button type="submit">Submit</Button>
+                        <Button type="submit" value={"visualize"} onClick={() => setAction("visualize")}>Visualize</Button>
+                        <Button type="submit" value={"fetch-data"} onClick={() => setAction("fetch-data")}>Fetch data</Button>
                     </form>
                 </Form>
             </DialogContent>

crawler/frontend/src/components/ui/badge.tsx

@@ -0,0 +1,46 @@
+import * as React from "react"
+import { Slot } from "@radix-ui/react-slot"
+import { cva, type VariantProps } from "class-variance-authority"
+
+import { cn } from "@/lib/utils"
+
+const badgeVariants = cva(
+  "inline-flex items-center justify-center rounded-md border px-2 py-0.5 text-xs font-medium w-fit whitespace-nowrap shrink-0 [&>svg]:size-3 gap-1 [&>svg]:pointer-events-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive transition-[color,box-shadow] overflow-hidden",
+  {
+    variants: {
+      variant: {
+        default:
+          "border-transparent bg-primary text-primary-foreground [a&]:hover:bg-primary/90",
+        secondary:
+          "border-transparent bg-secondary text-secondary-foreground [a&]:hover:bg-secondary/90",
+        destructive:
+          "border-transparent bg-destructive text-white [a&]:hover:bg-destructive/90 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/60",
+        outline:
+          "text-foreground [a&]:hover:bg-accent [a&]:hover:text-accent-foreground",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+    },
+  }
+)
+
+function Badge({
+  className,
+  variant,
+  asChild = false,
+  ...props
+}: React.ComponentProps<"span"> &
+  VariantProps<typeof badgeVariants> & { asChild?: boolean }) {
+  const Comp = asChild ? Slot : "span"
+
+  return (
+    <Comp
+      data-slot="badge"
+      className={cn(badgeVariants({ variant }), className)}
+      {...props}
+    />
+  )
+}
+
+export { Badge, badgeVariants }

crawler/frontend/src/components/ui/hover-card.tsx

@@ -0,0 +1,42 @@
+import * as React from "react"
+import * as HoverCardPrimitive from "@radix-ui/react-hover-card"
+
+import { cn } from "@/lib/utils"
+
+function HoverCard({
+  ...props
+}: React.ComponentProps<typeof HoverCardPrimitive.Root>) {
+  return <HoverCardPrimitive.Root data-slot="hover-card" {...props} />
+}
+
+function HoverCardTrigger({
+  ...props
+}: React.ComponentProps<typeof HoverCardPrimitive.Trigger>) {
+  return (
+    <HoverCardPrimitive.Trigger data-slot="hover-card-trigger" {...props} />
+  )
+}
+
+function HoverCardContent({
+  className,
+  align = "center",
+  sideOffset = 4,
+  ...props
+}: React.ComponentProps<typeof HoverCardPrimitive.Content>) {
+  return (
+    <HoverCardPrimitive.Portal data-slot="hover-card-portal">
+      <HoverCardPrimitive.Content
+        data-slot="hover-card-content"
+        align={align}
+        sideOffset={sideOffset}
+        className={cn(
+          "bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 w-64 origin-(--radix-hover-card-content-transform-origin) rounded-md border p-4 shadow-md outline-hidden",
+          className
+        )}
+        {...props}
+      />
+    </HoverCardPrimitive.Portal>
+  )
+}
+
+export { HoverCard, HoverCardTrigger, HoverCardContent }

crawler/frontend/src/components/ui/progress.tsx

@@ -0,0 +1,29 @@
+import * as React from "react"
+import * as ProgressPrimitive from "@radix-ui/react-progress"
+
+import { cn } from "@/lib/utils"
+
+function Progress({
+  className,
+  value,
+  ...props
+}: React.ComponentProps<typeof ProgressPrimitive.Root>) {
+  return (
+    <ProgressPrimitive.Root
+      data-slot="progress"
+      className={cn(
+        "bg-primary/20 relative h-2 w-full overflow-hidden rounded-full",
+        className
+      )}
+      {...props}
+    >
+      <ProgressPrimitive.Indicator
+        data-slot="progress-indicator"
+        className="bg-primary h-full w-full flex-1 transition-all"
+        style={{ transform: `translateX(-${100 - (value || 0)}%)` }}
+      />
+    </ProgressPrimitive.Root>
+  )
+}
+
+export { Progress }