Add services layer, tests, streaming UI, and cleanup legacy code

crawler/frontend/src/components/Parameters.tsx

@@ -1,8 +1,6 @@
-import { getUser } from "@/auth/authService";
 import { zodResolver } from "@hookform/resolvers/zod";
 import { DialogTitle } from "@radix-ui/react-dialog";
-import type { User } from "oidc-client-ts";
-import { useEffect, useState } from "react";
+import { useState } from "react";
 import { useForm } from "react-hook-form";
 import { z } from "zod";
 import { Button } from "./ui/button";
@@ -24,6 +22,12 @@ export enum ListingType {
     BUY = 'BUY'
 }
 
+export enum FurnishType {
+    FURNISHED = 'furnished',
+    PART_FURNISHED = 'partFurnished',
+    UNFURNISHED = 'unfurnished',
+}
+
 
 export interface ParameterValues {
     metric: Metric
@@ -33,30 +37,15 @@ export interface ParameterValues {
     min_price?: number
     max_price?: number
     min_sqm?: number
+    max_sqm?: number
+    min_price_per_sqm?: number
+    max_price_per_sqm?: number
     last_seen_days?: number
     available_from?: Date
     district: string
+    furnish_types?: FurnishType[]
 }
 
-const fetchDistricts = async (user: User | null) => {
-    const accessToken = user?.access_token;
-
-    const response = await fetch('/api/get_districts',
-        {
-            method: 'GET',
-            headers: {
-                'Authorization': `Bearer ${accessToken}`, // Pass the token
-                'Content-Type': 'application/json',
-            },
-        }
-    );
-    if (!response.ok) {
-        throw new Error('Error: ' + response.status);
-    }
-    const data: Response = await response.json();
-    return data;
-};
-
 export function Parameters(
     props: {
         isOpen: boolean,
@@ -69,15 +58,6 @@ export function Parameters(
     } = useForm<ParameterValues>()
     const [action, setAction] = useState<'fetch-data' | 'visualize' | null>(null)
     const [availableFromRawInput, setAvailableFromRawInput] = useState("now");
-    const [_districts, setDistricts] = useState<string[]>([]);
-
-    useEffect(() => {
-        getUser().then(user => {
-            fetchDistricts(user).then(data => {
-                setDistricts(Object.keys(data));
-            })
-        })
-    }, []);
 
     const formSchema = z.object({
         metric: z.nativeEnum(Metric, { required_error: "Metric is required" }),
@@ -177,29 +157,6 @@ export function Parameters(
                                 </FormItem>
                             )}
                         />
-                        {/* <FormField # listings don't have district stored as metadata; so only useful in rightmove querying
-                            control={form.control}
-                            name="district"
-                            render={({ field }) => (
-                                <FormItem className="flex flex-row items-center gap-4">
-                                    <FormLabel>District</FormLabel>
-                                    <Select onValueChange={field.onChange} defaultValue={field.value}>
-                                        <FormControl>
-                                            <SelectTrigger className="w-[180px]">
-                                                <SelectValue placeholder="District" />
-                                            </SelectTrigger>
-                                        </FormControl>
-                                        <SelectContent {...register('district')} >
-                                            {districts.map((district, index) => (
-                                                <SelectItem key={index} value={district}>{district}
-                                                </SelectItem>
-                                            ))}
-                                        </SelectContent>
-                                    </Select>
-                                    <FormMessage />
-                                </FormItem>
-                            )}
-                        /> */}
                         <FormField
                             control={form.control}
                             name="min_sqm"

crawler/frontend/src/components/StatsBar.tsx

@@ -0,0 +1,128 @@
+import { BarChart3, MapPin, PoundSterling, Maximize2, List, Map as MapIcon } from 'lucide-react';
+import { Button } from './ui/button';
+import type { GeoJSONFeatureCollection, PropertyFeature } from '@/types';
+
+export type ViewMode = 'map' | 'list' | 'split';
+
+interface StatsBarProps {
+    listingData: GeoJSONFeatureCollection | null;
+    viewMode: ViewMode;
+    onViewModeChange: (mode: ViewMode) => void;
+}
+
+interface ListingStats {
+    count: number;
+    avgPrice: number;
+    avgPricePerSqm: number;
+    avgSize: number;
+}
+
+function calculateStats(data: GeoJSONFeatureCollection | null): ListingStats {
+    if (!data || data.features.length === 0) {
+        return { count: 0, avgPrice: 0, avgPricePerSqm: 0, avgSize: 0 };
+    }
+
+    const features = data.features;
+    const count = features.length;
+
+    const validPrices = features
+        .map((f: PropertyFeature) => f.properties.total_price)
+        .filter((p): p is number => typeof p === 'number' && p > 0);
+
+    const validPricesPerSqm = features
+        .map((f: PropertyFeature) => f.properties.qmprice)
+        .filter((p): p is number => typeof p === 'number' && p > 0);
+
+    const validSizes = features
+        .map((f: PropertyFeature) => f.properties.qm)
+        .filter((s): s is number => typeof s === 'number' && s > 0);
+
+    const avgPrice = validPrices.length > 0
+        ? validPrices.reduce((a, b) => a + b, 0) / validPrices.length
+        : 0;
+
+    const avgPricePerSqm = validPricesPerSqm.length > 0
+        ? validPricesPerSqm.reduce((a, b) => a + b, 0) / validPricesPerSqm.length
+        : 0;
+
+    const avgSize = validSizes.length > 0
+        ? validSizes.reduce((a, b) => a + b, 0) / validSizes.length
+        : 0;
+
+    return { count, avgPrice, avgPricePerSqm, avgSize };
+}
+
+function formatCurrency(value: number): string {
+    if (value >= 1000) {
+        return `£${(value / 1000).toFixed(1)}k`;
+    }
+    return `£${Math.round(value)}`;
+}
+
+export function StatsBar({ listingData, viewMode, onViewModeChange }: StatsBarProps) {
+    const stats = calculateStats(listingData);
+
+    return (
+        <div className="flex items-center justify-between px-4 py-2 bg-muted/50 border-t text-sm">
+            {/* Stats */}
+            <div className="flex items-center gap-4 text-muted-foreground">
+                <div className="flex items-center gap-1.5">
+                    <MapPin className="h-4 w-4" />
+                    <span className="font-medium text-foreground">{stats.count.toLocaleString()}</span>
+                    <span className="hidden sm:inline">listings</span>
+                </div>
+
+                {stats.avgPrice > 0 && (
+                    <>
+                        <div className="hidden md:flex items-center gap-1.5">
+                            <PoundSterling className="h-4 w-4" />
+                            <span>Avg: <span className="font-medium text-foreground">{formatCurrency(stats.avgPrice)}</span></span>
+                        </div>
+                        <div className="hidden lg:flex items-center gap-1.5">
+                            <BarChart3 className="h-4 w-4" />
+                            <span>Avg £/m²: <span className="font-medium text-foreground">{formatCurrency(stats.avgPricePerSqm)}</span></span>
+                        </div>
+                        <div className="hidden lg:flex items-center gap-1.5">
+                            <Maximize2 className="h-4 w-4" />
+                            <span>Avg: <span className="font-medium text-foreground">{Math.round(stats.avgSize)} m²</span></span>
+                        </div>
+                    </>
+                )}
+            </div>
+
+            {/* View Mode Toggle */}
+            <div className="flex items-center gap-1 bg-background rounded-md border p-0.5">
+                <Button
+                    variant={viewMode === 'map' ? 'secondary' : 'ghost'}
+                    size="sm"
+                    className="h-7 px-2"
+                    onClick={() => onViewModeChange('map')}
+                >
+                    <MapIcon className="h-4 w-4" />
+                    <span className="hidden sm:inline ml-1">Map</span>
+                </Button>
+                <Button
+                    variant={viewMode === 'list' ? 'secondary' : 'ghost'}
+                    size="sm"
+                    className="h-7 px-2"
+                    onClick={() => onViewModeChange('list')}
+                >
+                    <List className="h-4 w-4" />
+                    <span className="hidden sm:inline ml-1">List</span>
+                </Button>
+                <Button
+                    variant={viewMode === 'split' ? 'secondary' : 'ghost'}
+                    size="sm"
+                    className="h-7 px-2 hidden md:flex"
+                    onClick={() => onViewModeChange('split')}
+                >
+                    <div className="flex gap-0.5">
+                        <div className="w-2 h-4 bg-current rounded-sm opacity-60" />
+                        <div className="w-2 h-4 border border-current rounded-sm" />
+                    </div>
+                    <span className="hidden sm:inline ml-1">Split</span>
+                </Button>
+            </div>
+        </div>
+    );
+}

crawler/frontend/src/components/StreamingProgressBar.tsx

@@ -0,0 +1,47 @@
+import { Loader2 } from 'lucide-react';
+import type { StreamingProgress } from '@/services';
+
+interface StreamingProgressBarProps {
+  progress: StreamingProgress | null;
+  isLoading: boolean;
+}
+
+export function StreamingProgressBar({ progress, isLoading }: StreamingProgressBarProps) {
+  if (!isLoading) return null;
+
+  return (
+    <div className="absolute top-0 left-0 right-0 z-10 bg-background/95 backdrop-blur-sm border-b px-4 py-2">
+      <div className="flex items-center gap-3">
+        <Loader2 className="h-4 w-4 animate-spin text-primary" />
+        <div className="flex-1">
+          <div className="flex items-center justify-between text-sm">
+            <span className="font-medium">
+              {progress
+                ? `Loading listings...`
+                : 'Loading...'}
+            </span>
+            {progress && (
+              <span className="text-muted-foreground">
+                {progress.count.toLocaleString()}
+                {progress.total ? ` / ${progress.total.toLocaleString()}` : ''} loaded
+              </span>
+            )}
+          </div>
+          {progress && (
+            <div className="mt-1 h-1.5 w-full bg-primary/20 rounded-full overflow-hidden">
+              <div
+                className="h-full bg-primary transition-all duration-300 ease-out rounded-full"
+                style={{
+                  width: progress.total
+                    ? `${Math.min((progress.count / progress.total) * 100, 100)}%`
+                    : '100%',
+                  animation: progress.total ? undefined : 'pulse 1.5s ease-in-out infinite',
+                }}
+              />
+            </div>
+          )}
+        </div>
+      </div>
+    </div>
+  );
+}

crawler/frontend/src/components/ui/accordion.tsx

@@ -0,0 +1,56 @@
+"use client"
+
+import * as React from "react"
+import * as AccordionPrimitive from "@radix-ui/react-accordion"
+import { ChevronDown } from "lucide-react"
+import { cn } from "@/lib/utils"
+
+const Accordion = AccordionPrimitive.Root
+
+const AccordionItem = React.forwardRef<
+  React.ComponentRef<typeof AccordionPrimitive.Item>,
+  React.ComponentPropsWithoutRef<typeof AccordionPrimitive.Item>
+>(({ className, ...props }, ref) => (
+  <AccordionPrimitive.Item
+    ref={ref}
+    className={cn("border-b", className)}
+    {...props}
+  />
+))
+AccordionItem.displayName = "AccordionItem"
+
+const AccordionTrigger = React.forwardRef<
+  React.ComponentRef<typeof AccordionPrimitive.Trigger>,
+  React.ComponentPropsWithoutRef<typeof AccordionPrimitive.Trigger>
+>(({ className, children, ...props }, ref) => (
+  <AccordionPrimitive.Header className="flex">
+    <AccordionPrimitive.Trigger
+      ref={ref}
+      className={cn(
+        "flex flex-1 items-center justify-between py-4 text-sm font-medium transition-all hover:underline text-left [&[data-state=open]>svg]:rotate-180",
+        className
+      )}
+      {...props}
+    >
+      {children}
+      <ChevronDown className="h-4 w-4 shrink-0 text-muted-foreground transition-transform duration-200" />
+    </AccordionPrimitive.Trigger>
+  </AccordionPrimitive.Header>
+))
+AccordionTrigger.displayName = AccordionPrimitive.Trigger.displayName
+
+const AccordionContent = React.forwardRef<
+  React.ComponentRef<typeof AccordionPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof AccordionPrimitive.Content>
+>(({ className, children, ...props }, ref) => (
+  <AccordionPrimitive.Content
+    ref={ref}
+    className="overflow-hidden text-sm data-[state=closed]:animate-accordion-up data-[state=open]:animate-accordion-down"
+    {...props}
+  >
+    <div className={cn("pb-4 pt-0", className)}>{children}</div>
+  </AccordionPrimitive.Content>
+))
+AccordionContent.displayName = AccordionPrimitive.Content.displayName
+
+export { Accordion, AccordionItem, AccordionTrigger, AccordionContent }

crawler/frontend/src/components/ui/alert.tsx

@@ -1,66 +0,0 @@
-import * as React from "react"
-import { cva, type VariantProps } from "class-variance-authority"
-
-import { cn } from "@/lib/utils"
-
-const alertVariants = cva(
-  "relative w-full rounded-lg border px-4 py-3 text-sm grid has-[>svg]:grid-cols-[calc(var(--spacing)*4)_1fr] grid-cols-[0_1fr] has-[>svg]:gap-x-3 gap-y-0.5 items-start [&>svg]:size-4 [&>svg]:translate-y-0.5 [&>svg]:text-current",
-  {
-    variants: {
-      variant: {
-        default: "bg-card text-card-foreground",
-        destructive:
-          "text-destructive bg-card [&>svg]:text-current *:data-[slot=alert-description]:text-destructive/90",
-      },
-    },
-    defaultVariants: {
-      variant: "default",
-    },
-  }
-)
-
-function Alert({
-  className,
-  variant,
-  ...props
-}: React.ComponentProps<"div"> & VariantProps<typeof alertVariants>) {
-  return (
-    <div
-      data-slot="alert"
-      role="alert"
-      className={cn(alertVariants({ variant }), className)}
-      {...props}
-    />
-  )
-}
-
-function AlertTitle({ className, ...props }: React.ComponentProps<"div">) {
-  return (
-    <div
-      data-slot="alert-title"
-      className={cn(
-        "col-start-2 line-clamp-1 min-h-4 font-medium tracking-tight",
-        className
-      )}
-      {...props}
-    />
-  )
-}
-
-function AlertDescription({
-  className,
-  ...props
-}: React.ComponentProps<"div">) {
-  return (
-    <div
-      data-slot="alert-description"
-      className={cn(
-        "text-muted-foreground col-start-2 grid justify-items-start gap-1 text-sm [&_p]:leading-relaxed",
-        className
-      )}
-      {...props}
-    />
-  )
-}
-
-export { Alert, AlertTitle, AlertDescription }

crawler/frontend/src/components/ui/badge.tsx

@@ -1,46 +0,0 @@
-import * as React from "react"
-import { Slot } from "@radix-ui/react-slot"
-import { cva, type VariantProps } from "class-variance-authority"
-
-import { cn } from "@/lib/utils"
-
-const badgeVariants = cva(
-  "inline-flex items-center justify-center rounded-md border px-2 py-0.5 text-xs font-medium w-fit whitespace-nowrap shrink-0 [&>svg]:size-3 gap-1 [&>svg]:pointer-events-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive transition-[color,box-shadow] overflow-hidden",
-  {
-    variants: {
-      variant: {
-        default:
-          "border-transparent bg-primary text-primary-foreground [a&]:hover:bg-primary/90",
-        secondary:
-          "border-transparent bg-secondary text-secondary-foreground [a&]:hover:bg-secondary/90",
-        destructive:
-          "border-transparent bg-destructive text-white [a&]:hover:bg-destructive/90 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/60",
-        outline:
-          "text-foreground [a&]:hover:bg-accent [a&]:hover:text-accent-foreground",
-      },
-    },
-    defaultVariants: {
-      variant: "default",
-    },
-  }
-)
-
-function Badge({
-  className,
-  variant,
-  asChild = false,
-  ...props
-}: React.ComponentProps<"span"> &
-  VariantProps<typeof badgeVariants> & { asChild?: boolean }) {
-  const Comp = asChild ? Slot : "span"
-
-  return (
-    <Comp
-      data-slot="badge"
-      className={cn(badgeVariants({ variant }), className)}
-      {...props}
-    />
-  )
-}
-
-export { Badge, badgeVariants }

crawler/frontend/src/components/ui/breadcrumb.tsx

@@ -1,6 +1,6 @@
 import * as React from "react"
 import { Slot } from "@radix-ui/react-slot"
-import { ChevronRight, MoreHorizontal } from "lucide-react"
+import { ChevronRight } from "lucide-react"
 
 import { cn } from "@/lib/utils"
 
@@ -80,24 +80,6 @@ function BreadcrumbSeparator({
   )
 }
 
-function BreadcrumbEllipsis({
-  className,
-  ...props
-}: React.ComponentProps<"span">) {
-  return (
-    <span
-      data-slot="breadcrumb-ellipsis"
-      role="presentation"
-      aria-hidden="true"
-      className={cn("flex size-9 items-center justify-center", className)}
-      {...props}
-    >
-      <MoreHorizontal className="size-4" />
-      <span className="sr-only">More</span>
-    </span>
-  )
-}
-
 export {
   Breadcrumb,
   BreadcrumbList,
@@ -105,5 +87,4 @@ export {
   BreadcrumbLink,
   BreadcrumbPage,
   BreadcrumbSeparator,
-  BreadcrumbEllipsis,
 }

crawler/frontend/src/components/ui/checkbox.tsx

@@ -0,0 +1,29 @@
+"use client"
+
+import * as React from "react"
+import * as CheckboxPrimitive from "@radix-ui/react-checkbox"
+import { Check } from "lucide-react"
+import { cn } from "@/lib/utils"
+
+const Checkbox = React.forwardRef<
+  React.ComponentRef<typeof CheckboxPrimitive.Root>,
+  React.ComponentPropsWithoutRef<typeof CheckboxPrimitive.Root>
+>(({ className, ...props }, ref) => (
+  <CheckboxPrimitive.Root
+    ref={ref}
+    className={cn(
+      "peer h-4 w-4 shrink-0 rounded-sm border border-primary shadow focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-primary data-[state=checked]:text-primary-foreground",
+      className
+    )}
+    {...props}
+  >
+    <CheckboxPrimitive.Indicator
+      className={cn("flex items-center justify-center text-current")}
+    >
+      <Check className="h-4 w-4" />
+    </CheckboxPrimitive.Indicator>
+  </CheckboxPrimitive.Root>
+))
+Checkbox.displayName = CheckboxPrimitive.Root.displayName
+
+export { Checkbox }

crawler/frontend/src/components/ui/slider.tsx

@@ -0,0 +1,34 @@
+"use client"
+
+import * as React from "react"
+import * as SliderPrimitive from "@radix-ui/react-slider"
+import { cn } from "@/lib/utils"
+
+const Slider = React.forwardRef<
+  React.ComponentRef<typeof SliderPrimitive.Root>,
+  React.ComponentPropsWithoutRef<typeof SliderPrimitive.Root>
+>(({ className, ...props }, ref) => (
+  <SliderPrimitive.Root
+    ref={ref}
+    className={cn(
+      "relative flex w-full touch-none select-none items-center",
+      className
+    )}
+    {...props}
+  >
+    <SliderPrimitive.Track className="relative h-1.5 w-full grow overflow-hidden rounded-full bg-primary/20">
+      <SliderPrimitive.Range className="absolute h-full bg-primary" />
+    </SliderPrimitive.Track>
+    {props.defaultValue?.map((_, index) => (
+      <SliderPrimitive.Thumb
+        key={index}
+        className="block h-4 w-4 rounded-full border border-primary/50 bg-background shadow transition-colors focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50"
+      />
+    )) ?? (
+      <SliderPrimitive.Thumb className="block h-4 w-4 rounded-full border border-primary/50 bg-background shadow transition-colors focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50" />
+    )}
+  </SliderPrimitive.Root>
+))
+Slider.displayName = SliderPrimitive.Root.displayName
+
+export { Slider }