Remove 1000-result limit, add Redis caching and virtual scrolling
- Remove hard-coded limit=1000 default from listing_geojson and streaming endpoints, allowing all matching results to be returned - Add Redis caching service (db=2, 30min TTL) that caches query results as Redis Lists for fast re-queries with reduced DB load - Integrate cache into streaming endpoint: serve from cache on hit, populate cache on miss during DB streaming - Invalidate cache after scrape completes (both success and no-new-listings) - Replace ScrollArea with react-virtuoso in ListView for virtual scrolling, keeping only ~20-30 DOM nodes regardless of list size - Handle metadata streaming message to show "0 / N" progress from start - Throttle frontend state updates with requestAnimationFrame to prevent UI jank from rapid re-renders during cached response streaming
This commit is contained in:
Viktor Barzin
Viktor Barzin
parent
c4b11ccfe9
commit
5514fa6381
8 changed files with 695 additions and 78 deletions
|
|
@ -67,16 +67,32 @@ function App() {
|
|||
setStreamingProgress({ count: 0 });
|
||||
setListingData(null);
|
||||
|
||||
let updateScheduled = false;
|
||||
|
||||
const flushUpdate = () => {
|
||||
updateScheduled = false;
|
||||
setListingData({
|
||||
type: 'FeatureCollection',
|
||||
features: [...accumulatedFeaturesRef.current]
|
||||
});
|
||||
};
|
||||
|
||||
const scheduleUpdate = () => {
|
||||
if (!updateScheduled) {
|
||||
updateScheduled = true;
|
||||
requestAnimationFrame(flushUpdate);
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
for await (const batch of streamListingGeoJSON(user, parameters, (progress) => {
|
||||
setStreamingProgress(progress);
|
||||
})) {
|
||||
accumulatedFeaturesRef.current.push(...batch);
|
||||
setListingData({
|
||||
type: 'FeatureCollection',
|
||||
features: [...accumulatedFeaturesRef.current]
|
||||
});
|
||||
scheduleUpdate();
|
||||
}
|
||||
// Final flush to ensure all data is rendered
|
||||
flushUpdate();
|
||||
} catch (error) {
|
||||
if (error instanceof Error) {
|
||||
setSubmitError(error.message);
|
||||
|
|
|
|||
151
crawler/frontend/src/components/ListView.tsx
Normal file
151
crawler/frontend/src/components/ListView.tsx
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
import { useState, useMemo, useCallback } from 'react';
|
||||
import { ArrowUpDown, ArrowUp, ArrowDown } from 'lucide-react';
|
||||
import { Virtuoso } from 'react-virtuoso';
|
||||
import { Button } from './ui/button';
|
||||
import { PropertyCard } from './PropertyCard';
|
||||
import type { GeoJSONFeatureCollection, PropertyFeature, PropertyProperties } from '@/types';
|
||||
|
||||
type SortField = 'total_price' | 'qmprice' | 'qm' | 'rooms' | 'last_seen';
|
||||
type SortOrder = 'asc' | 'desc';
|
||||
|
||||
interface ListViewProps {
|
||||
listingData: GeoJSONFeatureCollection;
|
||||
onPropertyClick?: (property: PropertyProperties, coordinates: [number, number]) => void;
|
||||
highlightedPropertyUrl?: string | null;
|
||||
}
|
||||
|
||||
interface SortConfig {
|
||||
field: SortField;
|
||||
order: SortOrder;
|
||||
}
|
||||
|
||||
const SORT_OPTIONS: { field: SortField; label: string }[] = [
|
||||
{ field: 'total_price', label: 'Price' },
|
||||
{ field: 'qmprice', label: '£/m²' },
|
||||
{ field: 'qm', label: 'Size' },
|
||||
{ field: 'rooms', label: 'Beds' },
|
||||
{ field: 'last_seen', label: 'Last Seen' },
|
||||
];
|
||||
|
||||
export function ListView({ listingData, onPropertyClick, highlightedPropertyUrl }: ListViewProps) {
|
||||
const [sortConfig, setSortConfig] = useState<SortConfig>({ field: 'qmprice', order: 'asc' });
|
||||
|
||||
// Calculate average price per sqm for "good deal" indicator
|
||||
const avgPricePerSqm = useMemo(() => {
|
||||
const validPrices = listingData.features
|
||||
.map((f) => f.properties.qmprice)
|
||||
.filter((p): p is number => typeof p === 'number' && p > 0);
|
||||
return validPrices.length > 0
|
||||
? validPrices.reduce((a, b) => a + b, 0) / validPrices.length
|
||||
: 0;
|
||||
}, [listingData]);
|
||||
|
||||
// Sort features
|
||||
const sortedFeatures = useMemo(() => {
|
||||
const features = [...listingData.features];
|
||||
|
||||
features.sort((a, b) => {
|
||||
let aValue: number | string;
|
||||
let bValue: number | string;
|
||||
|
||||
switch (sortConfig.field) {
|
||||
case 'total_price':
|
||||
aValue = a.properties.total_price || 0;
|
||||
bValue = b.properties.total_price || 0;
|
||||
break;
|
||||
case 'qmprice':
|
||||
aValue = a.properties.qmprice || 0;
|
||||
bValue = b.properties.qmprice || 0;
|
||||
break;
|
||||
case 'qm':
|
||||
aValue = a.properties.qm || 0;
|
||||
bValue = b.properties.qm || 0;
|
||||
break;
|
||||
case 'rooms':
|
||||
aValue = a.properties.rooms || 0;
|
||||
bValue = b.properties.rooms || 0;
|
||||
break;
|
||||
case 'last_seen':
|
||||
aValue = new Date(a.properties.last_seen).getTime();
|
||||
bValue = new Date(b.properties.last_seen).getTime();
|
||||
break;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (typeof aValue === 'number' && typeof bValue === 'number') {
|
||||
return sortConfig.order === 'asc' ? aValue - bValue : bValue - aValue;
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
|
||||
return features;
|
||||
}, [listingData.features, sortConfig]);
|
||||
|
||||
const handleSort = (field: SortField) => {
|
||||
setSortConfig((prev) => ({
|
||||
field,
|
||||
order: prev.field === field && prev.order === 'asc' ? 'desc' : 'asc',
|
||||
}));
|
||||
};
|
||||
|
||||
const handlePropertyClick = useCallback((feature: PropertyFeature) => {
|
||||
if (onPropertyClick) {
|
||||
onPropertyClick(feature.properties, feature.geometry.coordinates);
|
||||
}
|
||||
}, [onPropertyClick]);
|
||||
|
||||
const SortIcon = ({ field }: { field: SortField }) => {
|
||||
if (sortConfig.field !== field) {
|
||||
return <ArrowUpDown className="h-3.5 w-3.5" />;
|
||||
}
|
||||
return sortConfig.order === 'asc'
|
||||
? <ArrowUp className="h-3.5 w-3.5" />
|
||||
: <ArrowDown className="h-3.5 w-3.5" />;
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="h-full flex flex-col bg-background">
|
||||
{/* Sort controls */}
|
||||
<div className="flex items-center gap-1 p-2 border-b overflow-x-auto">
|
||||
<span className="text-xs text-muted-foreground mr-1 shrink-0">Sort:</span>
|
||||
{SORT_OPTIONS.map((option) => (
|
||||
<Button
|
||||
key={option.field}
|
||||
variant={sortConfig.field === option.field ? 'secondary' : 'ghost'}
|
||||
size="sm"
|
||||
className="h-7 px-2 text-xs shrink-0"
|
||||
onClick={() => handleSort(option.field)}
|
||||
>
|
||||
{option.label}
|
||||
<SortIcon field={option.field} />
|
||||
</Button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* Listing count */}
|
||||
<div className="px-3 py-2 text-sm text-muted-foreground border-b">
|
||||
Showing {sortedFeatures.length.toLocaleString()} properties
|
||||
</div>
|
||||
|
||||
{/* Property list */}
|
||||
<Virtuoso
|
||||
className="flex-1"
|
||||
data={sortedFeatures}
|
||||
overscan={200}
|
||||
itemContent={(_index, feature) => (
|
||||
<div className="px-3 pb-2 first:pt-3">
|
||||
<PropertyCard
|
||||
key={feature.properties.url}
|
||||
property={feature.properties}
|
||||
variant="compact"
|
||||
avgPricePerSqm={avgPricePerSqm}
|
||||
isHighlighted={feature.properties.url === highlightedPropertyUrl}
|
||||
onClick={() => handlePropertyClick(feature)}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
137
crawler/frontend/src/services/streamingService.ts
Normal file
137
crawler/frontend/src/services/streamingService.ts
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
// Streaming service for progressive listing data loading
|
||||
|
||||
import type { User } from 'oidc-client-ts';
|
||||
import type { PropertyFeature } from '@/types';
|
||||
import type { ParameterValues } from '@/components/FilterPanel';
|
||||
import { ApiError } from '@/types';
|
||||
import { API_ENDPOINTS } from '@/constants';
|
||||
|
||||
/**
|
||||
* Build query string from parameters object
|
||||
*/
|
||||
function buildQueryString(params: Record<string, string | number | boolean | Date | undefined>): string {
|
||||
const queryString = new URLSearchParams();
|
||||
|
||||
for (const [key, value] of Object.entries(params)) {
|
||||
if (value !== undefined && value !== null && value !== '') {
|
||||
if (value instanceof Date) {
|
||||
queryString.append(key, value.toISOString());
|
||||
} else {
|
||||
queryString.append(key, String(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return queryString.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Build listing query parameters from form values
|
||||
*/
|
||||
function buildListingParams(parameters: ParameterValues): Record<string, string | number | boolean | Date | undefined> {
|
||||
return {
|
||||
listing_type: parameters.listing_type,
|
||||
min_bedrooms: parameters.min_bedrooms,
|
||||
max_bedrooms: parameters.max_bedrooms,
|
||||
max_price: parameters.max_price,
|
||||
min_price: parameters.min_price,
|
||||
min_sqm: parameters.min_sqm,
|
||||
max_sqm: parameters.max_sqm,
|
||||
min_price_per_sqm: parameters.min_price_per_sqm,
|
||||
max_price_per_sqm: parameters.max_price_per_sqm,
|
||||
last_seen_days: parameters.last_seen_days,
|
||||
let_date_available_from: parameters.available_from,
|
||||
district_names: parameters.district || undefined,
|
||||
furnish_types: parameters.furnish_types?.join(',') || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export interface StreamMessage {
|
||||
type: 'metadata' | 'batch' | 'complete';
|
||||
features?: PropertyFeature[];
|
||||
total?: number;
|
||||
total_expected?: number;
|
||||
batch_size?: number;
|
||||
cached?: boolean;
|
||||
}
|
||||
|
||||
export interface StreamingProgress {
|
||||
count: number;
|
||||
total?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream listing GeoJSON data as an async generator.
|
||||
* Yields batches of features as they arrive from the server.
|
||||
*/
|
||||
export async function* streamListingGeoJSON(
|
||||
user: User,
|
||||
parameters: ParameterValues,
|
||||
onProgress?: (progress: StreamingProgress) => void
|
||||
): AsyncGenerator<PropertyFeature[], void, unknown> {
|
||||
const params = buildListingParams(parameters);
|
||||
const queryString = buildQueryString(params);
|
||||
const url = queryString
|
||||
? `${API_ENDPOINTS.LISTING_GEOJSON_STREAM}?${queryString}`
|
||||
: API_ENDPOINTS.LISTING_GEOJSON_STREAM;
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${user.access_token}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ApiError(`Error: ${response.status}`, response.status);
|
||||
}
|
||||
|
||||
if (!response.body) {
|
||||
throw new Error('No response body');
|
||||
}
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
let totalCount = 0;
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
const lines = buffer.split('\n');
|
||||
buffer = lines.pop() || ''; // Keep incomplete line in buffer
|
||||
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
|
||||
try {
|
||||
const message: StreamMessage = JSON.parse(line);
|
||||
|
||||
if (message.type === 'metadata') {
|
||||
onProgress?.({ count: 0, total: message.total_expected });
|
||||
} else if (message.type === 'batch' && message.features) {
|
||||
totalCount += message.features.length;
|
||||
onProgress?.({ count: totalCount });
|
||||
yield message.features;
|
||||
} else if (message.type === 'complete') {
|
||||
onProgress?.({ count: message.total ?? totalCount, total: message.total });
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Failed to parse streaming message:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process any remaining data in the buffer
|
||||
if (buffer.trim()) {
|
||||
try {
|
||||
const message: StreamMessage = JSON.parse(buffer);
|
||||
if (message.type === 'batch' && message.features) {
|
||||
yield message.features;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Failed to parse final streaming message:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue