Remove 1000-result limit, add Redis caching and virtual scrolling

- Remove hard-coded limit=1000 default from listing_geojson and streaming
  endpoints, allowing all matching results to be returned
- Add Redis caching service (db=2, 30min TTL) that caches query results
  as Redis Lists for fast re-queries with reduced DB load
- Integrate cache into streaming endpoint: serve from cache on hit,
  populate cache on miss during DB streaming
- Invalidate cache after scrape completes (both success and no-new-listings)
- Replace ScrollArea with react-virtuoso in ListView for virtual scrolling,
  keeping only ~20-30 DOM nodes regardless of list size
- Handle metadata streaming message to show "0 / N" progress from start
- Throttle frontend state updates with requestAnimationFrame to prevent
  UI jank from rapid re-renders during cached response streaming
This commit is contained in:
Viktor Barzin 2026-02-06 20:34:50 +00:00
parent 3267adca66
commit 500cb576db
8 changed files with 695 additions and 78 deletions

View file

@ -67,16 +67,32 @@ function App() {
setStreamingProgress({ count: 0 });
setListingData(null);
let updateScheduled = false;
const flushUpdate = () => {
updateScheduled = false;
setListingData({
type: 'FeatureCollection',
features: [...accumulatedFeaturesRef.current]
});
};
const scheduleUpdate = () => {
if (!updateScheduled) {
updateScheduled = true;
requestAnimationFrame(flushUpdate);
}
};
try {
for await (const batch of streamListingGeoJSON(user, parameters, (progress) => {
setStreamingProgress(progress);
})) {
accumulatedFeaturesRef.current.push(...batch);
setListingData({
type: 'FeatureCollection',
features: [...accumulatedFeaturesRef.current]
});
scheduleUpdate();
}
// Final flush to ensure all data is rendered
flushUpdate();
} catch (error) {
if (error instanceof Error) {
setSubmitError(error.message);