Remove 1000-result limit, add Redis caching and virtual scrolling
- Remove hard-coded limit=1000 default from listing_geojson and streaming endpoints, allowing all matching results to be returned - Add Redis caching service (db=2, 30min TTL) that caches query results as Redis Lists for fast re-queries with reduced DB load - Integrate cache into streaming endpoint: serve from cache on hit, populate cache on miss during DB streaming - Invalidate cache after scrape completes (both success and no-new-listings) - Replace ScrollArea with react-virtuoso in ListView for virtual scrolling, keeping only ~20-30 DOM nodes regardless of list size - Handle metadata streaming message to show "0 / N" progress from start - Throttle frontend state updates with requestAnimationFrame to prevent UI jank from rapid re-renders during cached response streaming
This commit is contained in:
parent
c4b11ccfe9
commit
5514fa6381
8 changed files with 695 additions and 78 deletions
|
|
@ -16,6 +16,7 @@ from repositories.listing_repository import ListingRepository
|
|||
from database import engine
|
||||
from services.query_splitter import QuerySplitter, SubQuery
|
||||
from utils.redis_lock import redis_lock
|
||||
from services.listing_cache import invalidate_cache
|
||||
|
||||
logger = logging.getLogger("uvicorn.error")
|
||||
|
||||
|
|
@ -88,6 +89,7 @@ async def dump_listings_full(
|
|||
if len(ids_to_process) == 0:
|
||||
elapsed = time.time() - start_time
|
||||
celery_logger.info(f"No new listings found. Completed in {elapsed:.1f}s")
|
||||
invalidate_cache()
|
||||
task.update_state(
|
||||
state="No new listings found",
|
||||
meta={"progress": 1, "processed": 0, "total": 0, "message": "All listings are up to date"},
|
||||
|
|
@ -111,6 +113,8 @@ async def dump_listings_full(
|
|||
celery_logger.info(f"COMPLETED: Processed {len(result)} listings in {elapsed:.1f}s")
|
||||
celery_logger.info("=" * 60)
|
||||
|
||||
invalidate_cache()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue