diff --git a/api/app.py b/api/app.py index f52d176..672c367 100644 --- a/api/app.py +++ b/api/app.py @@ -440,8 +440,10 @@ async def _stream_from_db( async def _repopulate_cache_background(query_parameters: QueryParameters) -> None: """Repopulate the cache from DB in the background (fire-and-forget).""" if not acquire_repopulation_lock(query_parameters): + app_metrics.cache_repopulation_total.add(1, {"result": "skipped"}) logger.debug("Skipping background repopulation — already in progress") return + app_metrics.cache_repopulation_total.add(1, {"result": "started"}) try: logger.info("Starting background cache repopulation for stale entry") repository = ListingRepository(engine) @@ -453,14 +455,46 @@ async def _repopulate_cache_background(query_parameters: QueryParameters) -> Non feature = convert_row_to_geojson(row, query_parameters.listing_type.value) cache_features_batch_staged(staging_key, [feature]) finalize_cache_population(staging_key, query_parameters) + app_metrics.cache_repopulation_total.add(1, {"result": "completed"}) logger.info("Background cache repopulation completed") except Exception: delete_staging_key(staging_key) raise except Exception: + app_metrics.cache_repopulation_total.add(1, {"result": "failed"}) logger.exception("Background cache repopulation failed") +async def _instrumented_stream( + inner: AsyncGenerator[str, None], + source: str, +) -> AsyncGenerator[str, None]: + """Wrap a streaming generator to record TTFB, total duration, and feature count.""" + t0 = time.monotonic() + first_yielded = False + feature_count = 0 + try: + async for chunk in inner: + if not first_yielded: + app_metrics.stream_time_to_first_byte_seconds.record( + time.monotonic() - t0, {"source": source} + ) + first_yielded = True + # Count features from batch messages + try: + msg = json.loads(chunk) + if msg.get("type") == "batch" and "features" in msg: + feature_count += len(msg["features"]) + except (json.JSONDecodeError, TypeError): + pass + yield chunk + finally: + duration = time.monotonic() - t0 + app_metrics.stream_total_duration_seconds.record(duration, {"source": source}) + app_metrics.stream_features_total.add(feature_count, {"source": source}) + app_metrics.stream_requests_total.add(1, {"source": source}) + + @app.get("/api/listing_geojson/stream") async def stream_listing_geojson( user: Annotated[User, Depends(get_current_user)], @@ -501,21 +535,28 @@ async def stream_listing_geojson( app_metrics.geojson_cache_operations.add(1, {"result": "hit"}) stale = is_cache_stale(query_parameters) if stale: + app_metrics.cache_stale_serves_total.add(1) # Fire-and-forget background repopulation asyncio.create_task(_repopulate_cache_background(query_parameters)) - generator = _stream_from_cache( - query_parameters, batch_size, limit, - user_email=user.email, - decision_filter=decision_filter, - stale=stale, + generator = _instrumented_stream( + _stream_from_cache( + query_parameters, batch_size, limit, + user_email=user.email, + decision_filter=decision_filter, + stale=stale, + ), + source="cache", ) else: app_metrics.geojson_cache_operations.add(1, {"result": "miss"}) - generator = _stream_from_db( - query_parameters, batch_size, limit, poi_distances_lookup, - skip_cache=include_poi_distances, - user_email=user.email, - decision_filter=decision_filter, + generator = _instrumented_stream( + _stream_from_db( + query_parameters, batch_size, limit, poi_distances_lookup, + skip_cache=include_poi_distances, + user_email=user.email, + decision_filter=decision_filter, + ), + source="db", ) return StreamingResponse( @@ -660,9 +701,13 @@ async def get_listing_detail( """Get detailed information for a single listing.""" repository = ListingRepository(engine) lt = ListingType(listing_type) + t_step = time.monotonic() listings = await repository.get_listings( only_ids=[listing_id], listing_type=lt ) + app_metrics.listing_detail_step_duration_seconds.record( + time.monotonic() - t_step, {"step": "fetch_listing"} + ) if not listings: raise HTTPException(status_code=404, detail="Listing not found") @@ -737,6 +782,7 @@ async def get_listing_detail( furnish_type_val = str(ft) # Load user's decision for this listing + t_step = time.monotonic() decision_val: str | None = None user_id = _get_user_id_safe(user.email) if user_id is not None: @@ -746,8 +792,12 @@ async def get_listing_detail( if d.listing_id == listing_id and d.listing_type == listing_type: decision_val = d.decision break + app_metrics.listing_detail_step_duration_seconds.record( + time.monotonic() - t_step, {"step": "load_decision"} + ) # Load POI distances + t_step = time.monotonic() poi_distances_list: list[dict] = [] if user_id is not None: poi_repo = POIRepository(engine) @@ -765,6 +815,9 @@ async def get_listing_detail( "duration_seconds": d.duration_seconds, "distance_meters": d.distance_meters, }) + app_metrics.listing_detail_step_duration_seconds.record( + time.monotonic() - t_step, {"step": "load_poi_distances"} + ) return ListingDetailResponse( id=listing.id, diff --git a/api/metrics.py b/api/metrics.py index 252f45d..edf1bbf 100644 --- a/api/metrics.py +++ b/api/metrics.py @@ -64,6 +64,40 @@ frontend_worker_compute: Histogram frontend_main_thread: Histogram frontend_feature_count: Histogram +# --------------------------------------------------------------------------- +# Database query metrics +# --------------------------------------------------------------------------- +db_query_duration_seconds: Histogram +db_query_rows_returned: Histogram + +# --------------------------------------------------------------------------- +# Streaming lifecycle metrics +# --------------------------------------------------------------------------- +stream_time_to_first_byte_seconds: Histogram +stream_total_duration_seconds: Histogram +stream_features_total: Counter +stream_requests_total: Counter + +# --------------------------------------------------------------------------- +# Cache performance metrics +# --------------------------------------------------------------------------- +cache_operation_duration_seconds: Histogram +cache_repopulation_total: Counter +cache_stale_serves_total: Counter + +# --------------------------------------------------------------------------- +# Listing detail metrics +# --------------------------------------------------------------------------- +listing_detail_step_duration_seconds: Histogram + +# --------------------------------------------------------------------------- +# Frontend navigation/usage metrics +# --------------------------------------------------------------------------- +frontend_page_load: Histogram +frontend_time_to_first_listing: Histogram +frontend_stream_download: Histogram +frontend_listing_detail_load: Histogram + def init_metrics(service_name: str = "realestate-crawler") -> PrometheusMetricReader: """Initialise the OTel MeterProvider and define all instruments. @@ -80,6 +114,14 @@ def init_metrics(service_name: str = "realestate-crawler") -> PrometheusMetricRe global celery_tasks_total, celery_task_duration_seconds, celery_tasks_active global frontend_worker_roundtrip, frontend_worker_compute global frontend_main_thread, frontend_feature_count + global db_query_duration_seconds, db_query_rows_returned + global stream_time_to_first_byte_seconds, stream_total_duration_seconds + global stream_features_total, stream_requests_total + global cache_operation_duration_seconds, cache_repopulation_total + global cache_stale_serves_total + global listing_detail_step_duration_seconds + global frontend_page_load, frontend_time_to_first_listing + global frontend_stream_download, frontend_listing_detail_load if _reader is not None: return _reader @@ -172,9 +214,93 @@ def init_metrics(service_name: str = "realestate-crawler") -> PrometheusMetricRe description="Number of features per heatmap load", ) + # -- Database query timing -- + db_query_duration_seconds = _meter.create_histogram( + "db_query_duration_seconds", + description="Duration of individual database queries in seconds", + ) + db_query_rows_returned = _meter.create_histogram( + "db_query_rows_returned", + description="Number of rows returned per database query", + ) + + # -- Streaming lifecycle -- + stream_time_to_first_byte_seconds = _meter.create_histogram( + "stream_time_to_first_byte_seconds", + description="Time from handler entry to first NDJSON line", + ) + stream_total_duration_seconds = _meter.create_histogram( + "stream_total_duration_seconds", + description="Total wall-clock time for a streaming response", + ) + stream_features_total = _meter.create_counter( + "stream_features_total", + description="Total GeoJSON features streamed to clients", + ) + stream_requests_total = _meter.create_counter( + "stream_requests_total", + description="Total streaming requests served", + ) + + # -- Cache performance -- + cache_operation_duration_seconds = _meter.create_histogram( + "cache_operation_duration_seconds", + description="Redis cache operation latency in seconds", + ) + cache_repopulation_total = _meter.create_counter( + "cache_repopulation_total", + description="Cache repopulation events by result", + ) + cache_stale_serves_total = _meter.create_counter( + "cache_stale_serves_total", + description="Number of times stale cache was served during repopulation", + ) + + # -- Listing detail -- + listing_detail_step_duration_seconds = _meter.create_histogram( + "listing_detail_step_duration_seconds", + description="Per-step timing in listing detail endpoint", + ) + + # -- Frontend navigation/usage -- + frontend_page_load = _meter.create_histogram( + "frontend_page_load_seconds", + description="Full page or filter load to data rendered", + ) + frontend_time_to_first_listing = _meter.create_histogram( + "frontend_time_to_first_listing_seconds", + description="Time from load trigger to first listing batch on screen", + ) + frontend_stream_download = _meter.create_histogram( + "frontend_stream_download_seconds", + description="Client-side total stream download duration", + ) + frontend_listing_detail_load = _meter.create_histogram( + "frontend_listing_detail_load_seconds", + description="Time from click to listing detail data rendered", + ) + return _reader +def record_db_query( + operation: str, + model: str, + duration: float, + rows: int | None = None, +) -> None: + """Record a database query timing metric. + + Safe to call even when ``init_metrics()`` has not been called (e.g. + from CLI usage) — silently no-ops in that case. + """ + if _meter is None: + return + db_query_duration_seconds.record(duration, {"operation": operation, "model": model}) + if rows is not None: + db_query_rows_returned.record(rows, {"operation": operation}) + + def get_metrics_asgi_app(): # type: ignore[no-untyped-def] """Return the Prometheus ASGI app for mounting at /metrics.""" return make_asgi_app() diff --git a/api/perf_routes.py b/api/perf_routes.py index bbf4a12..eab83bb 100644 --- a/api/perf_routes.py +++ b/api/perf_routes.py @@ -6,7 +6,10 @@ from pydantic import BaseModel, Field, field_validator import api.metrics as app_metrics -ALLOWED_METRICS = {"worker_roundtrip", "worker_compute", "main_thread", "feature_count"} +ALLOWED_METRICS = { + "worker_roundtrip", "worker_compute", "main_thread", "feature_count", + "page_load", "time_to_first_listing", "stream_download", "listing_detail_load", +} MAX_BATCH_SIZE = 100 @@ -41,5 +44,13 @@ async def record_perf(samples: list[PerfSample]) -> Response: app_metrics.frontend_main_thread.record(s.value, attrs) elif s.metric == "feature_count": app_metrics.frontend_feature_count.record(s.value) + elif s.metric == "page_load": + app_metrics.frontend_page_load.record(s.value, attrs) + elif s.metric == "time_to_first_listing": + app_metrics.frontend_time_to_first_listing.record(s.value, attrs) + elif s.metric == "stream_download": + app_metrics.frontend_stream_download.record(s.value) + elif s.metric == "listing_detail_load": + app_metrics.frontend_listing_detail_load.record(s.value) return Response(status_code=204) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 218dff9..dd440a3 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -17,7 +17,7 @@ import { Sheet, SheetContent, SheetTrigger } from './components/ui/sheet'; import { Button } from './components/ui/button'; import { Filter, Heart } from 'lucide-react'; import type { GeoJSONFeatureCollection, PropertyProperties, PropertyFeature, POI, POITravelFilter } from '@/types'; -import { refreshListings, streamListingGeoJSON, fetchUserPOIs, fetchBulkPOIDistances, type StreamingProgress } from '@/services'; +import { refreshListings, streamListingGeoJSON, fetchUserPOIs, fetchBulkPOIDistances, recordPerf, type StreamingProgress } from '@/services'; import { getCached, setCached, invalidateAll as invalidateListingCache } from '@/services/listingCache'; import { setOnUnauthorized } from '@/services/apiClient'; import { clearPasskeyUser } from './auth/passkeyService'; @@ -132,7 +132,7 @@ function App() { const loadListings = useCallback(async (parameters: ParameterValues) => { if (!user) return; - // Check in-memory cache first + const operation = queryParameters === null ? 'initial' : 'filter_change'; const cached = getCached(parameters); if (cached) { setQueryParameters(parameters); @@ -158,7 +158,8 @@ function App() { setStreamingProgress({ count: 0 }); setListingData(null); - // Dedup safety net: track seen URLs to prevent duplicate features + const loadStart = performance.now(); + let firstBatchSeen = false; const seenUrls = new Set(); let updateScheduled = false; @@ -191,11 +192,16 @@ function App() { }); if (uniqueBatch.length > 0) { accumulatedFeaturesRef.current.push(...uniqueBatch); + if (!firstBatchSeen) { + recordPerf('time_to_first_listing', operation, (performance.now() - loadStart) / 1000); + firstBatchSeen = true; + } scheduleUpdate(); } } // Final flush to ensure all data is rendered flushUpdate(); + recordPerf('page_load', operation, (performance.now() - loadStart) / 1000); // Store successful result in frontend cache setCached(parameters, accumulatedFeaturesRef.current); } catch (error) { diff --git a/frontend/src/hooks/useListingDetail.ts b/frontend/src/hooks/useListingDetail.ts index 7891239..a30d176 100644 --- a/frontend/src/hooks/useListingDetail.ts +++ b/frontend/src/hooks/useListingDetail.ts @@ -2,6 +2,7 @@ import { useState, useRef, useCallback } from 'react'; import type { AuthUser } from '@/auth/types'; import type { ListingDetailData } from '@/types'; import { fetchListingDetail } from '@/services'; +import { record as recordPerf } from '@/services/perfCollector'; export function useListingDetail(user: AuthUser | null) { const [detail, setDetail] = useState(null); @@ -20,8 +21,10 @@ export function useListingDetail(user: AuthUser | null) { } setIsLoading(true); setError(null); + const t0 = performance.now(); try { const data = await fetchListingDetail(user, listingId, listingType); + recordPerf('listing_detail_load', 'fetch', (performance.now() - t0) / 1000); cache.current.set(cacheKey, data); setDetail(data); } catch (e) { diff --git a/frontend/src/services/streamingService.ts b/frontend/src/services/streamingService.ts index 730ec78..5e7c5a4 100644 --- a/frontend/src/services/streamingService.ts +++ b/frontend/src/services/streamingService.ts @@ -6,6 +6,7 @@ import type { ParameterValues } from '@/components/FilterPanel'; import { ApiError } from '@/types'; import { API_ENDPOINTS } from '@/constants'; import { fireUnauthorized } from './apiClient'; +import { record as recordPerf } from './perfCollector'; /** * Build query string from parameters object @@ -99,6 +100,8 @@ export async function* streamListingGeoJSON( const decoder = new TextDecoder(); let buffer = ''; let totalCount = 0; + let streamStart = performance.now(); + let firstBatchRecorded = false; while (true) { if (options?.signal?.aborted) { @@ -123,6 +126,10 @@ export async function* streamListingGeoJSON( } else if (message.type === 'batch' && message.features) { totalCount += message.features.length; onProgress?.({ count: totalCount }); + if (!firstBatchRecorded) { + recordPerf('time_to_first_listing', 'stream', (performance.now() - streamStart) / 1000); + firstBatchRecorded = true; + } yield message.features; } else if (message.type === 'complete') { onProgress?.({ count: message.total ?? totalCount, total: message.total }); @@ -144,4 +151,7 @@ export async function* streamListingGeoJSON( console.error('Failed to parse final streaming message:', e); } } + + // Record total stream download duration + recordPerf('stream_download', 'stream', (performance.now() - streamStart) / 1000); } diff --git a/grafana/dashboard.json b/grafana/dashboard.json index 84fca5d..834b006 100644 --- a/grafana/dashboard.json +++ b/grafana/dashboard.json @@ -1847,8 +1847,8 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(rate(http_server_duration_milliseconds_count{job=\"realestate-crawler-api\"}[5m])) by (http_target)", - "legendFormat": "{{ http_target }}", + "expr": "sum(rate(http_server_duration_milliseconds_count{job=\"realestate-crawler-api\"}[5m])) by (http_server_name)", + "legendFormat": "{{ http_server_name }}", "range": true, "refId": "A" } @@ -2062,8 +2062,8 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(http_server_active_requests{job=\"realestate-crawler-api\"}) by (http_target)", - "legendFormat": "{{ http_target }}", + "expr": "sum(http_server_active_requests{job=\"realestate-crawler-api\"}) by (http_server_name)", + "legendFormat": "{{ http_server_name }}", "range": true, "refId": "A" } @@ -2583,6 +2583,954 @@ ], "title": "Feature Count Over Time", "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 69 }, + "id": 200, + "panels": [], + "title": "Database Query Performance", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Database query duration by operation (p50/p95)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 70 }, + "id": 201, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(db_query_duration_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))", + "legendFormat": "p50 {{ operation }}", + "range": true, + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(db_query_duration_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))", + "legendFormat": "p95 {{ operation }}", + "range": true, + "refId": "B" + } + ], + "title": "DB Query Duration by Operation (p50/p95)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Average query duration by operation — highlights top offenders", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.1 }, + { "color": "red", "value": 0.5 } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 70 }, + "id": 202, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { "calcs": ["mean"], "fields": "", "values": false }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "rate(db_query_duration_seconds_sum{job=\"realestate-crawler-api\"}[5m]) / rate(db_query_duration_seconds_count{job=\"realestate-crawler-api\"}[5m])", + "legendFormat": "{{ operation }}", + "range": true, + "refId": "A" + } + ], + "title": "Top Offenders (Avg Duration)", + "type": "bargauge" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Rows returned per query (avg/p95)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "min": 0, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 70 }, + "id": 203, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "rate(db_query_rows_returned_sum{job=\"realestate-crawler-api\"}[5m]) / rate(db_query_rows_returned_count{job=\"realestate-crawler-api\"}[5m])", + "legendFormat": "avg {{ operation }}", + "range": true, + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(db_query_rows_returned_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))", + "legendFormat": "p95 {{ operation }}", + "range": true, + "refId": "B" + } + ], + "title": "Rows Returned per Query (avg/p95)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 78 }, + "id": 210, + "panels": [], + "title": "Streaming Performance", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Time to first byte by stream source (cache vs db)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 79 }, + "id": 211, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(stream_time_to_first_byte_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, source))", + "legendFormat": "p50 {{ source }}", + "range": true, + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(stream_time_to_first_byte_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, source))", + "legendFormat": "p95 {{ source }}", + "range": true, + "refId": "B" + } + ], + "title": "Stream TTFB by Source (p50/p95)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Total stream duration by source", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 79 }, + "id": 212, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(stream_total_duration_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, source))", + "legendFormat": "p50 {{ source }}", + "range": true, + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(stream_total_duration_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, source))", + "legendFormat": "p95 {{ source }}", + "range": true, + "refId": "B" + } + ], + "title": "Stream Total Duration by Source (p50/p95)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Average features per streaming response", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 79 }, + "id": 213, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "rate(stream_features_total{job=\"realestate-crawler-api\"}[5m]) / rate(stream_requests_total{job=\"realestate-crawler-api\"}[5m])", + "legendFormat": "{{ source }}", + "range": true, + "refId": "A" + } + ], + "title": "Avg Features per Stream", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Cache vs DB stream request ratio", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 79 }, + "id": 214, + "options": { + "displayLabels": ["percent"], + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "pieType": "donut", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "increase(stream_requests_total{job=\"realestate-crawler-api\"}[1h])", + "legendFormat": "{{ source }}", + "range": true, + "refId": "A" + } + ], + "title": "Cache vs DB Stream Ratio", + "type": "piechart" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Total stream requests over time", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 83 }, + "id": 215, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "increase(stream_requests_total{job=\"realestate-crawler-api\"}[5m])", + "legendFormat": "{{ source }}", + "range": true, + "refId": "A" + } + ], + "title": "Stream Requests (5m increase)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 87 }, + "id": 220, + "panels": [], + "title": "Listing Detail Breakdown", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Per-step timing in listing detail endpoint (p50/p95)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 88 }, + "id": 221, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(listing_detail_step_duration_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, step))", + "legendFormat": "p50 {{ step }}", + "range": true, + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(listing_detail_step_duration_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, step))", + "legendFormat": "p95 {{ step }}", + "range": true, + "refId": "B" + } + ], + "title": "Listing Detail Step Duration (p50/p95)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Average duration by step — stacked to show total breakdown", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 88 }, + "id": 222, + "options": { + "legend": { "calcs": ["mean"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "rate(listing_detail_step_duration_seconds_sum{job=\"realestate-crawler-api\"}[5m]) / rate(listing_detail_step_duration_seconds_count{job=\"realestate-crawler-api\"}[5m])", + "legendFormat": "{{ step }}", + "range": true, + "refId": "A" + } + ], + "title": "Listing Detail Stacked Avg by Step", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 96 }, + "id": 230, + "panels": [], + "title": "Cache Performance", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Redis cache operation latency by operation type (p50/p95)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 10, "x": 0, "y": 97 }, + "id": 231, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(cache_operation_duration_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))", + "legendFormat": "p50 {{ operation }}", + "range": true, + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(cache_operation_duration_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))", + "legendFormat": "p95 {{ operation }}", + "range": true, + "refId": "B" + } + ], + "title": "Cache Operation Latency (p50/p95)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Cache repopulation events by result", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 10, "y": 97 }, + "id": 232, + "options": { + "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "increase(cache_repopulation_total{job=\"realestate-crawler-api\"}[1h])", + "legendFormat": "{{ result }}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Repopulation Events", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Number of times stale cache was served during background repopulation", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 10 }, + { "color": "red", "value": 50 } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 97 }, + "id": 233, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "increase(cache_stale_serves_total{job=\"realestate-crawler-api\"}[1h])", + "legendFormat": "Stale Serves", + "range": true, + "refId": "A" + } + ], + "title": "Stale Cache Serves (1h)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 105 }, + "id": 240, + "panels": [], + "title": "Frontend Navigation Metrics", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Full page or filter load time (p50/p95)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 6, "x": 0, "y": 106 }, + "id": 241, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(frontend_page_load_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))", + "legendFormat": "p50 {{ operation }}", + "range": true, + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(frontend_page_load_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))", + "legendFormat": "p95 {{ operation }}", + "range": true, + "refId": "B" + } + ], + "title": "Page Load Time (p50/p95)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Time to first listing batch on screen (p50/p95)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 6, "x": 6, "y": 106 }, + "id": 242, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(frontend_time_to_first_listing_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))", + "legendFormat": "p50 {{ operation }}", + "range": true, + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(frontend_time_to_first_listing_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))", + "legendFormat": "p95 {{ operation }}", + "range": true, + "refId": "B" + } + ], + "title": "Time to First Listing (p50/p95)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Client-side total stream download duration (p50/p95)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 106 }, + "id": 243, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(frontend_stream_download_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le))", + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(frontend_stream_download_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le))", + "legendFormat": "p95", + "range": true, + "refId": "B" + } + ], + "title": "Stream Download Duration (p50/p95)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Time from listing click to detail data rendered", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 3 } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 106 }, + "id": 244, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["mean"], "fields": "", "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "rate(frontend_listing_detail_load_seconds_sum{job=\"realestate-crawler-api\"}[5m]) / rate(frontend_listing_detail_load_seconds_count{job=\"realestate-crawler-api\"}[5m])", + "legendFormat": "Avg Detail Load", + "range": true, + "refId": "A" + } + ], + "title": "Listing Detail Load Time", + "type": "stat" } ], "preload": false, diff --git a/repositories/decision_repository.py b/repositories/decision_repository.py index 2c11aa3..4421ccf 100644 --- a/repositories/decision_repository.py +++ b/repositories/decision_repository.py @@ -1,5 +1,7 @@ from datetime import datetime +import time +from api.metrics import record_db_query from models.decision import ListingDecision from sqlalchemy import Engine from sqlmodel import Session, select @@ -19,6 +21,7 @@ class DecisionRepository: decision: str, ) -> ListingDecision: """Create or update a decision. Uses dialect-specific upsert.""" + t0 = time.monotonic() with Session(self.engine) as session: now = datetime.utcnow() values = { @@ -58,14 +61,18 @@ class DecisionRepository: ) ).first() assert result is not None - return result + record_db_query("upsert_decision", "decision", time.monotonic() - t0) + return result def get_decisions_for_user(self, user_id: int) -> list[ListingDecision]: + t0 = time.monotonic() with Session(self.engine) as session: statement = select(ListingDecision).where( ListingDecision.user_id == user_id ) - return list(session.exec(statement).all()) + results = list(session.exec(statement).all()) + record_db_query("get_decisions_for_user", "decision", time.monotonic() - t0, len(results)) + return results def delete_decision( self, @@ -92,13 +99,16 @@ class DecisionRepository: user_id: int, listing_type: str, ) -> set[int]: + t0 = time.monotonic() with Session(self.engine) as session: statement = select(ListingDecision.listing_id).where( ListingDecision.user_id == user_id, ListingDecision.listing_type == listing_type, ListingDecision.decision == "disliked", ) - return {row for row in session.exec(statement).all()} + ids = {row for row in session.exec(statement).all()} + record_db_query("get_disliked_listing_ids", "decision", time.monotonic() - t0, len(ids)) + return ids def get_liked_listing_ids( self, diff --git a/repositories/listing_repository.py b/repositories/listing_repository.py index b02f7e8..41eb362 100644 --- a/repositories/listing_repository.py +++ b/repositories/listing_repository.py @@ -1,6 +1,8 @@ from datetime import datetime, timedelta import logging +import time from typing import Any, Generator +from api.metrics import record_db_query from data_access import Listing from models.listing import ( BuyListing, @@ -55,8 +57,10 @@ class ListingRepository: if limit: query = query.limit(limit) + t0 = time.monotonic() with Session(self.engine) as session: rows = list(session.exec(query).all()) + record_db_query("get_listings", model.__tablename__, time.monotonic() - t0, len(rows)) logging.debug(f"Found {len(rows)} listings") return rows @@ -110,8 +114,11 @@ class ListingRepository: query = sa_select(func.count(model.id)) query = self._apply_query_filters(query, model, query_parameters) + t0 = time.monotonic() with Session(self.engine) as session: - return session.execute(query).scalar() or 0 + result = session.execute(query).scalar() or 0 + record_db_query("count_listings", model.__tablename__, time.monotonic() - t0, result) + return result def stream_listings_optimized( self, @@ -157,8 +164,10 @@ class ListingRepository: batch_limit = min(page_size, limit - total_yielded) query = query.order_by(model.id).limit(batch_limit) + t0 = time.monotonic() with Session(self.engine) as session: results = session.execute(query).fetchall() + record_db_query("stream_listings_page", model.__tablename__, time.monotonic() - t0, len(results)) if not results: break @@ -364,6 +373,7 @@ class ListingRepository: Checks both RentListing and BuyListing tables and returns the latest. """ + t0 = time.monotonic() with Session(self.engine) as session: rent_max = session.execute( sa_select(func.max(RentListing.last_seen)) @@ -371,6 +381,7 @@ class ListingRepository: buy_max = session.execute( sa_select(func.max(BuyListing.last_seen)) ).scalar() + record_db_query("get_last_updated", "rent", time.monotonic() - t0) candidates = [t for t in (rent_max, buy_max) if t is not None] return max(candidates) if candidates else None @@ -385,9 +396,12 @@ class ListingRepository: filtering against API results. """ model = RentListing if listing_type == ListingType.RENT else BuyListing + t0 = time.monotonic() with Session(self.engine) as session: result = session.execute(sa_select(model.id)) - return {row[0] for row in result.fetchall()} + ids = {row[0] for row in result.fetchall()} + record_db_query("get_listing_ids", model.__tablename__, time.monotonic() - t0, len(ids)) + return ids async def mark_seen(self, listing_id: int, listing_type: ListingType = ListingType.RENT) -> None: query_params = QueryParameters(listing_type=listing_type) diff --git a/repositories/poi_repository.py b/repositories/poi_repository.py index f6e4d29..5b9702a 100644 --- a/repositories/poi_repository.py +++ b/repositories/poi_repository.py @@ -1,3 +1,5 @@ +import time +from api.metrics import record_db_query from models.listing import ListingType from models.poi import PointOfInterest from models.poi_distance import POIDistance @@ -12,11 +14,14 @@ class POIRepository: self.engine = engine def get_pois_for_user(self, user_id: int) -> list[PointOfInterest]: + t0 = time.monotonic() with Session(self.engine) as session: statement = select(PointOfInterest).where( PointOfInterest.user_id == user_id ) - return list(session.exec(statement).all()) + results = list(session.exec(statement).all()) + record_db_query("get_pois_for_user", "poi", time.monotonic() - t0, len(results)) + return results def get_poi_by_id(self, poi_id: int) -> PointOfInterest | None: with Session(self.engine) as session: @@ -55,6 +60,7 @@ class POIRepository: """Insert or update POI distances, handling duplicate unique constraints.""" if not distances: return + t0 = time.monotonic() with Session(self.engine) as session: dialect = self.engine.dialect.name for dist in distances: @@ -88,6 +94,7 @@ class POIRepository: ) session.execute(stmt) session.commit() + record_db_query("upsert_distances", "poi_distance", time.monotonic() - t0, len(distances)) def get_distances_for_listings( self, @@ -95,6 +102,7 @@ class POIRepository: listing_type: ListingType, user_id: int, ) -> list[POIDistance]: + t0 = time.monotonic() with Session(self.engine) as session: # Join with POI to filter by user statement = ( @@ -106,7 +114,9 @@ class POIRepository: PointOfInterest.user_id == user_id, ) ) - return list(session.exec(statement).all()) + results = list(session.exec(statement).all()) + record_db_query("get_distances_for_listings", "poi_distance", time.monotonic() - t0, len(results)) + return results def get_distances_for_poi(self, poi_id: int) -> list[POIDistance]: with Session(self.engine) as session: diff --git a/services/listing_cache.py b/services/listing_cache.py index c3795a1..b693830 100644 --- a/services/listing_cache.py +++ b/services/listing_cache.py @@ -3,16 +3,25 @@ import hashlib import json import logging import os +import time import uuid from typing import Generator from urllib.parse import urlparse, urlunparse import redis +import api.metrics as app_metrics from models.listing import QueryParameters logger = logging.getLogger(__name__) + +def _record_cache_op(operation: str, duration: float) -> None: + """Record a cache operation timing metric, no-op if metrics aren't initialized.""" + if app_metrics._meter is None: + return + app_metrics.cache_operation_duration_seconds.record(duration, {"operation": operation}) + CACHE_PREFIX = "listings:geojson:" STAGING_PREFIX = "listings:geojson:staging:" CACHE_TTL_SECONDS = 24 * 60 * 60 # 24 hours @@ -40,11 +49,15 @@ def make_cache_key(query_params: QueryParameters) -> str: def get_cached_count(query_params: QueryParameters) -> int | None: """Return the number of cached features for a query, or None if not cached.""" try: + t0 = time.monotonic() client = _get_redis_client() key = make_cache_key(query_params) if not client.exists(key): + _record_cache_op("check", time.monotonic() - t0) return None - return client.llen(key) + count = client.llen(key) + _record_cache_op("check", time.monotonic() - t0) + return count except redis.RedisError as e: logger.warning(f"Redis cache read error: {e}") return None @@ -61,7 +74,9 @@ def get_cached_features( for start in range(0, total, batch_size): end = start + batch_size - 1 + t0 = time.monotonic() items = client.lrange(key, start, end) + _record_cache_op("read_batch", time.monotonic() - t0) batch = [json.loads(item) for item in items] if batch: yield batch @@ -100,12 +115,14 @@ def cache_features_batch_staged(staging_key: str, features: list[dict]) -> None: if not features: return try: + t0 = time.monotonic() client = _get_redis_client() pipeline = client.pipeline() for feature in features: pipeline.rpush(staging_key, json.dumps(feature)) pipeline.expire(staging_key, STAGING_TTL_SECONDS) pipeline.execute() + _record_cache_op("write_batch", time.monotonic() - t0) except redis.RedisError as e: logger.warning(f"Redis staged cache write error: {e}") @@ -113,11 +130,13 @@ def cache_features_batch_staged(staging_key: str, features: list[dict]) -> None: def finalize_cache_population(staging_key: str, query_params: QueryParameters) -> None: """Atomically rename the staging key to the live cache key and set TTL.""" try: + t0 = time.monotonic() client = _get_redis_client() live_key = make_cache_key(query_params) # RENAME is atomic — replaces the live key in one operation client.rename(staging_key, live_key) client.expire(live_key, CACHE_TTL_SECONDS) + _record_cache_op("finalize", time.monotonic() - t0) logger.debug(f"Finalized cache population for {live_key}") except redis.RedisError as e: logger.warning(f"Redis cache finalize error: {e}")