viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add navigation & usage metrics for end-user experience visibility

Browse source 
Instrument DB query timing (11 operations across 3 repositories),
streaming lifecycle (TTFB, duration, feature count), cache operation
latency, listing detail step breakdown, and frontend page load /
time-to-first-listing / stream download / detail load metrics.

Adds 16 new OTel instruments, extends the perf ingestion endpoint
with 4 new frontend metrics, and adds ~20 Grafana dashboard panels
across 4 new rows (DB Query Performance, Streaming Performance,
Listing Detail Breakdown, Cache Performance, Frontend Navigation).
This commit is contained in:
Viktor Barzin 2026-02-23 20:28:42 +00:00
parent 1ae00b7cbf
commit 35f1987ac1
No known key found for this signature in database
GPG key ID: 0EB088298288D958
11 changed files with 1236 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

73
api/app.py
View file

@ -440,8 +440,10 @@ async def _stream_from_db(
async def _repopulate_cache_background(query_parameters: QueryParameters) -> None:
"""Repopulate the cache from DB in the background (fire-and-forget)."""
if not acquire_repopulation_lock(query_parameters):
app_metrics.cache_repopulation_total.add(1, {"result": "skipped"})
logger.debug("Skipping background repopulation — already in progress")
return
app_metrics.cache_repopulation_total.add(1, {"result": "started"})
try:
logger.info("Starting background cache repopulation for stale entry")
repository = ListingRepository(engine)
@ -453,14 +455,46 @@ async def _repopulate_cache_background(query_parameters: QueryParameters) -> Non
feature = convert_row_to_geojson(row, query_parameters.listing_type.value)
cache_features_batch_staged(staging_key, [feature])
finalize_cache_population(staging_key, query_parameters)
app_metrics.cache_repopulation_total.add(1, {"result": "completed"})
logger.info("Background cache repopulation completed")
except Exception:
delete_staging_key(staging_key)
raise
except Exception:
app_metrics.cache_repopulation_total.add(1, {"result": "failed"})
logger.exception("Background cache repopulation failed")
async def _instrumented_stream(
inner: AsyncGenerator[str, None],
source: str,
) -> AsyncGenerator[str, None]:
"""Wrap a streaming generator to record TTFB, total duration, and feature count."""
t0 = time.monotonic()
first_yielded = False
feature_count = 0
try:
async for chunk in inner:
if not first_yielded:
app_metrics.stream_time_to_first_byte_seconds.record(
time.monotonic() - t0, {"source": source}
)
first_yielded = True
# Count features from batch messages
try:
msg = json.loads(chunk)
if msg.get("type") == "batch" and "features" in msg:
feature_count += len(msg["features"])
except (json.JSONDecodeError, TypeError):
pass
yield chunk
finally:
duration = time.monotonic() - t0
app_metrics.stream_total_duration_seconds.record(duration, {"source": source})
app_metrics.stream_features_total.add(feature_count, {"source": source})
app_metrics.stream_requests_total.add(1, {"source": source})
@app.get("/api/listing_geojson/stream")
async def stream_listing_geojson(
user: Annotated[User, Depends(get_current_user)],
@ -501,21 +535,28 @@ async def stream_listing_geojson(
app_metrics.geojson_cache_operations.add(1, {"result": "hit"})
stale = is_cache_stale(query_parameters)
if stale:
app_metrics.cache_stale_serves_total.add(1)
# Fire-and-forget background repopulation
asyncio.create_task(_repopulate_cache_background(query_parameters))
generator = _stream_from_cache(
query_parameters, batch_size, limit,
user_email=user.email,
decision_filter=decision_filter,
stale=stale,
generator = _instrumented_stream(
_stream_from_cache(
query_parameters, batch_size, limit,
user_email=user.email,
decision_filter=decision_filter,
stale=stale,
),
source="cache",
)
else:
app_metrics.geojson_cache_operations.add(1, {"result": "miss"})
generator = _stream_from_db(
query_parameters, batch_size, limit, poi_distances_lookup,
skip_cache=include_poi_distances,
user_email=user.email,
decision_filter=decision_filter,
generator = _instrumented_stream(
_stream_from_db(
query_parameters, batch_size, limit, poi_distances_lookup,
skip_cache=include_poi_distances,
user_email=user.email,
decision_filter=decision_filter,
),
source="db",
)
return StreamingResponse(
@ -660,9 +701,13 @@ async def get_listing_detail(
"""Get detailed information for a single listing."""
repository = ListingRepository(engine)
lt = ListingType(listing_type)
t_step = time.monotonic()
listings = await repository.get_listings(
only_ids=[listing_id], listing_type=lt
)
app_metrics.listing_detail_step_duration_seconds.record(
time.monotonic() - t_step, {"step": "fetch_listing"}
)
if not listings:
raise HTTPException(status_code=404, detail="Listing not found")
@ -737,6 +782,7 @@ async def get_listing_detail(
furnish_type_val = str(ft)
# Load user's decision for this listing
t_step = time.monotonic()
decision_val: str | None = None
user_id = _get_user_id_safe(user.email)
if user_id is not None:
@ -746,8 +792,12 @@ async def get_listing_detail(
if d.listing_id == listing_id and d.listing_type == listing_type:
decision_val = d.decision
break
app_metrics.listing_detail_step_duration_seconds.record(
time.monotonic() - t_step, {"step": "load_decision"}
)
# Load POI distances
t_step = time.monotonic()
poi_distances_list: list[dict] = []
if user_id is not None:
poi_repo = POIRepository(engine)
@ -765,6 +815,9 @@ async def get_listing_detail(
"duration_seconds": d.duration_seconds,
"distance_meters": d.distance_meters,
})
app_metrics.listing_detail_step_duration_seconds.record(
time.monotonic() - t_step, {"step": "load_poi_distances"}
)
return ListingDetailResponse(
id=listing.id,

126
api/metrics.py
View file

@ -64,6 +64,40 @@ frontend_worker_compute: Histogram
frontend_main_thread: Histogram
frontend_feature_count: Histogram
# ---------------------------------------------------------------------------
# Database query metrics
# ---------------------------------------------------------------------------
db_query_duration_seconds: Histogram
db_query_rows_returned: Histogram
# ---------------------------------------------------------------------------
# Streaming lifecycle metrics
# ---------------------------------------------------------------------------
stream_time_to_first_byte_seconds: Histogram
stream_total_duration_seconds: Histogram
stream_features_total: Counter
stream_requests_total: Counter
# ---------------------------------------------------------------------------
# Cache performance metrics
# ---------------------------------------------------------------------------
cache_operation_duration_seconds: Histogram
cache_repopulation_total: Counter
cache_stale_serves_total: Counter
# ---------------------------------------------------------------------------
# Listing detail metrics
# ---------------------------------------------------------------------------
listing_detail_step_duration_seconds: Histogram
# ---------------------------------------------------------------------------
# Frontend navigation/usage metrics
# ---------------------------------------------------------------------------
frontend_page_load: Histogram
frontend_time_to_first_listing: Histogram
frontend_stream_download: Histogram
frontend_listing_detail_load: Histogram
def init_metrics(service_name: str = "realestate-crawler") -> PrometheusMetricReader:
"""Initialise the OTel MeterProvider and define all instruments.
@ -80,6 +114,14 @@ def init_metrics(service_name: str = "realestate-crawler") -> PrometheusMetricRe
global celery_tasks_total, celery_task_duration_seconds, celery_tasks_active
global frontend_worker_roundtrip, frontend_worker_compute
global frontend_main_thread, frontend_feature_count
global db_query_duration_seconds, db_query_rows_returned
global stream_time_to_first_byte_seconds, stream_total_duration_seconds
global stream_features_total, stream_requests_total
global cache_operation_duration_seconds, cache_repopulation_total
global cache_stale_serves_total
global listing_detail_step_duration_seconds
global frontend_page_load, frontend_time_to_first_listing
global frontend_stream_download, frontend_listing_detail_load
if _reader is not None:
return _reader
@ -172,9 +214,93 @@ def init_metrics(service_name: str = "realestate-crawler") -> PrometheusMetricRe
description="Number of features per heatmap load",
)
# -- Database query timing --
db_query_duration_seconds = _meter.create_histogram(
"db_query_duration_seconds",
description="Duration of individual database queries in seconds",
)
db_query_rows_returned = _meter.create_histogram(
"db_query_rows_returned",
description="Number of rows returned per database query",
)
# -- Streaming lifecycle --
stream_time_to_first_byte_seconds = _meter.create_histogram(
"stream_time_to_first_byte_seconds",
description="Time from handler entry to first NDJSON line",
)
stream_total_duration_seconds = _meter.create_histogram(
"stream_total_duration_seconds",
description="Total wall-clock time for a streaming response",
)
stream_features_total = _meter.create_counter(
"stream_features_total",
description="Total GeoJSON features streamed to clients",
)
stream_requests_total = _meter.create_counter(
"stream_requests_total",
description="Total streaming requests served",
)
# -- Cache performance --
cache_operation_duration_seconds = _meter.create_histogram(
"cache_operation_duration_seconds",
description="Redis cache operation latency in seconds",
)
cache_repopulation_total = _meter.create_counter(
"cache_repopulation_total",
description="Cache repopulation events by result",
)
cache_stale_serves_total = _meter.create_counter(
"cache_stale_serves_total",
description="Number of times stale cache was served during repopulation",
)
# -- Listing detail --
listing_detail_step_duration_seconds = _meter.create_histogram(
"listing_detail_step_duration_seconds",
description="Per-step timing in listing detail endpoint",
)
# -- Frontend navigation/usage --
frontend_page_load = _meter.create_histogram(
"frontend_page_load_seconds",
description="Full page or filter load to data rendered",
)
frontend_time_to_first_listing = _meter.create_histogram(
"frontend_time_to_first_listing_seconds",
description="Time from load trigger to first listing batch on screen",
)
frontend_stream_download = _meter.create_histogram(
"frontend_stream_download_seconds",
description="Client-side total stream download duration",
)
frontend_listing_detail_load = _meter.create_histogram(
"frontend_listing_detail_load_seconds",
description="Time from click to listing detail data rendered",
)
return _reader
def record_db_query(
operation: str,
model: str,
duration: float,
rows: int | None = None,
) -> None:
"""Record a database query timing metric.
Safe to call even when ``init_metrics()`` has not been called (e.g.
from CLI usage) — silently no-ops in that case.
"""
if _meter is None:
return
db_query_duration_seconds.record(duration, {"operation": operation, "model": model})
if rows is not None:
db_query_rows_returned.record(rows, {"operation": operation})
def get_metrics_asgi_app(): # type: ignore[no-untyped-def]
"""Return the Prometheus ASGI app for mounting at /metrics."""
return make_asgi_app()

13
api/perf_routes.py
View file

@ -6,7 +6,10 @@ from pydantic import BaseModel, Field, field_validator
import api.metrics as app_metrics
ALLOWED_METRICS = {"worker_roundtrip", "worker_compute", "main_thread", "feature_count"}
ALLOWED_METRICS = {
"worker_roundtrip", "worker_compute", "main_thread", "feature_count",
"page_load", "time_to_first_listing", "stream_download", "listing_detail_load",
}
MAX_BATCH_SIZE = 100
@ -41,5 +44,13 @@ async def record_perf(samples: list[PerfSample]) -> Response:
app_metrics.frontend_main_thread.record(s.value, attrs)
elif s.metric == "feature_count":
app_metrics.frontend_feature_count.record(s.value)
elif s.metric == "page_load":
app_metrics.frontend_page_load.record(s.value, attrs)
elif s.metric == "time_to_first_listing":
app_metrics.frontend_time_to_first_listing.record(s.value, attrs)
elif s.metric == "stream_download":
app_metrics.frontend_stream_download.record(s.value)
elif s.metric == "listing_detail_load":
app_metrics.frontend_listing_detail_load.record(s.value)
return Response(status_code=204)

12
frontend/src/App.tsx
View file

@ -17,7 +17,7 @@ import { Sheet, SheetContent, SheetTrigger } from './components/ui/sheet';
import { Button } from './components/ui/button';
import { Filter, Heart } from 'lucide-react';
import type { GeoJSONFeatureCollection, PropertyProperties, PropertyFeature, POI, POITravelFilter } from '@/types';
import { refreshListings, streamListingGeoJSON, fetchUserPOIs, fetchBulkPOIDistances, type StreamingProgress } from '@/services';
import { refreshListings, streamListingGeoJSON, fetchUserPOIs, fetchBulkPOIDistances, recordPerf, type StreamingProgress } from '@/services';
import { getCached, setCached, invalidateAll as invalidateListingCache } from '@/services/listingCache';
import { setOnUnauthorized } from '@/services/apiClient';
import { clearPasskeyUser } from './auth/passkeyService';
@ -132,7 +132,7 @@ function App() {
const loadListings = useCallback(async (parameters: ParameterValues) => {
if (!user) return;
// Check in-memory cache first
const operation = queryParameters === null ? 'initial' : 'filter_change';
const cached = getCached(parameters);
if (cached) {
setQueryParameters(parameters);
@ -158,7 +158,8 @@ function App() {
setStreamingProgress({ count: 0 });
setListingData(null);
// Dedup safety net: track seen URLs to prevent duplicate features
const loadStart = performance.now();
let firstBatchSeen = false;
const seenUrls = new Set<string>();
let updateScheduled = false;
@ -191,11 +192,16 @@ function App() {
});
if (uniqueBatch.length > 0) {
accumulatedFeaturesRef.current.push(...uniqueBatch);
if (!firstBatchSeen) {
recordPerf('time_to_first_listing', operation, (performance.now() - loadStart) / 1000);
firstBatchSeen = true;
}
scheduleUpdate();
}
}
// Final flush to ensure all data is rendered
flushUpdate();
recordPerf('page_load', operation, (performance.now() - loadStart) / 1000);
// Store successful result in frontend cache
setCached(parameters, accumulatedFeaturesRef.current);
} catch (error) {

3
frontend/src/hooks/useListingDetail.ts
View file

@ -2,6 +2,7 @@ import { useState, useRef, useCallback } from 'react';
import type { AuthUser } from '@/auth/types';
import type { ListingDetailData } from '@/types';
import { fetchListingDetail } from '@/services';
import { record as recordPerf } from '@/services/perfCollector';
export function useListingDetail(user: AuthUser | null) {
const [detail, setDetail] = useState<ListingDetailData | null>(null);
@ -20,8 +21,10 @@ export function useListingDetail(user: AuthUser | null) {
}
setIsLoading(true);
setError(null);
const t0 = performance.now();
try {
const data = await fetchListingDetail(user, listingId, listingType);
recordPerf('listing_detail_load', 'fetch', (performance.now() - t0) / 1000);
cache.current.set(cacheKey, data);
setDetail(data);
} catch (e) {

10
frontend/src/services/streamingService.ts
View file

@ -6,6 +6,7 @@ import type { ParameterValues } from '@/components/FilterPanel';
import { ApiError } from '@/types';
import { API_ENDPOINTS } from '@/constants';
import { fireUnauthorized } from './apiClient';
import { record as recordPerf } from './perfCollector';
/**
* Build query string from parameters object
@ -99,6 +100,8 @@ export async function* streamListingGeoJSON(
const decoder = new TextDecoder();
let buffer = '';
let totalCount = 0;
let streamStart = performance.now();
let firstBatchRecorded = false;
while (true) {
if (options?.signal?.aborted) {
@ -123,6 +126,10 @@ export async function* streamListingGeoJSON(
} else if (message.type === 'batch' && message.features) {
totalCount += message.features.length;
onProgress?.({ count: totalCount });
if (!firstBatchRecorded) {
recordPerf('time_to_first_listing', 'stream', (performance.now() - streamStart) / 1000);
firstBatchRecorded = true;
}
yield message.features;
} else if (message.type === 'complete') {
onProgress?.({ count: message.total ?? totalCount, total: message.total });
@ -144,4 +151,7 @@ export async function* streamListingGeoJSON(
console.error('Failed to parse final streaming message:', e);
}
}
// Record total stream download duration
recordPerf('stream_download', 'stream', (performance.now() - streamStart) / 1000);
}

956
grafana/dashboard.json
View file

@ -1847,8 +1847,8 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(rate(http_server_duration_milliseconds_count{job=\"realestate-crawler-api\"}[5m])) by (http_target)",
"legendFormat": "{{ http_target }}",
"expr": "sum(rate(http_server_duration_milliseconds_count{job=\"realestate-crawler-api\"}[5m])) by (http_server_name)",
"legendFormat": "{{ http_server_name }}",
"range": true,
"refId": "A"
}
@ -2062,8 +2062,8 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(http_server_active_requests{job=\"realestate-crawler-api\"}) by (http_target)",
"legendFormat": "{{ http_target }}",
"expr": "sum(http_server_active_requests{job=\"realestate-crawler-api\"}) by (http_server_name)",
"legendFormat": "{{ http_server_name }}",
"range": true,
"refId": "A"
}
@ -2583,6 +2583,954 @@
],
"title": "Feature Count Over Time",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 69 },
"id": 200,
"panels": [],
"title": "Database Query Performance",
"type": "row"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"description": "Database query duration by operation (p50/p95)",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 70 },
"id": 201,
"options": {
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "histogram_quantile(0.50, sum(rate(db_query_duration_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))",
"legendFormat": "p50 {{ operation }}",
"range": true,
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum(rate(db_query_duration_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))",
"legendFormat": "p95 {{ operation }}",
"range": true,
"refId": "B"
}
],
"title": "DB Query Duration by Operation (p50/p95)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"description": "Average query duration by operation — highlights top offenders",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 0.1 },
{ "color": "red", "value": 0.5 }
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 6, "x": 12, "y": 70 },
"id": 202,
"options": {
"displayMode": "gradient",
"maxVizHeight": 300,
"minVizHeight": 16,
"minVizWidth": 8,
"namePlacement": "auto",
"orientation": "horizontal",
"reduceOptions": { "calcs": ["mean"], "fields": "", "values": false },
"showUnfilled": true,
"sizing": "auto",
"valueMode": "color"
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "rate(db_query_duration_seconds_sum{job=\"realestate-crawler-api\"}[5m]) / rate(db_query_duration_seconds_count{job=\"realestate-crawler-api\"}[5m])",
"legendFormat": "{{ operation }}",
"range": true,
"refId": "A"
}
],
"title": "Top Offenders (Avg Duration)",
"type": "bargauge"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"description": "Rows returned per query (avg/p95)",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"min": 0,
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"unit": "short"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 6, "x": 18, "y": 70 },
"id": 203,
"options": {
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "rate(db_query_rows_returned_sum{job=\"realestate-crawler-api\"}[5m]) / rate(db_query_rows_returned_count{job=\"realestate-crawler-api\"}[5m])",
"legendFormat": "avg {{ operation }}",
"range": true,
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum(rate(db_query_rows_returned_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))",
"legendFormat": "p95 {{ operation }}",
"range": true,
"refId": "B"
}
],
"title": "Rows Returned per Query (avg/p95)",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 78 },
"id": 210,
"panels": [],
"title": "Streaming Performance",
"type": "row"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"description": "Time to first byte by stream source (cache vs db)",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 79 },
"id": 211,
"options": {
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "histogram_quantile(0.50, sum(rate(stream_time_to_first_byte_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, source))",
"legendFormat": "p50 {{ source }}",
"range": true,
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum(rate(stream_time_to_first_byte_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, source))",
"legendFormat": "p95 {{ source }}",
"range": true,
"refId": "B"
}
],
"title": "Stream TTFB by Source (p50/p95)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"description": "Total stream duration by source",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 79 },
"id": 212,
"options": {
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "histogram_quantile(0.50, sum(rate(stream_total_duration_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, source))",
"legendFormat": "p50 {{ source }}",
"range": true,
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum(rate(stream_total_duration_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, source))",
"legendFormat": "p95 {{ source }}",
"range": true,
"refId": "B"
}
],
"title": "Stream Total Duration by Source (p50/p95)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"description": "Average features per streaming response",
"fieldConfig": {
"defaults": {
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "blue", "value": null }
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 79 },
"id": 213,
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "rate(stream_features_total{job=\"realestate-crawler-api\"}[5m]) / rate(stream_requests_total{job=\"realestate-crawler-api\"}[5m])",
"legendFormat": "{{ source }}",
"range": true,
"refId": "A"
}
],
"title": "Avg Features per Stream",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"description": "Cache vs DB stream request ratio",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"unit": "short"
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 79 },
"id": 214,
"options": {
"displayLabels": ["percent"],
"legend": { "displayMode": "list", "placement": "bottom", "showLegend": true },
"pieType": "donut",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"tooltip": { "mode": "multi", "sort": "none" }
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "increase(stream_requests_total{job=\"realestate-crawler-api\"}[1h])",
"legendFormat": "{{ source }}",
"range": true,
"refId": "A"
}
],
"title": "Cache vs DB Stream Ratio",
"type": "piechart"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"description": "Total stream requests over time",
"fieldConfig": {
"defaults": {
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{ "color": "green", "value": null }]
},
"unit": "short"
},
"overrides": []
},
"gridPos": { "h": 4, "w": 8, "x": 16, "y": 83 },
"id": 215,
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "increase(stream_requests_total{job=\"realestate-crawler-api\"}[5m])",
"legendFormat": "{{ source }}",
"range": true,
"refId": "A"
}
],
"title": "Stream Requests (5m increase)",
"type": "stat"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 87 },
"id": 220,
"panels": [],
"title": "Listing Detail Breakdown",
"type": "row"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"description": "Per-step timing in listing detail endpoint (p50/p95)",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 88 },
"id": 221,
"options": {
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "histogram_quantile(0.50, sum(rate(listing_detail_step_duration_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, step))",
"legendFormat": "p50 {{ step }}",
"range": true,
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum(rate(listing_detail_step_duration_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, step))",
"legendFormat": "p95 {{ step }}",
"range": true,
"refId": "B"
}
],
"title": "Listing Detail Step Duration (p50/p95)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"description": "Average duration by step — stacked to show total breakdown",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "bars",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "normal" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 88 },
"id": 222,
"options": {
"legend": { "calcs": ["mean"], "displayMode": "table", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "rate(listing_detail_step_duration_seconds_sum{job=\"realestate-crawler-api\"}[5m]) / rate(listing_detail_step_duration_seconds_count{job=\"realestate-crawler-api\"}[5m])",
"legendFormat": "{{ step }}",
"range": true,
"refId": "A"
}
],
"title": "Listing Detail Stacked Avg by Step",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 96 },
"id": 230,
"panels": [],
"title": "Cache Performance",
"type": "row"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"description": "Redis cache operation latency by operation type (p50/p95)",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 10, "x": 0, "y": 97 },
"id": 231,
"options": {
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "histogram_quantile(0.50, sum(rate(cache_operation_duration_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))",
"legendFormat": "p50 {{ operation }}",
"range": true,
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum(rate(cache_operation_duration_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))",
"legendFormat": "p95 {{ operation }}",
"range": true,
"refId": "B"
}
],
"title": "Cache Operation Latency (p50/p95)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"description": "Cache repopulation events by result",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "bars",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "normal" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"unit": "short"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 8, "x": 10, "y": 97 },
"id": 232,
"options": {
"legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "increase(cache_repopulation_total{job=\"realestate-crawler-api\"}[1h])",
"legendFormat": "{{ result }}",
"range": true,
"refId": "A"
}
],
"title": "Cache Repopulation Events",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"description": "Number of times stale cache was served during background repopulation",
"fieldConfig": {
"defaults": {
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 10 },
{ "color": "red", "value": 50 }
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 6, "x": 18, "y": 97 },
"id": 233,
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "increase(cache_stale_serves_total{job=\"realestate-crawler-api\"}[1h])",
"legendFormat": "Stale Serves",
"range": true,
"refId": "A"
}
],
"title": "Stale Cache Serves (1h)",
"type": "stat"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 105 },
"id": 240,
"panels": [],
"title": "Frontend Navigation Metrics",
"type": "row"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"description": "Full page or filter load time (p50/p95)",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 6, "x": 0, "y": 106 },
"id": 241,
"options": {
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "histogram_quantile(0.50, sum(rate(frontend_page_load_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))",
"legendFormat": "p50 {{ operation }}",
"range": true,
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum(rate(frontend_page_load_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))",
"legendFormat": "p95 {{ operation }}",
"range": true,
"refId": "B"
}
],
"title": "Page Load Time (p50/p95)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"description": "Time to first listing batch on screen (p50/p95)",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 6, "x": 6, "y": 106 },
"id": 242,
"options": {
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "histogram_quantile(0.50, sum(rate(frontend_time_to_first_listing_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))",
"legendFormat": "p50 {{ operation }}",
"range": true,
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum(rate(frontend_time_to_first_listing_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le, operation))",
"legendFormat": "p95 {{ operation }}",
"range": true,
"refId": "B"
}
],
"title": "Time to First Listing (p50/p95)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"description": "Client-side total stream download duration (p50/p95)",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 6, "x": 12, "y": 106 },
"id": 243,
"options": {
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "histogram_quantile(0.50, sum(rate(frontend_stream_download_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le))",
"legendFormat": "p50",
"range": true,
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum(rate(frontend_stream_download_seconds_bucket{job=\"realestate-crawler-api\"}[5m])) by (le))",
"legendFormat": "p95",
"range": true,
"refId": "B"
}
],
"title": "Stream Download Duration (p50/p95)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"description": "Time from listing click to detail data rendered",
"fieldConfig": {
"defaults": {
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 1 },
{ "color": "red", "value": 3 }
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 6, "x": 18, "y": 106 },
"id": 244,
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": { "calcs": ["mean"], "fields": "", "values": false },
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"editorMode": "code",
"expr": "rate(frontend_listing_detail_load_seconds_sum{job=\"realestate-crawler-api\"}[5m]) / rate(frontend_listing_detail_load_seconds_count{job=\"realestate-crawler-api\"}[5m])",
"legendFormat": "Avg Detail Load",
"range": true,
"refId": "A"
}
],
"title": "Listing Detail Load Time",
"type": "stat"
}
],
"preload": false,

16
repositories/decision_repository.py
View file

@ -1,5 +1,7 @@
from datetime import datetime
import time
from api.metrics import record_db_query
from models.decision import ListingDecision
from sqlalchemy import Engine
from sqlmodel import Session, select
@ -19,6 +21,7 @@ class DecisionRepository:
decision: str,
) -> ListingDecision:
"""Create or update a decision. Uses dialect-specific upsert."""
t0 = time.monotonic()
with Session(self.engine) as session:
now = datetime.utcnow()
values = {
@ -58,14 +61,18 @@ class DecisionRepository:
)
).first()
assert result is not None
return result
record_db_query("upsert_decision", "decision", time.monotonic() - t0)
return result
def get_decisions_for_user(self, user_id: int) -> list[ListingDecision]:
t0 = time.monotonic()
with Session(self.engine) as session:
statement = select(ListingDecision).where(
ListingDecision.user_id == user_id
)
return list(session.exec(statement).all())
results = list(session.exec(statement).all())
record_db_query("get_decisions_for_user", "decision", time.monotonic() - t0, len(results))
return results
def delete_decision(
self,
@ -92,13 +99,16 @@ class DecisionRepository:
user_id: int,
listing_type: str,
) -> set[int]:
t0 = time.monotonic()
with Session(self.engine) as session:
statement = select(ListingDecision.listing_id).where(
ListingDecision.user_id == user_id,
ListingDecision.listing_type == listing_type,
ListingDecision.decision == "disliked",
)
return {row for row in session.exec(statement).all()}
ids = {row for row in session.exec(statement).all()}
record_db_query("get_disliked_listing_ids", "decision", time.monotonic() - t0, len(ids))
return ids
def get_liked_listing_ids(
self,

18
repositories/listing_repository.py
View file

@ -1,6 +1,8 @@
from datetime import datetime, timedelta
import logging
import time
from typing import Any, Generator
from api.metrics import record_db_query
from data_access import Listing
from models.listing import (
BuyListing,
@ -55,8 +57,10 @@ class ListingRepository:
if limit:
query = query.limit(limit)
t0 = time.monotonic()
with Session(self.engine) as session:
rows = list(session.exec(query).all())
record_db_query("get_listings", model.__tablename__, time.monotonic() - t0, len(rows))
logging.debug(f"Found {len(rows)} listings")
return rows
@ -110,8 +114,11 @@ class ListingRepository:
query = sa_select(func.count(model.id))
query = self._apply_query_filters(query, model, query_parameters)
t0 = time.monotonic()
with Session(self.engine) as session:
return session.execute(query).scalar() or 0
result = session.execute(query).scalar() or 0
record_db_query("count_listings", model.__tablename__, time.monotonic() - t0, result)
return result
def stream_listings_optimized(
self,
@ -157,8 +164,10 @@ class ListingRepository:
batch_limit = min(page_size, limit - total_yielded)
query = query.order_by(model.id).limit(batch_limit)
t0 = time.monotonic()
with Session(self.engine) as session:
results = session.execute(query).fetchall()
record_db_query("stream_listings_page", model.__tablename__, time.monotonic() - t0, len(results))
if not results:
break
@ -364,6 +373,7 @@ class ListingRepository:
Checks both RentListing and BuyListing tables and returns the latest.
"""
t0 = time.monotonic()
with Session(self.engine) as session:
rent_max = session.execute(
sa_select(func.max(RentListing.last_seen))
@ -371,6 +381,7 @@ class ListingRepository:
buy_max = session.execute(
sa_select(func.max(BuyListing.last_seen))
).scalar()
record_db_query("get_last_updated", "rent", time.monotonic() - t0)
candidates = [t for t in (rent_max, buy_max) if t is not None]
return max(candidates) if candidates else None
@ -385,9 +396,12 @@ class ListingRepository:
filtering against API results.
"""
model = RentListing if listing_type == ListingType.RENT else BuyListing
t0 = time.monotonic()
with Session(self.engine) as session:
result = session.execute(sa_select(model.id))
return {row[0] for row in result.fetchall()}
ids = {row[0] for row in result.fetchall()}
record_db_query("get_listing_ids", model.__tablename__, time.monotonic() - t0, len(ids))
return ids
async def mark_seen(self, listing_id: int, listing_type: ListingType = ListingType.RENT) -> None:
query_params = QueryParameters(listing_type=listing_type)

14
repositories/poi_repository.py
View file

@ -1,3 +1,5 @@
import time
from api.metrics import record_db_query
from models.listing import ListingType
from models.poi import PointOfInterest
from models.poi_distance import POIDistance
@ -12,11 +14,14 @@ class POIRepository:
self.engine = engine
def get_pois_for_user(self, user_id: int) -> list[PointOfInterest]:
t0 = time.monotonic()
with Session(self.engine) as session:
statement = select(PointOfInterest).where(
PointOfInterest.user_id == user_id
)
return list(session.exec(statement).all())
results = list(session.exec(statement).all())
record_db_query("get_pois_for_user", "poi", time.monotonic() - t0, len(results))
return results
def get_poi_by_id(self, poi_id: int) -> PointOfInterest | None:
with Session(self.engine) as session:
@ -55,6 +60,7 @@ class POIRepository:
"""Insert or update POI distances, handling duplicate unique constraints."""
if not distances:
return
t0 = time.monotonic()
with Session(self.engine) as session:
dialect = self.engine.dialect.name
for dist in distances:
@ -88,6 +94,7 @@ class POIRepository:
)
session.execute(stmt)
session.commit()
record_db_query("upsert_distances", "poi_distance", time.monotonic() - t0, len(distances))
def get_distances_for_listings(
self,
@ -95,6 +102,7 @@ class POIRepository:
listing_type: ListingType,
user_id: int,
) -> list[POIDistance]:
t0 = time.monotonic()
with Session(self.engine) as session:
# Join with POI to filter by user
statement = (
@ -106,7 +114,9 @@ class POIRepository:
PointOfInterest.user_id == user_id,
)
)
return list(session.exec(statement).all())
results = list(session.exec(statement).all())
record_db_query("get_distances_for_listings", "poi_distance", time.monotonic() - t0, len(results))
return results
def get_distances_for_poi(self, poi_id: int) -> list[POIDistance]:
with Session(self.engine) as session:

21
services/listing_cache.py
View file

@ -3,16 +3,25 @@ import hashlib
import json
import logging
import os
import time
import uuid
from typing import Generator
from urllib.parse import urlparse, urlunparse
import redis
import api.metrics as app_metrics
from models.listing import QueryParameters
logger = logging.getLogger(__name__)
def _record_cache_op(operation: str, duration: float) -> None:
"""Record a cache operation timing metric, no-op if metrics aren't initialized."""
if app_metrics._meter is None:
return
app_metrics.cache_operation_duration_seconds.record(duration, {"operation": operation})
CACHE_PREFIX = "listings:geojson:"
STAGING_PREFIX = "listings:geojson:staging:"
CACHE_TTL_SECONDS = 24 * 60 * 60 # 24 hours
@ -40,11 +49,15 @@ def make_cache_key(query_params: QueryParameters) -> str:
def get_cached_count(query_params: QueryParameters) -> int | None:
"""Return the number of cached features for a query, or None if not cached."""
try:
t0 = time.monotonic()
client = _get_redis_client()
key = make_cache_key(query_params)
if not client.exists(key):
_record_cache_op("check", time.monotonic() - t0)
return None
return client.llen(key)
count = client.llen(key)
_record_cache_op("check", time.monotonic() - t0)
return count
except redis.RedisError as e:
logger.warning(f"Redis cache read error: {e}")
return None
@ -61,7 +74,9 @@ def get_cached_features(
for start in range(0, total, batch_size):
end = start + batch_size - 1
t0 = time.monotonic()
items = client.lrange(key, start, end)
_record_cache_op("read_batch", time.monotonic() - t0)
batch = [json.loads(item) for item in items]
if batch:
yield batch
@ -100,12 +115,14 @@ def cache_features_batch_staged(staging_key: str, features: list[dict]) -> None:
if not features:
return
try:
t0 = time.monotonic()
client = _get_redis_client()
pipeline = client.pipeline()
for feature in features:
pipeline.rpush(staging_key, json.dumps(feature))
pipeline.expire(staging_key, STAGING_TTL_SECONDS)
pipeline.execute()
_record_cache_op("write_batch", time.monotonic() - t0)
except redis.RedisError as e:
logger.warning(f"Redis staged cache write error: {e}")
@ -113,11 +130,13 @@ def cache_features_batch_staged(staging_key: str, features: list[dict]) -> None:
def finalize_cache_population(staging_key: str, query_params: QueryParameters) -> None:
"""Atomically rename the staging key to the live cache key and set TTL."""
try:
t0 = time.monotonic()
client = _get_redis_client()
live_key = make_cache_key(query_params)
# RENAME is atomic — replaces the live key in one operation
client.rename(staging_key, live_key)
client.expire(live_key, CACHE_TTL_SECONDS)
_record_cache_op("finalize", time.monotonic() - t0)
logger.debug(f"Finalized cache population for {live_key}")
except redis.RedisError as e:
logger.warning(f"Redis cache finalize error: {e}")