From a3bcb5e12f5dadabda3678cc22ff528409f7729c Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Fri, 22 May 2026 14:15:38 +0000 Subject: [PATCH] fire-planner: COL refresh CronJob + Grafana Cost-of-Living dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operational layer for the new col_snapshot cache shipped in fire-planner@e72fd22: stacks/fire-planner: - fire-planner-col-refresh CronJob — Sun 04:00 UTC, no-op until rows age toward the 1-year TTL boundary (within 7 days). Calls python -m fire_planner col-refresh-stale, upserts via cache.upsert. monitoring/dashboards/cost-of-living.json (Finance folder): - Two template variables: $city (single-select from col_snapshot), $baseline_city (for COL ratio computation, defaults London). - Stat row: total w/rent, w/o rent, 1-bed rent, ratio (color-coded). - All-cities ranked table with gradient-gauged total + colored ratio. - Cache-freshness table flags rows approaching TTL expiry. Initial population needs a one-shot: post-Keel-rollout, kubectl -n fire-planner exec deploy/fire-planner -- \\ python -m fire_planner col-seed Co-Authored-By: Claude Opus 4.7 --- stacks/fire-planner/main.tf | 71 +++++ .../monitoring/dashboards/cost-of-living.json | 271 ++++++++++++++++++ .../monitoring/modules/monitoring/grafana.tf | 1 + 3 files changed, 343 insertions(+) create mode 100644 stacks/monitoring/modules/monitoring/dashboards/cost-of-living.json diff --git a/stacks/fire-planner/main.tf b/stacks/fire-planner/main.tf index 72d1da06..6e9e04a3 100644 --- a/stacks/fire-planner/main.tf +++ b/stacks/fire-planner/main.tf @@ -428,6 +428,77 @@ resource "kubernetes_cron_job_v1" "fire_planner_recompute" { ] } +# Weekly refresh of the COL cache: walks col_snapshot for rows +# expiring within 7 days, re-scrapes Numbeo + Expatistan, upserts. With +# the user-chosen 1-year TTL, a healthy cache has 0 stale rows on most +# Sundays — the job is a no-op until rows age out. Schedule Sunday 04:00 +# UTC so Numbeo's contributor activity (mostly weekday) doesn't race +# our reads. +resource "kubernetes_cron_job_v1" "fire_planner_col_refresh" { + metadata { + name = "fire-planner-col-refresh" + namespace = kubernetes_namespace.fire_planner.metadata[0].name + } + spec { + schedule = "0 4 * * 0" + concurrency_policy = "Forbid" + successful_jobs_history_limit = 3 + failed_jobs_history_limit = 5 + starting_deadline_seconds = 600 + + job_template { + metadata { + labels = local.labels + } + spec { + backoff_limit = 1 + ttl_seconds_after_finished = 86400 + template { + metadata { + labels = local.labels + } + spec { + restart_policy = "OnFailure" + image_pull_secrets { + name = "registry-credentials" + } + container { + name = "col-refresh" + image = local.image + command = ["python", "-m", "fire_planner", "col-refresh-stale", "--within-days", "7"] + + env_from { + secret_ref { + name = "fire-planner-db-creds" + } + } + + resources { + requests = { + cpu = "100m" + memory = "256Mi" + } + limits = { + memory = "512Mi" + } + } + } + } + } + } + } + } + + lifecycle { + # KYVERNO_LIFECYCLE_V1 + ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config] + } + + depends_on = [ + kubernetes_manifest.db_external_secret, + ] +} + # Public ingress at fire-planner.viktorbarzin.me. Authentik-protected # (forward-auth at the Traefik layer); Cloudflare-proxied for CDN + # DDoS shielding. Backend FastAPI serves the SPA at / and the API diff --git a/stacks/monitoring/modules/monitoring/dashboards/cost-of-living.json b/stacks/monitoring/modules/monitoring/dashboards/cost-of-living.json new file mode 100644 index 00000000..a45efe81 --- /dev/null +++ b/stacks/monitoring/modules/monitoring/dashboards/cost-of-living.json @@ -0,0 +1,271 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": {"type": "datasource", "uid": "grafana"}, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Cost-of-living per city — Numbeo + Expatistan snapshots cached in fire_planner.col_snapshot (1-year TTL). Powers the FIRE simulator's auto-COL adjustment. Use the city dropdown to drill into one city; the bottom panels rank all cities.", + "editable": true, + "fiscalYearStartMonth": 0, + "id": null, + "templating": { + "list": [ + { + "name": "city", + "type": "query", + "label": "City", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "query": "SELECT DISTINCT city_slug AS __value, city_display AS __text FROM fire_planner.col_snapshot ORDER BY city_display", + "refresh": 1, + "includeAll": false, + "multi": false, + "current": {"selected": false, "text": "Sofia", "value": "sofia"} + }, + { + "name": "baseline_city", + "type": "query", + "label": "Baseline city (for ratio)", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "query": "SELECT DISTINCT city_slug AS __value, city_display AS __text FROM fire_planner.col_snapshot ORDER BY city_display", + "refresh": 1, + "includeAll": false, + "multi": false, + "current": {"selected": false, "text": "London", "value": "london"} + } + ] + }, + "links": [], + "panels": [ + { + "id": 1, + "title": "$city — Total monthly cost (with rent, single person)", + "type": "stat", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "gridPos": {"h": 5, "w": 6, "x": 0, "y": 0}, + "fieldConfig": { + "defaults": { + "unit": "currencyGBP", + "decimals": 0, + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 1500}, + {"color": "orange", "value": 2500}, + {"color": "red", "value": 3500} + ] + } + }, + "overrides": [] + }, + "options": {"colorMode": "background", "graphMode": "none", "textMode": "value_and_name", "reduceOptions": {"calcs": ["lastNotNull"]}}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "table", + "rawSql": "SELECT total_with_rent_gbp FROM fire_planner.col_snapshot WHERE city_slug = '$city' ORDER BY fetched_at DESC LIMIT 1" + } + ] + }, + { + "id": 2, + "title": "$city — Without rent", + "type": "stat", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "gridPos": {"h": 5, "w": 6, "x": 6, "y": 0}, + "fieldConfig": {"defaults": {"unit": "currencyGBP", "decimals": 0}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"]}}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "table", + "rawSql": "SELECT total_no_rent_gbp FROM fire_planner.col_snapshot WHERE city_slug = '$city' ORDER BY fetched_at DESC LIMIT 1" + } + ] + }, + { + "id": 3, + "title": "$city — 1-bed rent (center)", + "type": "stat", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "gridPos": {"h": 5, "w": 6, "x": 12, "y": 0}, + "fieldConfig": {"defaults": {"unit": "currencyGBP", "decimals": 0}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"]}}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "table", + "rawSql": "SELECT rent_1bed_center_gbp FROM fire_planner.col_snapshot WHERE city_slug = '$city' ORDER BY fetched_at DESC LIMIT 1" + } + ] + }, + { + "id": 4, + "title": "COL ratio vs $baseline_city", + "type": "stat", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "description": "Multiplier the simulator applies to spending_gbp when moving from $baseline_city to $city. 0.5x = half-the-price; 1.0x = same; 2.0x = double.", + "gridPos": {"h": 5, "w": 6, "x": 18, "y": 0}, + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 2, + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 0.6}, + {"color": "orange", "value": 0.85}, + {"color": "red", "value": 1.0} + ] + } + }, + "overrides": [] + }, + "options": {"colorMode": "background", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"]}}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "table", + "rawSql": "SELECT (c.total_with_rent_gbp / b.total_with_rent_gbp)::numeric(8,3) AS ratio FROM (SELECT total_with_rent_gbp FROM fire_planner.col_snapshot WHERE city_slug = '$city' ORDER BY fetched_at DESC LIMIT 1) c, (SELECT total_with_rent_gbp FROM fire_planner.col_snapshot WHERE city_slug = '$baseline_city' ORDER BY fetched_at DESC LIMIT 1) b" + } + ] + }, + { + "id": 5, + "title": "$city — Snapshot metadata", + "type": "table", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "description": "Provenance for the current city. Snapshot date is when the source page was crawled; expires_at is when the cache row will be re-scraped (1-year TTL).", + "gridPos": {"h": 6, "w": 12, "x": 0, "y": 5}, + "fieldConfig": { + "defaults": {"custom": {"align": "left"}}, + "overrides": [ + { + "matcher": {"id": "byName", "options": "source_url"}, + "properties": [{"id": "links", "value": [{"title": "Open source on $1", "url": "${__value.text}", "targetBlank": true}]}] + } + ] + }, + "options": {"showHeader": true}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "table", + "rawSql": "SELECT city_display, country, source_name, source_url, snapshot_date, fetched_at, expires_at, raw_currency, gbp_per_unit FROM fire_planner.col_snapshot WHERE city_slug = '$city' ORDER BY fetched_at DESC LIMIT 1" + } + ] + }, + { + "id": 6, + "title": "$city — Cost breakdown (single person, monthly)", + "type": "barchart", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "description": "Categories from baseline.py (Phase 1) — 0 for cities where the live scraper hasn't populated per-category yet. Rent and headline totals are always populated.", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 5}, + "fieldConfig": {"defaults": {"unit": "currencyGBP", "decimals": 0}, "overrides": []}, + "options": {"orientation": "horizontal", "showValue": "auto", "stacking": "none", "legend": {"displayMode": "list"}, "xTickLabelRotation": 0}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "table", + "rawSql": "SELECT 'rent_center' AS category, rent_1bed_center_gbp AS amount_gbp FROM fire_planner.col_snapshot WHERE city_slug = '$city' ORDER BY fetched_at DESC LIMIT 1 UNION ALL SELECT 'rent_outside', rent_1bed_outside_gbp FROM fire_planner.col_snapshot WHERE city_slug = '$city' AND rent_1bed_outside_gbp IS NOT NULL ORDER BY fetched_at DESC LIMIT 1 UNION ALL SELECT 'all_other', total_no_rent_gbp FROM fire_planner.col_snapshot WHERE city_slug = '$city' ORDER BY fetched_at DESC LIMIT 1" + } + ] + }, + { + "id": 7, + "title": "All cities — ranked by Total monthly (with rent)", + "type": "table", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "description": "Sortable table of every city in the cache. Click a column header to sort. Lower = cheaper.", + "gridPos": {"h": 16, "w": 24, "x": 0, "y": 13}, + "fieldConfig": { + "defaults": { + "custom": {"align": "left", "displayMode": "auto"}, + "decimals": 0 + }, + "overrides": [ + {"matcher": {"id": "byName", "options": "total_with_rent_gbp"}, "properties": [{"id": "unit", "value": "currencyGBP"}, {"id": "custom.displayMode", "value": "gradient-gauge"}, {"id": "color", "value": {"mode": "thresholds"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 1500}, {"color": "orange", "value": 2500}, {"color": "red", "value": 3500}]}}]}, + {"matcher": {"id": "byName", "options": "total_no_rent_gbp"}, "properties": [{"id": "unit", "value": "currencyGBP"}]}, + {"matcher": {"id": "byName", "options": "rent_1bed_center_gbp"}, "properties": [{"id": "unit", "value": "currencyGBP"}]}, + {"matcher": {"id": "byName", "options": "ratio_vs_baseline"}, "properties": [{"id": "decimals", "value": 2}, {"id": "custom.displayMode", "value": "color-text"}, {"id": "color", "value": {"mode": "thresholds"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 0.5}, {"color": "orange", "value": 0.85}, {"color": "red", "value": 1.0}]}}]}, + {"matcher": {"id": "byName", "options": "source_url"}, "properties": [{"id": "links", "value": [{"title": "Open source", "url": "${__value.text}", "targetBlank": true}]}]} + ] + }, + "options": {"showHeader": true, "sortBy": [{"displayName": "total_with_rent_gbp", "desc": false}]}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "table", + "rawSql": "WITH latest AS (SELECT DISTINCT ON (city_slug) city_slug, city_display, country, total_no_rent_gbp, total_with_rent_gbp, rent_1bed_center_gbp, source_name, source_url, snapshot_date, fetched_at FROM fire_planner.col_snapshot ORDER BY city_slug, fetched_at DESC), baseline AS (SELECT total_with_rent_gbp AS b FROM fire_planner.col_snapshot WHERE city_slug = '$baseline_city' ORDER BY fetched_at DESC LIMIT 1) SELECT l.city_display, l.country, l.total_with_rent_gbp, l.total_no_rent_gbp, l.rent_1bed_center_gbp, (l.total_with_rent_gbp / b.b)::numeric(8,3) AS ratio_vs_baseline, l.source_name, l.snapshot_date, l.source_url FROM latest l CROSS JOIN baseline b ORDER BY l.total_with_rent_gbp ASC" + } + ] + }, + { + "id": 8, + "title": "Cache freshness — fetched_at across all cities", + "type": "table", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "description": "When each city was last refreshed. Rows turning red are within 30 days of expiry (1-year TTL) — the refresh CronJob should pick them up before they go stale.", + "gridPos": {"h": 10, "w": 24, "x": 0, "y": 29}, + "fieldConfig": { + "defaults": {"custom": {"align": "left"}}, + "overrides": [ + {"matcher": {"id": "byName", "options": "expires_in_days"}, "properties": [{"id": "unit", "value": "d"}, {"id": "decimals", "value": 0}, {"id": "custom.displayMode", "value": "color-background-solid"}, {"id": "color", "value": {"mode": "thresholds"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "orange", "value": 30}, {"color": "yellow", "value": 90}, {"color": "green", "value": 180}]}}]} + ] + }, + "options": {"showHeader": true, "sortBy": [{"displayName": "expires_in_days", "desc": false}]}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "table", + "rawSql": "SELECT DISTINCT ON (city_slug) city_display, country, source_name, snapshot_date, fetched_at, expires_at, EXTRACT(DAY FROM expires_at - NOW())::int AS expires_in_days FROM fire_planner.col_snapshot ORDER BY city_slug, fetched_at DESC" + } + ] + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": ["fire-planner", "col", "cost-of-living"], + "time": {"from": "now-30d", "to": "now"}, + "timepicker": {}, + "timezone": "Europe/London", + "title": "Cost of Living", + "uid": "fire-col", + "version": 1 +} diff --git a/stacks/monitoring/modules/monitoring/grafana.tf b/stacks/monitoring/modules/monitoring/grafana.tf index 2e4fa74e..5df70818 100644 --- a/stacks/monitoring/modules/monitoring/grafana.tf +++ b/stacks/monitoring/modules/monitoring/grafana.tf @@ -139,6 +139,7 @@ locals { "wealth.json" = "Finance (Personal)" "job-hunter.json" = "Finance" "fire-planner.json" = "Finance" + "cost-of-living.json" = "Finance" } # Folders restricted to the Grafana admin user (anonymous Viewer + any future