fire-planner: COL refresh CronJob + Grafana Cost-of-Living dashboard
Operational layer for the new col_snapshot cache shipped in
fire-planner@e72fd22:
stacks/fire-planner:
- fire-planner-col-refresh CronJob — Sun 04:00 UTC, no-op until rows
age toward the 1-year TTL boundary (within 7 days). Calls
python -m fire_planner col-refresh-stale, upserts via cache.upsert.
monitoring/dashboards/cost-of-living.json (Finance folder):
- Two template variables: $city (single-select from col_snapshot),
$baseline_city (for COL ratio computation, defaults London).
- Stat row: total w/rent, w/o rent, 1-bed rent, ratio (color-coded).
- All-cities ranked table with gradient-gauged total + colored ratio.
- Cache-freshness table flags rows approaching TTL expiry.
Initial population needs a one-shot: post-Keel-rollout,
kubectl -n fire-planner exec deploy/fire-planner -- \\
python -m fire_planner col-seed
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Viktor Barzin
parent
d4c76a07a2
commit
a3bcb5e12f
3 changed files with 343 additions and 0 deletions
|
|
@ -428,6 +428,77 @@ resource "kubernetes_cron_job_v1" "fire_planner_recompute" {
|
|||
]
|
||||
}
|
||||
|
||||
# Weekly refresh of the COL cache: walks col_snapshot for rows
|
||||
# expiring within 7 days, re-scrapes Numbeo + Expatistan, upserts. With
|
||||
# the user-chosen 1-year TTL, a healthy cache has 0 stale rows on most
|
||||
# Sundays — the job is a no-op until rows age out. Schedule Sunday 04:00
|
||||
# UTC so Numbeo's contributor activity (mostly weekday) doesn't race
|
||||
# our reads.
|
||||
resource "kubernetes_cron_job_v1" "fire_planner_col_refresh" {
|
||||
metadata {
|
||||
name = "fire-planner-col-refresh"
|
||||
namespace = kubernetes_namespace.fire_planner.metadata[0].name
|
||||
}
|
||||
spec {
|
||||
schedule = "0 4 * * 0"
|
||||
concurrency_policy = "Forbid"
|
||||
successful_jobs_history_limit = 3
|
||||
failed_jobs_history_limit = 5
|
||||
starting_deadline_seconds = 600
|
||||
|
||||
job_template {
|
||||
metadata {
|
||||
labels = local.labels
|
||||
}
|
||||
spec {
|
||||
backoff_limit = 1
|
||||
ttl_seconds_after_finished = 86400
|
||||
template {
|
||||
metadata {
|
||||
labels = local.labels
|
||||
}
|
||||
spec {
|
||||
restart_policy = "OnFailure"
|
||||
image_pull_secrets {
|
||||
name = "registry-credentials"
|
||||
}
|
||||
container {
|
||||
name = "col-refresh"
|
||||
image = local.image
|
||||
command = ["python", "-m", "fire_planner", "col-refresh-stale", "--within-days", "7"]
|
||||
|
||||
env_from {
|
||||
secret_ref {
|
||||
name = "fire-planner-db-creds"
|
||||
}
|
||||
}
|
||||
|
||||
resources {
|
||||
requests = {
|
||||
cpu = "100m"
|
||||
memory = "256Mi"
|
||||
}
|
||||
limits = {
|
||||
memory = "512Mi"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
lifecycle {
|
||||
# KYVERNO_LIFECYCLE_V1
|
||||
ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config]
|
||||
}
|
||||
|
||||
depends_on = [
|
||||
kubernetes_manifest.db_external_secret,
|
||||
]
|
||||
}
|
||||
|
||||
# Public ingress at fire-planner.viktorbarzin.me. Authentik-protected
|
||||
# (forward-auth at the Traefik layer); Cloudflare-proxied for CDN +
|
||||
# DDoS shielding. Backend FastAPI serves the SPA at / and the API
|
||||
|
|
|
|||
|
|
@ -0,0 +1,271 @@
|
|||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {"type": "datasource", "uid": "grafana"},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "Cost-of-living per city — Numbeo + Expatistan snapshots cached in fire_planner.col_snapshot (1-year TTL). Powers the FIRE simulator's auto-COL adjustment. Use the city dropdown to drill into one city; the bottom panels rank all cities.",
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"id": null,
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "city",
|
||||
"type": "query",
|
||||
"label": "City",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"query": "SELECT DISTINCT city_slug AS __value, city_display AS __text FROM fire_planner.col_snapshot ORDER BY city_display",
|
||||
"refresh": 1,
|
||||
"includeAll": false,
|
||||
"multi": false,
|
||||
"current": {"selected": false, "text": "Sofia", "value": "sofia"}
|
||||
},
|
||||
{
|
||||
"name": "baseline_city",
|
||||
"type": "query",
|
||||
"label": "Baseline city (for ratio)",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"query": "SELECT DISTINCT city_slug AS __value, city_display AS __text FROM fire_planner.col_snapshot ORDER BY city_display",
|
||||
"refresh": 1,
|
||||
"includeAll": false,
|
||||
"multi": false,
|
||||
"current": {"selected": false, "text": "London", "value": "london"}
|
||||
}
|
||||
]
|
||||
},
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "$city — Total monthly cost (with rent, single person)",
|
||||
"type": "stat",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"gridPos": {"h": 5, "w": 6, "x": 0, "y": 0},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "currencyGBP",
|
||||
"decimals": 0,
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 1500},
|
||||
{"color": "orange", "value": 2500},
|
||||
{"color": "red", "value": 3500}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {"colorMode": "background", "graphMode": "none", "textMode": "value_and_name", "reduceOptions": {"calcs": ["lastNotNull"]}},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "table",
|
||||
"rawSql": "SELECT total_with_rent_gbp FROM fire_planner.col_snapshot WHERE city_slug = '$city' ORDER BY fetched_at DESC LIMIT 1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "$city — Without rent",
|
||||
"type": "stat",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"gridPos": {"h": 5, "w": 6, "x": 6, "y": 0},
|
||||
"fieldConfig": {"defaults": {"unit": "currencyGBP", "decimals": 0}, "overrides": []},
|
||||
"options": {"colorMode": "value", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"]}},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "table",
|
||||
"rawSql": "SELECT total_no_rent_gbp FROM fire_planner.col_snapshot WHERE city_slug = '$city' ORDER BY fetched_at DESC LIMIT 1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "$city — 1-bed rent (center)",
|
||||
"type": "stat",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"gridPos": {"h": 5, "w": 6, "x": 12, "y": 0},
|
||||
"fieldConfig": {"defaults": {"unit": "currencyGBP", "decimals": 0}, "overrides": []},
|
||||
"options": {"colorMode": "value", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"]}},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "table",
|
||||
"rawSql": "SELECT rent_1bed_center_gbp FROM fire_planner.col_snapshot WHERE city_slug = '$city' ORDER BY fetched_at DESC LIMIT 1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "COL ratio vs $baseline_city",
|
||||
"type": "stat",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"description": "Multiplier the simulator applies to spending_gbp when moving from $baseline_city to $city. 0.5x = half-the-price; 1.0x = same; 2.0x = double.",
|
||||
"gridPos": {"h": 5, "w": 6, "x": 18, "y": 0},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none",
|
||||
"decimals": 2,
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 0.6},
|
||||
{"color": "orange", "value": 0.85},
|
||||
{"color": "red", "value": 1.0}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {"colorMode": "background", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"]}},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "table",
|
||||
"rawSql": "SELECT (c.total_with_rent_gbp / b.total_with_rent_gbp)::numeric(8,3) AS ratio FROM (SELECT total_with_rent_gbp FROM fire_planner.col_snapshot WHERE city_slug = '$city' ORDER BY fetched_at DESC LIMIT 1) c, (SELECT total_with_rent_gbp FROM fire_planner.col_snapshot WHERE city_slug = '$baseline_city' ORDER BY fetched_at DESC LIMIT 1) b"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "$city — Snapshot metadata",
|
||||
"type": "table",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"description": "Provenance for the current city. Snapshot date is when the source page was crawled; expires_at is when the cache row will be re-scraped (1-year TTL).",
|
||||
"gridPos": {"h": 6, "w": 12, "x": 0, "y": 5},
|
||||
"fieldConfig": {
|
||||
"defaults": {"custom": {"align": "left"}},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "source_url"},
|
||||
"properties": [{"id": "links", "value": [{"title": "Open source on $1", "url": "${__value.text}", "targetBlank": true}]}]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {"showHeader": true},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "table",
|
||||
"rawSql": "SELECT city_display, country, source_name, source_url, snapshot_date, fetched_at, expires_at, raw_currency, gbp_per_unit FROM fire_planner.col_snapshot WHERE city_slug = '$city' ORDER BY fetched_at DESC LIMIT 1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "$city — Cost breakdown (single person, monthly)",
|
||||
"type": "barchart",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"description": "Categories from baseline.py (Phase 1) — 0 for cities where the live scraper hasn't populated per-category yet. Rent and headline totals are always populated.",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 5},
|
||||
"fieldConfig": {"defaults": {"unit": "currencyGBP", "decimals": 0}, "overrides": []},
|
||||
"options": {"orientation": "horizontal", "showValue": "auto", "stacking": "none", "legend": {"displayMode": "list"}, "xTickLabelRotation": 0},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "table",
|
||||
"rawSql": "SELECT 'rent_center' AS category, rent_1bed_center_gbp AS amount_gbp FROM fire_planner.col_snapshot WHERE city_slug = '$city' ORDER BY fetched_at DESC LIMIT 1 UNION ALL SELECT 'rent_outside', rent_1bed_outside_gbp FROM fire_planner.col_snapshot WHERE city_slug = '$city' AND rent_1bed_outside_gbp IS NOT NULL ORDER BY fetched_at DESC LIMIT 1 UNION ALL SELECT 'all_other', total_no_rent_gbp FROM fire_planner.col_snapshot WHERE city_slug = '$city' ORDER BY fetched_at DESC LIMIT 1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "All cities — ranked by Total monthly (with rent)",
|
||||
"type": "table",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"description": "Sortable table of every city in the cache. Click a column header to sort. Lower = cheaper.",
|
||||
"gridPos": {"h": 16, "w": 24, "x": 0, "y": 13},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {"align": "left", "displayMode": "auto"},
|
||||
"decimals": 0
|
||||
},
|
||||
"overrides": [
|
||||
{"matcher": {"id": "byName", "options": "total_with_rent_gbp"}, "properties": [{"id": "unit", "value": "currencyGBP"}, {"id": "custom.displayMode", "value": "gradient-gauge"}, {"id": "color", "value": {"mode": "thresholds"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 1500}, {"color": "orange", "value": 2500}, {"color": "red", "value": 3500}]}}]},
|
||||
{"matcher": {"id": "byName", "options": "total_no_rent_gbp"}, "properties": [{"id": "unit", "value": "currencyGBP"}]},
|
||||
{"matcher": {"id": "byName", "options": "rent_1bed_center_gbp"}, "properties": [{"id": "unit", "value": "currencyGBP"}]},
|
||||
{"matcher": {"id": "byName", "options": "ratio_vs_baseline"}, "properties": [{"id": "decimals", "value": 2}, {"id": "custom.displayMode", "value": "color-text"}, {"id": "color", "value": {"mode": "thresholds"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 0.5}, {"color": "orange", "value": 0.85}, {"color": "red", "value": 1.0}]}}]},
|
||||
{"matcher": {"id": "byName", "options": "source_url"}, "properties": [{"id": "links", "value": [{"title": "Open source", "url": "${__value.text}", "targetBlank": true}]}]}
|
||||
]
|
||||
},
|
||||
"options": {"showHeader": true, "sortBy": [{"displayName": "total_with_rent_gbp", "desc": false}]},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "table",
|
||||
"rawSql": "WITH latest AS (SELECT DISTINCT ON (city_slug) city_slug, city_display, country, total_no_rent_gbp, total_with_rent_gbp, rent_1bed_center_gbp, source_name, source_url, snapshot_date, fetched_at FROM fire_planner.col_snapshot ORDER BY city_slug, fetched_at DESC), baseline AS (SELECT total_with_rent_gbp AS b FROM fire_planner.col_snapshot WHERE city_slug = '$baseline_city' ORDER BY fetched_at DESC LIMIT 1) SELECT l.city_display, l.country, l.total_with_rent_gbp, l.total_no_rent_gbp, l.rent_1bed_center_gbp, (l.total_with_rent_gbp / b.b)::numeric(8,3) AS ratio_vs_baseline, l.source_name, l.snapshot_date, l.source_url FROM latest l CROSS JOIN baseline b ORDER BY l.total_with_rent_gbp ASC"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Cache freshness — fetched_at across all cities",
|
||||
"type": "table",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"description": "When each city was last refreshed. Rows turning red are within 30 days of expiry (1-year TTL) — the refresh CronJob should pick them up before they go stale.",
|
||||
"gridPos": {"h": 10, "w": 24, "x": 0, "y": 29},
|
||||
"fieldConfig": {
|
||||
"defaults": {"custom": {"align": "left"}},
|
||||
"overrides": [
|
||||
{"matcher": {"id": "byName", "options": "expires_in_days"}, "properties": [{"id": "unit", "value": "d"}, {"id": "decimals", "value": 0}, {"id": "custom.displayMode", "value": "color-background-solid"}, {"id": "color", "value": {"mode": "thresholds"}}, {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "orange", "value": 30}, {"color": "yellow", "value": 90}, {"color": "green", "value": 180}]}}]}
|
||||
]
|
||||
},
|
||||
"options": {"showHeader": true, "sortBy": [{"displayName": "expires_in_days", "desc": false}]},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "fire-planner-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "table",
|
||||
"rawSql": "SELECT DISTINCT ON (city_slug) city_display, country, source_name, snapshot_date, fetched_at, expires_at, EXTRACT(DAY FROM expires_at - NOW())::int AS expires_in_days FROM fire_planner.col_snapshot ORDER BY city_slug, fetched_at DESC"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"refresh": "",
|
||||
"schemaVersion": 39,
|
||||
"tags": ["fire-planner", "col", "cost-of-living"],
|
||||
"time": {"from": "now-30d", "to": "now"},
|
||||
"timepicker": {},
|
||||
"timezone": "Europe/London",
|
||||
"title": "Cost of Living",
|
||||
"uid": "fire-col",
|
||||
"version": 1
|
||||
}
|
||||
|
|
@ -139,6 +139,7 @@ locals {
|
|||
"wealth.json" = "Finance (Personal)"
|
||||
"job-hunter.json" = "Finance"
|
||||
"fire-planner.json" = "Finance"
|
||||
"cost-of-living.json" = "Finance"
|
||||
}
|
||||
|
||||
# Folders restricted to the Grafana admin user (anonymous Viewer + any future
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue