wealth: SQLite→PG ETL sidecar + new Grafana dashboard
Mirrors Wealthfolio's daily_account_valuation / accounts / activities from SQLite into a new PG database (wealthfolio_sync) every hour, so Grafana can chart net worth, contributions, and growth over time. Components: - dbaas: null_resource creates wealthfolio_sync DB + role on the CNPG cluster (dynamic primary lookup so it survives failover). - vault: pg-wealthfolio-sync static role rotates the password every 7d. - wealthfolio: ExternalSecret pulls the rotated password into the WF namespace; new pg-sync sidecar (alpine + sqlite + postgresql-client + busybox crond) does sqlite3 .backup → TSV dump → truncate-and-reload psql, hourly at :07. Plus a grafana-wealth-datasource ConfigMap in the monitoring namespace (uid: wealth-pg). - monitoring: new Wealth dashboard (wealth.json, 10 panels) — current net worth / contribution / growth / ROI% stats, then time-series for net worth, contribution-vs-market, growth area, per-account stacked area, cash-vs-invested, and a 100-row activity log. Initial sync: 6 accounts, 10,798 daily valuations, 518 activities. Verified PG totals match SQLite latest snapshot exactly.
This commit is contained in:
Viktor Barzin
parent
7dd580972a
commit
bf4c7618d8
6 changed files with 762 additions and 1 deletions
|
|
@ -1209,6 +1209,33 @@ resource "null_resource" "pg_job_hunter_db" {
|
|||
}
|
||||
}
|
||||
|
||||
# Create wealthfolio_sync database for the SQLite→PG ETL sidecar that mirrors
|
||||
# Wealthfolio's daily_account_valuation/accounts/activities into PG so Grafana
|
||||
# can chart net worth, contributions, and growth.
|
||||
# Role password is managed by Vault Database Secrets Engine (static role `pg-wealthfolio-sync`, 7d rotation).
|
||||
resource "null_resource" "pg_wealthfolio_sync_db" {
|
||||
depends_on = [null_resource.pg_cluster]
|
||||
|
||||
triggers = {
|
||||
db_name = "wealthfolio_sync"
|
||||
username = "wealthfolio_sync"
|
||||
}
|
||||
|
||||
provisioner "local-exec" {
|
||||
command = <<-EOT
|
||||
PRIMARY=$(kubectl --kubeconfig ${var.kube_config_path} get cluster -n dbaas pg-cluster -o jsonpath='{.status.currentPrimary}')
|
||||
kubectl --kubeconfig ${var.kube_config_path} exec -n dbaas $PRIMARY -c postgres -- \
|
||||
bash -c '
|
||||
psql -U postgres -tc "SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = '"'"'wealthfolio_sync'"'"'" | grep -q 1 || \
|
||||
psql -U postgres -c "CREATE ROLE wealthfolio_sync WITH LOGIN PASSWORD '"'"'changeme-vault-will-rotate'"'"'"
|
||||
psql -U postgres -tc "SELECT 1 FROM pg_catalog.pg_database WHERE datname = '"'"'wealthfolio_sync'"'"'" | grep -q 1 || \
|
||||
psql -U postgres -c "CREATE DATABASE wealthfolio_sync OWNER wealthfolio_sync"
|
||||
psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE wealthfolio_sync TO wealthfolio_sync"
|
||||
'
|
||||
EOT
|
||||
}
|
||||
}
|
||||
|
||||
# Old PostgreSQL deployment — kept commented for rollback reference
|
||||
# resource "kubernetes_deployment" "postgres" {
|
||||
# metadata {
|
||||
|
|
|
|||
451
stacks/monitoring/modules/monitoring/dashboards/wealth.json
Normal file
451
stacks/monitoring/modules/monitoring/dashboards/wealth.json
Normal file
|
|
@ -0,0 +1,451 @@
|
|||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {"type": "datasource", "uid": "grafana"},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "Wealth — net worth, contributions, and growth over time. Backed by the wealthfolio_sync PG mirror of Wealthfolio's SQLite, refreshed hourly by the pg-sync sidecar.",
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Net worth (current)",
|
||||
"type": "stat",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "currencyGBP",
|
||||
"color": {"mode": "fixed", "fixedColor": "green"},
|
||||
"decimals": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "center",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "table",
|
||||
"rawSql": "SELECT SUM(total_value) AS net_worth FROM daily_account_valuation WHERE valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation)"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Net contribution (cumulative)",
|
||||
"description": "Total deposits minus withdrawals across all accounts.",
|
||||
"type": "stat",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "currencyGBP",
|
||||
"color": {"mode": "fixed", "fixedColor": "blue"},
|
||||
"decimals": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "center",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "table",
|
||||
"rawSql": "SELECT SUM(net_contribution) AS contribution FROM daily_account_valuation WHERE valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation)"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Growth (unrealised)",
|
||||
"description": "Net worth minus net contribution — the gain on everything you've put in.",
|
||||
"type": "stat",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "currencyGBP",
|
||||
"color": {"mode": "thresholds"},
|
||||
"decimals": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "red", "value": null},
|
||||
{"color": "green", "value": 0}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "center",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "table",
|
||||
"rawSql": "SELECT (SUM(total_value) - SUM(net_contribution)) AS growth FROM daily_account_valuation WHERE valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation)"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "ROI %",
|
||||
"description": "Growth / net contribution × 100. Excludes accounts with zero/negative contribution (Schwab) to avoid distortion.",
|
||||
"type": "stat",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"color": {"mode": "thresholds"},
|
||||
"decimals": 1,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "red", "value": null},
|
||||
{"color": "yellow", "value": 0},
|
||||
{"color": "green", "value": 5}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "center",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "table",
|
||||
"rawSql": "WITH latest AS (SELECT * FROM daily_account_valuation WHERE valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation) AND net_contribution > 0) SELECT (SUM(total_value - net_contribution) / NULLIF(SUM(net_contribution), 0) * 100) AS roi_pct FROM latest"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Net worth — total over time",
|
||||
"description": "Daily total_value summed across all accounts (base GBP).",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"gridPos": {"h": 10, "w": 24, "x": 0, "y": 4},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "fixed", "fixedColor": "green"},
|
||||
"unit": "currencyGBP",
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 20,
|
||||
"pointSize": 4,
|
||||
"showPoints": "never",
|
||||
"spanNulls": true,
|
||||
"axisPlacement": "auto",
|
||||
"stacking": {"group": "A", "mode": "none"}
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "net_worth"},
|
||||
"properties": [{"id": "displayName", "value": "Net worth"}]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": {"calcs": ["last", "max"], "displayMode": "table", "placement": "bottom"},
|
||||
"tooltip": {"mode": "multi", "sort": "desc"}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "time_series",
|
||||
"rawSql": "SELECT valuation_date::timestamp AS \"time\", SUM(total_value) AS net_worth FROM daily_account_valuation WHERE $__timeFilter(valuation_date) GROUP BY valuation_date ORDER BY valuation_date"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Net contribution vs market value",
|
||||
"description": "Net contribution = cumulative deposits − withdrawals. Market value = total_value (cash + investments). Gap between the two = unrealised growth.",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"gridPos": {"h": 10, "w": 12, "x": 0, "y": 14},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"unit": "currencyGBP",
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 0,
|
||||
"pointSize": 4,
|
||||
"showPoints": "never",
|
||||
"spanNulls": true,
|
||||
"axisPlacement": "auto",
|
||||
"stacking": {"group": "A", "mode": "none"}
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "market_value"},
|
||||
"properties": [
|
||||
{"id": "color", "value": {"mode": "fixed", "fixedColor": "green"}},
|
||||
{"id": "displayName", "value": "Market value"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "net_contribution"},
|
||||
"properties": [
|
||||
{"id": "color", "value": {"mode": "fixed", "fixedColor": "blue"}},
|
||||
{"id": "displayName", "value": "Net contribution"}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": {"calcs": ["last"], "displayMode": "table", "placement": "bottom"},
|
||||
"tooltip": {"mode": "multi", "sort": "desc"}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "time_series",
|
||||
"rawSql": "SELECT valuation_date::timestamp AS \"time\", SUM(net_contribution) AS net_contribution, SUM(total_value) AS market_value FROM daily_account_valuation WHERE $__timeFilter(valuation_date) GROUP BY valuation_date ORDER BY valuation_date"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Growth (market value − contribution) over time",
|
||||
"description": "Unrealised gain across all accounts. Filled area to emphasise the wealth created above the contributed capital.",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"gridPos": {"h": 10, "w": 12, "x": 12, "y": 14},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "fixed", "fixedColor": "#56A64B"},
|
||||
"unit": "currencyGBP",
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 50,
|
||||
"gradientMode": "opacity",
|
||||
"pointSize": 4,
|
||||
"showPoints": "never",
|
||||
"spanNulls": true,
|
||||
"axisPlacement": "auto",
|
||||
"stacking": {"group": "A", "mode": "none"}
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "growth"},
|
||||
"properties": [{"id": "displayName", "value": "Growth"}]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": {"calcs": ["last", "max"], "displayMode": "table", "placement": "bottom"},
|
||||
"tooltip": {"mode": "multi", "sort": "desc"}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "time_series",
|
||||
"rawSql": "SELECT valuation_date::timestamp AS \"time\", (SUM(total_value) - SUM(net_contribution)) AS growth FROM daily_account_valuation WHERE $__timeFilter(valuation_date) GROUP BY valuation_date ORDER BY valuation_date"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Per-account stacked — total value",
|
||||
"description": "Stacked area showing each account's contribution to total net worth over time. Useful for spotting which account drives the trajectory.",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"gridPos": {"h": 11, "w": 24, "x": 0, "y": 24},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"unit": "currencyGBP",
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 70,
|
||||
"pointSize": 3,
|
||||
"showPoints": "never",
|
||||
"spanNulls": true,
|
||||
"axisPlacement": "auto",
|
||||
"stacking": {"group": "A", "mode": "normal"}
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {"calcs": ["last"], "displayMode": "table", "placement": "bottom"},
|
||||
"tooltip": {"mode": "multi", "sort": "desc"}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "time_series",
|
||||
"rawSql": "SELECT d.valuation_date::timestamp AS \"time\", a.name AS metric, d.total_value AS value FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id WHERE $__timeFilter(d.valuation_date) ORDER BY d.valuation_date, a.name"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Cash vs invested (stacked)",
|
||||
"description": "Daily breakdown of cash holdings vs market value of investments, summed across all accounts.",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"gridPos": {"h": 10, "w": 24, "x": 0, "y": 35},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"unit": "currencyGBP",
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 70,
|
||||
"pointSize": 3,
|
||||
"showPoints": "never",
|
||||
"spanNulls": true,
|
||||
"axisPlacement": "auto",
|
||||
"stacking": {"group": "A", "mode": "normal"}
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "cash"},
|
||||
"properties": [
|
||||
{"id": "color", "value": {"mode": "fixed", "fixedColor": "#FADE2A"}},
|
||||
{"id": "displayName", "value": "Cash"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "invested"},
|
||||
"properties": [
|
||||
{"id": "color", "value": {"mode": "fixed", "fixedColor": "#56A64B"}},
|
||||
{"id": "displayName", "value": "Invested"}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": {"calcs": ["last"], "displayMode": "table", "placement": "bottom"},
|
||||
"tooltip": {"mode": "multi", "sort": "desc"}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "time_series",
|
||||
"rawSql": "SELECT valuation_date::timestamp AS \"time\", SUM(cash_balance) AS cash, SUM(investment_market_value) AS invested FROM daily_account_valuation WHERE $__timeFilter(valuation_date) GROUP BY valuation_date ORDER BY valuation_date"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Activity log",
|
||||
"description": "Recent activities (BUY / SELL / DEPOSIT / WITHDRAWAL / DIVIDEND / etc.) across all accounts. Limited to 100 most recent.",
|
||||
"type": "table",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"gridPos": {"h": 14, "w": 24, "x": 0, "y": 45},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {"align": "auto", "displayMode": "auto"}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "amount"},
|
||||
"properties": [{"id": "unit", "value": "currencyGBP"}]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"cellHeight": "sm",
|
||||
"footer": {"show": false}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"},
|
||||
"rawQuery": true,
|
||||
"editorMode": "code",
|
||||
"format": "table",
|
||||
"rawSql": "SELECT a.activity_date AS \"date\", acc.name AS \"account\", a.activity_type AS \"type\", a.asset_id AS \"asset\", a.quantity AS \"qty\", a.unit_price AS \"unit_price\", a.amount AS \"amount\", a.currency AS \"ccy\", a.notes AS \"notes\" FROM activities a LEFT JOIN accounts acc ON acc.id = a.account_id WHERE $__timeFilter(a.activity_date) ORDER BY a.activity_date DESC LIMIT 100"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"refresh": "5m",
|
||||
"schemaVersion": 39,
|
||||
"tags": ["finance", "personal", "wealth"],
|
||||
"templating": {"list": []},
|
||||
"time": {"from": "now-5y", "to": "now"},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "Wealth",
|
||||
"uid": "wealth",
|
||||
"version": 1
|
||||
}
|
||||
|
|
@ -136,6 +136,7 @@ locals {
|
|||
"realestate-crawler.json" = "Applications"
|
||||
"uk-payslip.json" = "Finance"
|
||||
"job-hunter.json" = "Finance"
|
||||
"wealth.json" = "Finance"
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -554,7 +554,8 @@ resource "vault_database_secret_backend_connection" "postgresql" {
|
|||
# "pg-trading", # Commented out 2026-04-06 - trading-bot disabled
|
||||
"pg-health", "pg-linkwarden",
|
||||
"pg-affine", "pg-woodpecker", "pg-claude-memory",
|
||||
"pg-terraform-state", "pg-payslip-ingest", "pg-job-hunter"
|
||||
"pg-terraform-state", "pg-payslip-ingest", "pg-job-hunter",
|
||||
"pg-wealthfolio-sync"
|
||||
]
|
||||
|
||||
postgresql {
|
||||
|
|
@ -708,6 +709,14 @@ resource "vault_database_secret_backend_static_role" "pg_job_hunter" {
|
|||
rotation_period = 604800
|
||||
}
|
||||
|
||||
resource "vault_database_secret_backend_static_role" "pg_wealthfolio_sync" {
|
||||
backend = vault_mount.database.path
|
||||
db_name = vault_database_secret_backend_connection.postgresql.name
|
||||
name = "pg-wealthfolio-sync"
|
||||
username = "wealthfolio_sync"
|
||||
rotation_period = 604800
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# Kubernetes Secrets Engine — Dynamic K8s Credentials
|
||||
# =============================================================================
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ variable "tls_secret_name" {
|
|||
sensitive = true
|
||||
}
|
||||
variable "nfs_server" { type = string }
|
||||
variable "postgresql_host" { type = string }
|
||||
|
||||
resource "kubernetes_namespace" "wealthfolio" {
|
||||
metadata {
|
||||
|
|
@ -45,6 +46,52 @@ resource "kubernetes_manifest" "external_secret" {
|
|||
depends_on = [kubernetes_namespace.wealthfolio]
|
||||
}
|
||||
|
||||
# DB credentials for the SQLite→PG ETL sidecar. Vault DB engine static role
|
||||
# `pg-wealthfolio-sync` rotates this every 7 days; ExternalSecret refreshes
|
||||
# the K8s Secret every 15m so the sidecar always has a valid password.
|
||||
resource "kubernetes_manifest" "wealthfolio_sync_db_external_secret" {
|
||||
manifest = {
|
||||
apiVersion = "external-secrets.io/v1beta1"
|
||||
kind = "ExternalSecret"
|
||||
metadata = {
|
||||
name = "wealthfolio-sync-db-creds"
|
||||
namespace = "wealthfolio"
|
||||
}
|
||||
spec = {
|
||||
refreshInterval = "15m"
|
||||
secretStoreRef = {
|
||||
name = "vault-database"
|
||||
kind = "ClusterSecretStore"
|
||||
}
|
||||
target = {
|
||||
name = "wealthfolio-sync-db-creds"
|
||||
template = {
|
||||
metadata = {
|
||||
annotations = {
|
||||
"reloader.stakater.com/match" = "true"
|
||||
}
|
||||
}
|
||||
data = {
|
||||
PGHOST = var.postgresql_host
|
||||
PGPORT = "5432"
|
||||
PGDATABASE = "wealthfolio_sync"
|
||||
PGUSER = "wealthfolio_sync"
|
||||
PGPASSWORD = "{{ .password }}"
|
||||
}
|
||||
}
|
||||
}
|
||||
data = [{
|
||||
secretKey = "password"
|
||||
remoteRef = {
|
||||
key = "static-creds/pg-wealthfolio-sync"
|
||||
property = "password"
|
||||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
depends_on = [kubernetes_namespace.wealthfolio]
|
||||
}
|
||||
|
||||
module "tls_secret" {
|
||||
source = "../../modules/kubernetes/setup_tls_secret"
|
||||
namespace = kubernetes_namespace.wealthfolio.metadata[0].name
|
||||
|
|
@ -214,6 +261,180 @@ resource "kubernetes_deployment" "wealthfolio" {
|
|||
limits = { memory = "64Mi" }
|
||||
}
|
||||
}
|
||||
|
||||
# pg-sync sidecar — mirrors a small subset of SQLite into PG every hour
|
||||
# so Grafana can chart net worth / contributions / growth via the
|
||||
# `wealthfolio_sync` database. Mounts /data RO; writes to a tmp dir
|
||||
# for the sqlite3 .backup snapshot to avoid blocking writers. Bootstrap
|
||||
# DDL runs each iteration (CREATE TABLE IF NOT EXISTS — idempotent).
|
||||
# Truncate-and-reload pattern: tables are small (~10k DAV rows, ~500
|
||||
# activities, 6 accounts), so a full reload each hour is simpler than
|
||||
# incremental upserts and gives clean cold-start behaviour.
|
||||
container {
|
||||
name = "pg-sync"
|
||||
image = "alpine:3.20"
|
||||
env {
|
||||
name = "PGHOST"
|
||||
value_from {
|
||||
secret_key_ref {
|
||||
name = "wealthfolio-sync-db-creds"
|
||||
key = "PGHOST"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "PGPORT"
|
||||
value_from {
|
||||
secret_key_ref {
|
||||
name = "wealthfolio-sync-db-creds"
|
||||
key = "PGPORT"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "PGDATABASE"
|
||||
value_from {
|
||||
secret_key_ref {
|
||||
name = "wealthfolio-sync-db-creds"
|
||||
key = "PGDATABASE"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "PGUSER"
|
||||
value_from {
|
||||
secret_key_ref {
|
||||
name = "wealthfolio-sync-db-creds"
|
||||
key = "PGUSER"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "PGPASSWORD"
|
||||
value_from {
|
||||
secret_key_ref {
|
||||
name = "wealthfolio-sync-db-creds"
|
||||
key = "PGPASSWORD"
|
||||
}
|
||||
}
|
||||
}
|
||||
command = ["/bin/sh", "-c", <<-EOT
|
||||
set -eu
|
||||
apk add --no-cache --quiet sqlite postgresql-client busybox-suid
|
||||
mkdir -p /etc/crontabs /scripts /tmp/wf-sync
|
||||
cat >/etc/crontabs/root <<'CRON'
|
||||
# Hourly: snapshot SQLite, reload PG mirror.
|
||||
7 * * * * /scripts/sync.sh >>/proc/1/fd/1 2>&1
|
||||
CRON
|
||||
cat >/scripts/sync.sh <<'SCRIPT'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
echo "[$TS] wealthfolio-pg-sync: starting"
|
||||
|
||||
# Bootstrap schema (idempotent).
|
||||
psql -v ON_ERROR_STOP=1 <<'SQL'
|
||||
CREATE TABLE IF NOT EXISTS accounts (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT,
|
||||
account_type TEXT,
|
||||
currency TEXT,
|
||||
is_active BOOLEAN
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS daily_account_valuation (
|
||||
id TEXT PRIMARY KEY,
|
||||
account_id TEXT NOT NULL,
|
||||
valuation_date DATE NOT NULL,
|
||||
account_currency TEXT,
|
||||
base_currency TEXT,
|
||||
fx_rate_to_base NUMERIC,
|
||||
cash_balance NUMERIC,
|
||||
investment_market_value NUMERIC,
|
||||
total_value NUMERIC,
|
||||
cost_basis NUMERIC,
|
||||
net_contribution NUMERIC
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_dav_acct_date ON daily_account_valuation(account_id, valuation_date);
|
||||
CREATE INDEX IF NOT EXISTS idx_dav_date ON daily_account_valuation(valuation_date);
|
||||
CREATE TABLE IF NOT EXISTS activities (
|
||||
id TEXT PRIMARY KEY,
|
||||
account_id TEXT,
|
||||
asset_id TEXT,
|
||||
activity_type TEXT,
|
||||
activity_date TIMESTAMPTZ,
|
||||
quantity NUMERIC,
|
||||
unit_price NUMERIC,
|
||||
amount NUMERIC,
|
||||
fee NUMERIC,
|
||||
currency TEXT,
|
||||
fx_rate NUMERIC,
|
||||
notes TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_act_date ON activities(activity_date);
|
||||
SQL
|
||||
|
||||
# Snapshot SQLite (online backup — non-blocking).
|
||||
rm -f /tmp/wf-sync/snapshot.db
|
||||
sqlite3 /data/wealthfolio.db ".backup /tmp/wf-sync/snapshot.db"
|
||||
|
||||
# Dump source rows to TSV.
|
||||
sqlite3 -separator $'\t' /tmp/wf-sync/snapshot.db \
|
||||
"SELECT id, name, account_type, currency, is_active FROM accounts;" \
|
||||
> /tmp/wf-sync/accounts.tsv
|
||||
|
||||
sqlite3 -separator $'\t' /tmp/wf-sync/snapshot.db <<'SQ' > /tmp/wf-sync/dav.tsv
|
||||
SELECT id, account_id, valuation_date, account_currency, base_currency,
|
||||
CAST(fx_rate_to_base AS REAL),
|
||||
CAST(cash_balance AS REAL),
|
||||
CAST(investment_market_value AS REAL),
|
||||
CAST(total_value AS REAL),
|
||||
CAST(cost_basis AS REAL),
|
||||
CAST(net_contribution AS REAL)
|
||||
FROM daily_account_valuation;
|
||||
SQ
|
||||
|
||||
sqlite3 -separator $'\t' /tmp/wf-sync/snapshot.db <<'SQ' > /tmp/wf-sync/activities.tsv
|
||||
SELECT id, account_id, asset_id, activity_type, activity_date,
|
||||
CAST(quantity AS REAL),
|
||||
CAST(unit_price AS REAL),
|
||||
CAST(amount AS REAL),
|
||||
CAST(fee AS REAL),
|
||||
currency,
|
||||
CAST(fx_rate AS REAL),
|
||||
notes
|
||||
FROM activities WHERE status='POSTED';
|
||||
SQ
|
||||
|
||||
# Truncate-and-reload (small tables; simpler than upserts).
|
||||
psql -v ON_ERROR_STOP=1 <<SQL
|
||||
BEGIN;
|
||||
TRUNCATE accounts, daily_account_valuation, activities;
|
||||
\copy accounts FROM '/tmp/wf-sync/accounts.tsv' WITH (FORMAT csv, DELIMITER E'\t', NULL '');
|
||||
\copy daily_account_valuation FROM '/tmp/wf-sync/dav.tsv' WITH (FORMAT csv, DELIMITER E'\t', NULL '');
|
||||
\copy activities FROM '/tmp/wf-sync/activities.tsv' WITH (FORMAT csv, DELIMITER E'\t', NULL '');
|
||||
COMMIT;
|
||||
SQL
|
||||
|
||||
ROWS=$(psql -tAc "SELECT COUNT(*) FROM daily_account_valuation;")
|
||||
echo "[$TS] wealthfolio-pg-sync: ok (daily_account_valuation rows=$ROWS)"
|
||||
rm -f /tmp/wf-sync/*.tsv /tmp/wf-sync/snapshot.db
|
||||
SCRIPT
|
||||
chmod +x /scripts/sync.sh
|
||||
echo "wealthfolio-pg-sync sidecar ready; running initial sync, then hourly at :07"
|
||||
/scripts/sync.sh || echo "initial sync failed (will retry on next cron tick)"
|
||||
exec crond -f -l 8
|
||||
EOT
|
||||
]
|
||||
volume_mount {
|
||||
name = "data"
|
||||
mount_path = "/data"
|
||||
read_only = true
|
||||
}
|
||||
resources {
|
||||
requests = { cpu = "10m", memory = "32Mi" }
|
||||
limits = { memory = "128Mi" }
|
||||
}
|
||||
}
|
||||
volume {
|
||||
name = "data"
|
||||
persistent_volume_claim {
|
||||
|
|
@ -374,6 +595,53 @@ resource "kubernetes_cron_job_v1" "wealthfolio_sync" {
|
|||
}
|
||||
}
|
||||
|
||||
# Plan-time read of the ESO-created K8s Secret for Grafana datasource password.
|
||||
# First apply: -target=kubernetes_manifest.wealthfolio_sync_db_external_secret first.
|
||||
data "kubernetes_secret" "wealthfolio_sync_db_creds" {
|
||||
metadata {
|
||||
name = "wealthfolio-sync-db-creds"
|
||||
namespace = kubernetes_namespace.wealthfolio.metadata[0].name
|
||||
}
|
||||
depends_on = [kubernetes_manifest.wealthfolio_sync_db_external_secret]
|
||||
}
|
||||
|
||||
# Grafana datasource for wealthfolio_sync PostgreSQL DB.
|
||||
# Lives in the monitoring namespace so the Grafana sidecar (grafana_datasource=1) picks it up.
|
||||
resource "kubernetes_config_map" "grafana_wealth_datasource" {
|
||||
metadata {
|
||||
name = "grafana-wealth-datasource"
|
||||
namespace = "monitoring"
|
||||
labels = {
|
||||
grafana_datasource = "1"
|
||||
}
|
||||
}
|
||||
data = {
|
||||
"wealth-datasource.yaml" = yamlencode({
|
||||
apiVersion = 1
|
||||
datasources = [{
|
||||
name = "Wealth"
|
||||
type = "postgres"
|
||||
access = "proxy"
|
||||
url = "${var.postgresql_host}:5432"
|
||||
user = "wealthfolio_sync"
|
||||
uid = "wealth-pg"
|
||||
# Grafana 11.2+ Postgres plugin reads DB name from jsonData.database
|
||||
# (top-level `database` is silently ignored).
|
||||
jsonData = {
|
||||
database = "wealthfolio_sync"
|
||||
sslmode = "disable"
|
||||
postgresVersion = 1600
|
||||
timescaledb = false
|
||||
}
|
||||
secureJsonData = {
|
||||
password = data.kubernetes_secret.wealthfolio_sync_db_creds.data["PGPASSWORD"]
|
||||
}
|
||||
editable = true
|
||||
}]
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
############################################################################
|
||||
# Backup — sidecar approach
|
||||
#
|
||||
|
|
|
|||
|
|
@ -11,3 +11,8 @@ dependency "vault" {
|
|||
config_path = "../vault"
|
||||
skip_outputs = true
|
||||
}
|
||||
|
||||
dependency "external-secrets" {
|
||||
config_path = "../external-secrets"
|
||||
skip_outputs = true
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue