diff --git a/stacks/dbaas/modules/dbaas/main.tf b/stacks/dbaas/modules/dbaas/main.tf index d3e33617..68ea9f44 100644 --- a/stacks/dbaas/modules/dbaas/main.tf +++ b/stacks/dbaas/modules/dbaas/main.tf @@ -1209,6 +1209,33 @@ resource "null_resource" "pg_job_hunter_db" { } } +# Create wealthfolio_sync database for the SQLite→PG ETL sidecar that mirrors +# Wealthfolio's daily_account_valuation/accounts/activities into PG so Grafana +# can chart net worth, contributions, and growth. +# Role password is managed by Vault Database Secrets Engine (static role `pg-wealthfolio-sync`, 7d rotation). +resource "null_resource" "pg_wealthfolio_sync_db" { + depends_on = [null_resource.pg_cluster] + + triggers = { + db_name = "wealthfolio_sync" + username = "wealthfolio_sync" + } + + provisioner "local-exec" { + command = <<-EOT + PRIMARY=$(kubectl --kubeconfig ${var.kube_config_path} get cluster -n dbaas pg-cluster -o jsonpath='{.status.currentPrimary}') + kubectl --kubeconfig ${var.kube_config_path} exec -n dbaas $PRIMARY -c postgres -- \ + bash -c ' + psql -U postgres -tc "SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = '"'"'wealthfolio_sync'"'"'" | grep -q 1 || \ + psql -U postgres -c "CREATE ROLE wealthfolio_sync WITH LOGIN PASSWORD '"'"'changeme-vault-will-rotate'"'"'" + psql -U postgres -tc "SELECT 1 FROM pg_catalog.pg_database WHERE datname = '"'"'wealthfolio_sync'"'"'" | grep -q 1 || \ + psql -U postgres -c "CREATE DATABASE wealthfolio_sync OWNER wealthfolio_sync" + psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE wealthfolio_sync TO wealthfolio_sync" + ' + EOT + } +} + # Old PostgreSQL deployment — kept commented for rollback reference # resource "kubernetes_deployment" "postgres" { # metadata { diff --git a/stacks/monitoring/modules/monitoring/dashboards/wealth.json b/stacks/monitoring/modules/monitoring/dashboards/wealth.json new file mode 100644 index 00000000..8fa3704c --- /dev/null +++ b/stacks/monitoring/modules/monitoring/dashboards/wealth.json @@ -0,0 +1,451 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": {"type": "datasource", "uid": "grafana"}, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Wealth — net worth, contributions, and growth over time. Backed by the wealthfolio_sync PG mirror of Wealthfolio's SQLite, refreshed hourly by the pg-sync sidecar.", + "editable": true, + "fiscalYearStartMonth": 0, + "id": null, + "links": [], + "panels": [ + { + "id": 1, + "title": "Net worth (current)", + "type": "stat", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0}, + "fieldConfig": { + "defaults": { + "unit": "currencyGBP", + "color": {"mode": "fixed", "fixedColor": "green"}, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "textMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "table", + "rawSql": "SELECT SUM(total_value) AS net_worth FROM daily_account_valuation WHERE valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation)" + } + ] + }, + { + "id": 2, + "title": "Net contribution (cumulative)", + "description": "Total deposits minus withdrawals across all accounts.", + "type": "stat", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "gridPos": {"h": 4, "w": 6, "x": 6, "y": 0}, + "fieldConfig": { + "defaults": { + "unit": "currencyGBP", + "color": {"mode": "fixed", "fixedColor": "blue"}, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "textMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "table", + "rawSql": "SELECT SUM(net_contribution) AS contribution FROM daily_account_valuation WHERE valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation)" + } + ] + }, + { + "id": 3, + "title": "Growth (unrealised)", + "description": "Net worth minus net contribution — the gain on everything you've put in.", + "type": "stat", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "gridPos": {"h": 4, "w": 6, "x": 12, "y": 0}, + "fieldConfig": { + "defaults": { + "unit": "currencyGBP", + "color": {"mode": "thresholds"}, + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 0} + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "textMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "table", + "rawSql": "SELECT (SUM(total_value) - SUM(net_contribution)) AS growth FROM daily_account_valuation WHERE valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation)" + } + ] + }, + { + "id": 4, + "title": "ROI %", + "description": "Growth / net contribution × 100. Excludes accounts with zero/negative contribution (Schwab) to avoid distortion.", + "type": "stat", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "gridPos": {"h": 4, "w": 6, "x": 18, "y": 0}, + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": {"mode": "thresholds"}, + "decimals": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "yellow", "value": 0}, + {"color": "green", "value": 5} + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "textMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "table", + "rawSql": "WITH latest AS (SELECT * FROM daily_account_valuation WHERE valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation) AND net_contribution > 0) SELECT (SUM(total_value - net_contribution) / NULLIF(SUM(net_contribution), 0) * 100) AS roi_pct FROM latest" + } + ] + }, + { + "id": 5, + "title": "Net worth — total over time", + "description": "Daily total_value summed across all accounts (base GBP).", + "type": "timeseries", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "gridPos": {"h": 10, "w": 24, "x": 0, "y": 4}, + "fieldConfig": { + "defaults": { + "color": {"mode": "fixed", "fixedColor": "green"}, + "unit": "currencyGBP", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 20, + "pointSize": 4, + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "stacking": {"group": "A", "mode": "none"} + } + }, + "overrides": [ + { + "matcher": {"id": "byName", "options": "net_worth"}, + "properties": [{"id": "displayName", "value": "Net worth"}] + } + ] + }, + "options": { + "legend": {"calcs": ["last", "max"], "displayMode": "table", "placement": "bottom"}, + "tooltip": {"mode": "multi", "sort": "desc"} + }, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "time_series", + "rawSql": "SELECT valuation_date::timestamp AS \"time\", SUM(total_value) AS net_worth FROM daily_account_valuation WHERE $__timeFilter(valuation_date) GROUP BY valuation_date ORDER BY valuation_date" + } + ] + }, + { + "id": 6, + "title": "Net contribution vs market value", + "description": "Net contribution = cumulative deposits − withdrawals. Market value = total_value (cash + investments). Gap between the two = unrealised growth.", + "type": "timeseries", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "gridPos": {"h": 10, "w": 12, "x": 0, "y": 14}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "unit": "currencyGBP", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "pointSize": 4, + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "stacking": {"group": "A", "mode": "none"} + } + }, + "overrides": [ + { + "matcher": {"id": "byName", "options": "market_value"}, + "properties": [ + {"id": "color", "value": {"mode": "fixed", "fixedColor": "green"}}, + {"id": "displayName", "value": "Market value"} + ] + }, + { + "matcher": {"id": "byName", "options": "net_contribution"}, + "properties": [ + {"id": "color", "value": {"mode": "fixed", "fixedColor": "blue"}}, + {"id": "displayName", "value": "Net contribution"} + ] + } + ] + }, + "options": { + "legend": {"calcs": ["last"], "displayMode": "table", "placement": "bottom"}, + "tooltip": {"mode": "multi", "sort": "desc"} + }, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "time_series", + "rawSql": "SELECT valuation_date::timestamp AS \"time\", SUM(net_contribution) AS net_contribution, SUM(total_value) AS market_value FROM daily_account_valuation WHERE $__timeFilter(valuation_date) GROUP BY valuation_date ORDER BY valuation_date" + } + ] + }, + { + "id": 7, + "title": "Growth (market value − contribution) over time", + "description": "Unrealised gain across all accounts. Filled area to emphasise the wealth created above the contributed capital.", + "type": "timeseries", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "gridPos": {"h": 10, "w": 12, "x": 12, "y": 14}, + "fieldConfig": { + "defaults": { + "color": {"mode": "fixed", "fixedColor": "#56A64B"}, + "unit": "currencyGBP", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 50, + "gradientMode": "opacity", + "pointSize": 4, + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "stacking": {"group": "A", "mode": "none"} + } + }, + "overrides": [ + { + "matcher": {"id": "byName", "options": "growth"}, + "properties": [{"id": "displayName", "value": "Growth"}] + } + ] + }, + "options": { + "legend": {"calcs": ["last", "max"], "displayMode": "table", "placement": "bottom"}, + "tooltip": {"mode": "multi", "sort": "desc"} + }, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "time_series", + "rawSql": "SELECT valuation_date::timestamp AS \"time\", (SUM(total_value) - SUM(net_contribution)) AS growth FROM daily_account_valuation WHERE $__timeFilter(valuation_date) GROUP BY valuation_date ORDER BY valuation_date" + } + ] + }, + { + "id": 8, + "title": "Per-account stacked — total value", + "description": "Stacked area showing each account's contribution to total net worth over time. Useful for spotting which account drives the trajectory.", + "type": "timeseries", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "gridPos": {"h": 11, "w": 24, "x": 0, "y": 24}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "unit": "currencyGBP", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 70, + "pointSize": 3, + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "stacking": {"group": "A", "mode": "normal"} + } + }, + "overrides": [] + }, + "options": { + "legend": {"calcs": ["last"], "displayMode": "table", "placement": "bottom"}, + "tooltip": {"mode": "multi", "sort": "desc"} + }, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "time_series", + "rawSql": "SELECT d.valuation_date::timestamp AS \"time\", a.name AS metric, d.total_value AS value FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id WHERE $__timeFilter(d.valuation_date) ORDER BY d.valuation_date, a.name" + } + ] + }, + { + "id": 9, + "title": "Cash vs invested (stacked)", + "description": "Daily breakdown of cash holdings vs market value of investments, summed across all accounts.", + "type": "timeseries", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "gridPos": {"h": 10, "w": 24, "x": 0, "y": 35}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "unit": "currencyGBP", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 70, + "pointSize": 3, + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "stacking": {"group": "A", "mode": "normal"} + } + }, + "overrides": [ + { + "matcher": {"id": "byName", "options": "cash"}, + "properties": [ + {"id": "color", "value": {"mode": "fixed", "fixedColor": "#FADE2A"}}, + {"id": "displayName", "value": "Cash"} + ] + }, + { + "matcher": {"id": "byName", "options": "invested"}, + "properties": [ + {"id": "color", "value": {"mode": "fixed", "fixedColor": "#56A64B"}}, + {"id": "displayName", "value": "Invested"} + ] + } + ] + }, + "options": { + "legend": {"calcs": ["last"], "displayMode": "table", "placement": "bottom"}, + "tooltip": {"mode": "multi", "sort": "desc"} + }, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "time_series", + "rawSql": "SELECT valuation_date::timestamp AS \"time\", SUM(cash_balance) AS cash, SUM(investment_market_value) AS invested FROM daily_account_valuation WHERE $__timeFilter(valuation_date) GROUP BY valuation_date ORDER BY valuation_date" + } + ] + }, + { + "id": 10, + "title": "Activity log", + "description": "Recent activities (BUY / SELL / DEPOSIT / WITHDRAWAL / DIVIDEND / etc.) across all accounts. Limited to 100 most recent.", + "type": "table", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "gridPos": {"h": 14, "w": 24, "x": 0, "y": 45}, + "fieldConfig": { + "defaults": { + "custom": {"align": "auto", "displayMode": "auto"} + }, + "overrides": [ + { + "matcher": {"id": "byName", "options": "amount"}, + "properties": [{"id": "unit", "value": "currencyGBP"}] + } + ] + }, + "options": { + "cellHeight": "sm", + "footer": {"show": false} + }, + "targets": [ + { + "refId": "A", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "wealth-pg"}, + "rawQuery": true, + "editorMode": "code", + "format": "table", + "rawSql": "SELECT a.activity_date AS \"date\", acc.name AS \"account\", a.activity_type AS \"type\", a.asset_id AS \"asset\", a.quantity AS \"qty\", a.unit_price AS \"unit_price\", a.amount AS \"amount\", a.currency AS \"ccy\", a.notes AS \"notes\" FROM activities a LEFT JOIN accounts acc ON acc.id = a.account_id WHERE $__timeFilter(a.activity_date) ORDER BY a.activity_date DESC LIMIT 100" + } + ] + } + ], + "refresh": "5m", + "schemaVersion": 39, + "tags": ["finance", "personal", "wealth"], + "templating": {"list": []}, + "time": {"from": "now-5y", "to": "now"}, + "timepicker": {}, + "timezone": "browser", + "title": "Wealth", + "uid": "wealth", + "version": 1 +} diff --git a/stacks/monitoring/modules/monitoring/grafana.tf b/stacks/monitoring/modules/monitoring/grafana.tf index 2c5089ee..ed1f4451 100644 --- a/stacks/monitoring/modules/monitoring/grafana.tf +++ b/stacks/monitoring/modules/monitoring/grafana.tf @@ -136,6 +136,7 @@ locals { "realestate-crawler.json" = "Applications" "uk-payslip.json" = "Finance" "job-hunter.json" = "Finance" + "wealth.json" = "Finance" } } diff --git a/stacks/vault/main.tf b/stacks/vault/main.tf index 52ea3b6a..e98b2f35 100644 --- a/stacks/vault/main.tf +++ b/stacks/vault/main.tf @@ -554,7 +554,8 @@ resource "vault_database_secret_backend_connection" "postgresql" { # "pg-trading", # Commented out 2026-04-06 - trading-bot disabled "pg-health", "pg-linkwarden", "pg-affine", "pg-woodpecker", "pg-claude-memory", - "pg-terraform-state", "pg-payslip-ingest", "pg-job-hunter" + "pg-terraform-state", "pg-payslip-ingest", "pg-job-hunter", + "pg-wealthfolio-sync" ] postgresql { @@ -708,6 +709,14 @@ resource "vault_database_secret_backend_static_role" "pg_job_hunter" { rotation_period = 604800 } +resource "vault_database_secret_backend_static_role" "pg_wealthfolio_sync" { + backend = vault_mount.database.path + db_name = vault_database_secret_backend_connection.postgresql.name + name = "pg-wealthfolio-sync" + username = "wealthfolio_sync" + rotation_period = 604800 +} + # ============================================================================= # Kubernetes Secrets Engine — Dynamic K8s Credentials # ============================================================================= diff --git a/stacks/wealthfolio/main.tf b/stacks/wealthfolio/main.tf index a469e9b3..258578a2 100644 --- a/stacks/wealthfolio/main.tf +++ b/stacks/wealthfolio/main.tf @@ -3,6 +3,7 @@ variable "tls_secret_name" { sensitive = true } variable "nfs_server" { type = string } +variable "postgresql_host" { type = string } resource "kubernetes_namespace" "wealthfolio" { metadata { @@ -45,6 +46,52 @@ resource "kubernetes_manifest" "external_secret" { depends_on = [kubernetes_namespace.wealthfolio] } +# DB credentials for the SQLite→PG ETL sidecar. Vault DB engine static role +# `pg-wealthfolio-sync` rotates this every 7 days; ExternalSecret refreshes +# the K8s Secret every 15m so the sidecar always has a valid password. +resource "kubernetes_manifest" "wealthfolio_sync_db_external_secret" { + manifest = { + apiVersion = "external-secrets.io/v1beta1" + kind = "ExternalSecret" + metadata = { + name = "wealthfolio-sync-db-creds" + namespace = "wealthfolio" + } + spec = { + refreshInterval = "15m" + secretStoreRef = { + name = "vault-database" + kind = "ClusterSecretStore" + } + target = { + name = "wealthfolio-sync-db-creds" + template = { + metadata = { + annotations = { + "reloader.stakater.com/match" = "true" + } + } + data = { + PGHOST = var.postgresql_host + PGPORT = "5432" + PGDATABASE = "wealthfolio_sync" + PGUSER = "wealthfolio_sync" + PGPASSWORD = "{{ .password }}" + } + } + } + data = [{ + secretKey = "password" + remoteRef = { + key = "static-creds/pg-wealthfolio-sync" + property = "password" + } + }] + } + } + depends_on = [kubernetes_namespace.wealthfolio] +} + module "tls_secret" { source = "../../modules/kubernetes/setup_tls_secret" namespace = kubernetes_namespace.wealthfolio.metadata[0].name @@ -214,6 +261,180 @@ resource "kubernetes_deployment" "wealthfolio" { limits = { memory = "64Mi" } } } + + # pg-sync sidecar — mirrors a small subset of SQLite into PG every hour + # so Grafana can chart net worth / contributions / growth via the + # `wealthfolio_sync` database. Mounts /data RO; writes to a tmp dir + # for the sqlite3 .backup snapshot to avoid blocking writers. Bootstrap + # DDL runs each iteration (CREATE TABLE IF NOT EXISTS — idempotent). + # Truncate-and-reload pattern: tables are small (~10k DAV rows, ~500 + # activities, 6 accounts), so a full reload each hour is simpler than + # incremental upserts and gives clean cold-start behaviour. + container { + name = "pg-sync" + image = "alpine:3.20" + env { + name = "PGHOST" + value_from { + secret_key_ref { + name = "wealthfolio-sync-db-creds" + key = "PGHOST" + } + } + } + env { + name = "PGPORT" + value_from { + secret_key_ref { + name = "wealthfolio-sync-db-creds" + key = "PGPORT" + } + } + } + env { + name = "PGDATABASE" + value_from { + secret_key_ref { + name = "wealthfolio-sync-db-creds" + key = "PGDATABASE" + } + } + } + env { + name = "PGUSER" + value_from { + secret_key_ref { + name = "wealthfolio-sync-db-creds" + key = "PGUSER" + } + } + } + env { + name = "PGPASSWORD" + value_from { + secret_key_ref { + name = "wealthfolio-sync-db-creds" + key = "PGPASSWORD" + } + } + } + command = ["/bin/sh", "-c", <<-EOT + set -eu + apk add --no-cache --quiet sqlite postgresql-client busybox-suid + mkdir -p /etc/crontabs /scripts /tmp/wf-sync + cat >/etc/crontabs/root <<'CRON' + # Hourly: snapshot SQLite, reload PG mirror. + 7 * * * * /scripts/sync.sh >>/proc/1/fd/1 2>&1 + CRON + cat >/scripts/sync.sh <<'SCRIPT' + #!/bin/sh + set -eu + TS=$(date -u +%Y-%m-%dT%H:%M:%SZ) + echo "[$TS] wealthfolio-pg-sync: starting" + + # Bootstrap schema (idempotent). + psql -v ON_ERROR_STOP=1 <<'SQL' + CREATE TABLE IF NOT EXISTS accounts ( + id TEXT PRIMARY KEY, + name TEXT, + account_type TEXT, + currency TEXT, + is_active BOOLEAN + ); + CREATE TABLE IF NOT EXISTS daily_account_valuation ( + id TEXT PRIMARY KEY, + account_id TEXT NOT NULL, + valuation_date DATE NOT NULL, + account_currency TEXT, + base_currency TEXT, + fx_rate_to_base NUMERIC, + cash_balance NUMERIC, + investment_market_value NUMERIC, + total_value NUMERIC, + cost_basis NUMERIC, + net_contribution NUMERIC + ); + CREATE INDEX IF NOT EXISTS idx_dav_acct_date ON daily_account_valuation(account_id, valuation_date); + CREATE INDEX IF NOT EXISTS idx_dav_date ON daily_account_valuation(valuation_date); + CREATE TABLE IF NOT EXISTS activities ( + id TEXT PRIMARY KEY, + account_id TEXT, + asset_id TEXT, + activity_type TEXT, + activity_date TIMESTAMPTZ, + quantity NUMERIC, + unit_price NUMERIC, + amount NUMERIC, + fee NUMERIC, + currency TEXT, + fx_rate NUMERIC, + notes TEXT + ); + CREATE INDEX IF NOT EXISTS idx_act_date ON activities(activity_date); + SQL + + # Snapshot SQLite (online backup — non-blocking). + rm -f /tmp/wf-sync/snapshot.db + sqlite3 /data/wealthfolio.db ".backup /tmp/wf-sync/snapshot.db" + + # Dump source rows to TSV. + sqlite3 -separator $'\t' /tmp/wf-sync/snapshot.db \ + "SELECT id, name, account_type, currency, is_active FROM accounts;" \ + > /tmp/wf-sync/accounts.tsv + + sqlite3 -separator $'\t' /tmp/wf-sync/snapshot.db <<'SQ' > /tmp/wf-sync/dav.tsv + SELECT id, account_id, valuation_date, account_currency, base_currency, + CAST(fx_rate_to_base AS REAL), + CAST(cash_balance AS REAL), + CAST(investment_market_value AS REAL), + CAST(total_value AS REAL), + CAST(cost_basis AS REAL), + CAST(net_contribution AS REAL) + FROM daily_account_valuation; + SQ + + sqlite3 -separator $'\t' /tmp/wf-sync/snapshot.db <<'SQ' > /tmp/wf-sync/activities.tsv + SELECT id, account_id, asset_id, activity_type, activity_date, + CAST(quantity AS REAL), + CAST(unit_price AS REAL), + CAST(amount AS REAL), + CAST(fee AS REAL), + currency, + CAST(fx_rate AS REAL), + notes + FROM activities WHERE status='POSTED'; + SQ + + # Truncate-and-reload (small tables; simpler than upserts). + psql -v ON_ERROR_STOP=1 <