wealth: SQLite→PG ETL sidecar + new Grafana dashboard
Mirrors Wealthfolio's daily_account_valuation / accounts / activities from SQLite into a new PG database (wealthfolio_sync) every hour, so Grafana can chart net worth, contributions, and growth over time. Components: - dbaas: null_resource creates wealthfolio_sync DB + role on the CNPG cluster (dynamic primary lookup so it survives failover). - vault: pg-wealthfolio-sync static role rotates the password every 7d. - wealthfolio: ExternalSecret pulls the rotated password into the WF namespace; new pg-sync sidecar (alpine + sqlite + postgresql-client + busybox crond) does sqlite3 .backup → TSV dump → truncate-and-reload psql, hourly at :07. Plus a grafana-wealth-datasource ConfigMap in the monitoring namespace (uid: wealth-pg). - monitoring: new Wealth dashboard (wealth.json, 10 panels) — current net worth / contribution / growth / ROI% stats, then time-series for net worth, contribution-vs-market, growth area, per-account stacked area, cash-vs-invested, and a 100-row activity log. Initial sync: 6 accounts, 10,798 daily valuations, 518 activities. Verified PG totals match SQLite latest snapshot exactly.
This commit is contained in:
Viktor Barzin
parent
7dd580972a
commit
bf4c7618d8
6 changed files with 762 additions and 1 deletions
|
|
@ -3,6 +3,7 @@ variable "tls_secret_name" {
|
|||
sensitive = true
|
||||
}
|
||||
variable "nfs_server" { type = string }
|
||||
variable "postgresql_host" { type = string }
|
||||
|
||||
resource "kubernetes_namespace" "wealthfolio" {
|
||||
metadata {
|
||||
|
|
@ -45,6 +46,52 @@ resource "kubernetes_manifest" "external_secret" {
|
|||
depends_on = [kubernetes_namespace.wealthfolio]
|
||||
}
|
||||
|
||||
# DB credentials for the SQLite→PG ETL sidecar. Vault DB engine static role
|
||||
# `pg-wealthfolio-sync` rotates this every 7 days; ExternalSecret refreshes
|
||||
# the K8s Secret every 15m so the sidecar always has a valid password.
|
||||
resource "kubernetes_manifest" "wealthfolio_sync_db_external_secret" {
|
||||
manifest = {
|
||||
apiVersion = "external-secrets.io/v1beta1"
|
||||
kind = "ExternalSecret"
|
||||
metadata = {
|
||||
name = "wealthfolio-sync-db-creds"
|
||||
namespace = "wealthfolio"
|
||||
}
|
||||
spec = {
|
||||
refreshInterval = "15m"
|
||||
secretStoreRef = {
|
||||
name = "vault-database"
|
||||
kind = "ClusterSecretStore"
|
||||
}
|
||||
target = {
|
||||
name = "wealthfolio-sync-db-creds"
|
||||
template = {
|
||||
metadata = {
|
||||
annotations = {
|
||||
"reloader.stakater.com/match" = "true"
|
||||
}
|
||||
}
|
||||
data = {
|
||||
PGHOST = var.postgresql_host
|
||||
PGPORT = "5432"
|
||||
PGDATABASE = "wealthfolio_sync"
|
||||
PGUSER = "wealthfolio_sync"
|
||||
PGPASSWORD = "{{ .password }}"
|
||||
}
|
||||
}
|
||||
}
|
||||
data = [{
|
||||
secretKey = "password"
|
||||
remoteRef = {
|
||||
key = "static-creds/pg-wealthfolio-sync"
|
||||
property = "password"
|
||||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
depends_on = [kubernetes_namespace.wealthfolio]
|
||||
}
|
||||
|
||||
module "tls_secret" {
|
||||
source = "../../modules/kubernetes/setup_tls_secret"
|
||||
namespace = kubernetes_namespace.wealthfolio.metadata[0].name
|
||||
|
|
@ -214,6 +261,180 @@ resource "kubernetes_deployment" "wealthfolio" {
|
|||
limits = { memory = "64Mi" }
|
||||
}
|
||||
}
|
||||
|
||||
# pg-sync sidecar — mirrors a small subset of SQLite into PG every hour
|
||||
# so Grafana can chart net worth / contributions / growth via the
|
||||
# `wealthfolio_sync` database. Mounts /data RO; writes to a tmp dir
|
||||
# for the sqlite3 .backup snapshot to avoid blocking writers. Bootstrap
|
||||
# DDL runs each iteration (CREATE TABLE IF NOT EXISTS — idempotent).
|
||||
# Truncate-and-reload pattern: tables are small (~10k DAV rows, ~500
|
||||
# activities, 6 accounts), so a full reload each hour is simpler than
|
||||
# incremental upserts and gives clean cold-start behaviour.
|
||||
container {
|
||||
name = "pg-sync"
|
||||
image = "alpine:3.20"
|
||||
env {
|
||||
name = "PGHOST"
|
||||
value_from {
|
||||
secret_key_ref {
|
||||
name = "wealthfolio-sync-db-creds"
|
||||
key = "PGHOST"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "PGPORT"
|
||||
value_from {
|
||||
secret_key_ref {
|
||||
name = "wealthfolio-sync-db-creds"
|
||||
key = "PGPORT"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "PGDATABASE"
|
||||
value_from {
|
||||
secret_key_ref {
|
||||
name = "wealthfolio-sync-db-creds"
|
||||
key = "PGDATABASE"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "PGUSER"
|
||||
value_from {
|
||||
secret_key_ref {
|
||||
name = "wealthfolio-sync-db-creds"
|
||||
key = "PGUSER"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "PGPASSWORD"
|
||||
value_from {
|
||||
secret_key_ref {
|
||||
name = "wealthfolio-sync-db-creds"
|
||||
key = "PGPASSWORD"
|
||||
}
|
||||
}
|
||||
}
|
||||
command = ["/bin/sh", "-c", <<-EOT
|
||||
set -eu
|
||||
apk add --no-cache --quiet sqlite postgresql-client busybox-suid
|
||||
mkdir -p /etc/crontabs /scripts /tmp/wf-sync
|
||||
cat >/etc/crontabs/root <<'CRON'
|
||||
# Hourly: snapshot SQLite, reload PG mirror.
|
||||
7 * * * * /scripts/sync.sh >>/proc/1/fd/1 2>&1
|
||||
CRON
|
||||
cat >/scripts/sync.sh <<'SCRIPT'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
echo "[$TS] wealthfolio-pg-sync: starting"
|
||||
|
||||
# Bootstrap schema (idempotent).
|
||||
psql -v ON_ERROR_STOP=1 <<'SQL'
|
||||
CREATE TABLE IF NOT EXISTS accounts (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT,
|
||||
account_type TEXT,
|
||||
currency TEXT,
|
||||
is_active BOOLEAN
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS daily_account_valuation (
|
||||
id TEXT PRIMARY KEY,
|
||||
account_id TEXT NOT NULL,
|
||||
valuation_date DATE NOT NULL,
|
||||
account_currency TEXT,
|
||||
base_currency TEXT,
|
||||
fx_rate_to_base NUMERIC,
|
||||
cash_balance NUMERIC,
|
||||
investment_market_value NUMERIC,
|
||||
total_value NUMERIC,
|
||||
cost_basis NUMERIC,
|
||||
net_contribution NUMERIC
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_dav_acct_date ON daily_account_valuation(account_id, valuation_date);
|
||||
CREATE INDEX IF NOT EXISTS idx_dav_date ON daily_account_valuation(valuation_date);
|
||||
CREATE TABLE IF NOT EXISTS activities (
|
||||
id TEXT PRIMARY KEY,
|
||||
account_id TEXT,
|
||||
asset_id TEXT,
|
||||
activity_type TEXT,
|
||||
activity_date TIMESTAMPTZ,
|
||||
quantity NUMERIC,
|
||||
unit_price NUMERIC,
|
||||
amount NUMERIC,
|
||||
fee NUMERIC,
|
||||
currency TEXT,
|
||||
fx_rate NUMERIC,
|
||||
notes TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_act_date ON activities(activity_date);
|
||||
SQL
|
||||
|
||||
# Snapshot SQLite (online backup — non-blocking).
|
||||
rm -f /tmp/wf-sync/snapshot.db
|
||||
sqlite3 /data/wealthfolio.db ".backup /tmp/wf-sync/snapshot.db"
|
||||
|
||||
# Dump source rows to TSV.
|
||||
sqlite3 -separator $'\t' /tmp/wf-sync/snapshot.db \
|
||||
"SELECT id, name, account_type, currency, is_active FROM accounts;" \
|
||||
> /tmp/wf-sync/accounts.tsv
|
||||
|
||||
sqlite3 -separator $'\t' /tmp/wf-sync/snapshot.db <<'SQ' > /tmp/wf-sync/dav.tsv
|
||||
SELECT id, account_id, valuation_date, account_currency, base_currency,
|
||||
CAST(fx_rate_to_base AS REAL),
|
||||
CAST(cash_balance AS REAL),
|
||||
CAST(investment_market_value AS REAL),
|
||||
CAST(total_value AS REAL),
|
||||
CAST(cost_basis AS REAL),
|
||||
CAST(net_contribution AS REAL)
|
||||
FROM daily_account_valuation;
|
||||
SQ
|
||||
|
||||
sqlite3 -separator $'\t' /tmp/wf-sync/snapshot.db <<'SQ' > /tmp/wf-sync/activities.tsv
|
||||
SELECT id, account_id, asset_id, activity_type, activity_date,
|
||||
CAST(quantity AS REAL),
|
||||
CAST(unit_price AS REAL),
|
||||
CAST(amount AS REAL),
|
||||
CAST(fee AS REAL),
|
||||
currency,
|
||||
CAST(fx_rate AS REAL),
|
||||
notes
|
||||
FROM activities WHERE status='POSTED';
|
||||
SQ
|
||||
|
||||
# Truncate-and-reload (small tables; simpler than upserts).
|
||||
psql -v ON_ERROR_STOP=1 <<SQL
|
||||
BEGIN;
|
||||
TRUNCATE accounts, daily_account_valuation, activities;
|
||||
\copy accounts FROM '/tmp/wf-sync/accounts.tsv' WITH (FORMAT csv, DELIMITER E'\t', NULL '');
|
||||
\copy daily_account_valuation FROM '/tmp/wf-sync/dav.tsv' WITH (FORMAT csv, DELIMITER E'\t', NULL '');
|
||||
\copy activities FROM '/tmp/wf-sync/activities.tsv' WITH (FORMAT csv, DELIMITER E'\t', NULL '');
|
||||
COMMIT;
|
||||
SQL
|
||||
|
||||
ROWS=$(psql -tAc "SELECT COUNT(*) FROM daily_account_valuation;")
|
||||
echo "[$TS] wealthfolio-pg-sync: ok (daily_account_valuation rows=$ROWS)"
|
||||
rm -f /tmp/wf-sync/*.tsv /tmp/wf-sync/snapshot.db
|
||||
SCRIPT
|
||||
chmod +x /scripts/sync.sh
|
||||
echo "wealthfolio-pg-sync sidecar ready; running initial sync, then hourly at :07"
|
||||
/scripts/sync.sh || echo "initial sync failed (will retry on next cron tick)"
|
||||
exec crond -f -l 8
|
||||
EOT
|
||||
]
|
||||
volume_mount {
|
||||
name = "data"
|
||||
mount_path = "/data"
|
||||
read_only = true
|
||||
}
|
||||
resources {
|
||||
requests = { cpu = "10m", memory = "32Mi" }
|
||||
limits = { memory = "128Mi" }
|
||||
}
|
||||
}
|
||||
volume {
|
||||
name = "data"
|
||||
persistent_volume_claim {
|
||||
|
|
@ -374,6 +595,53 @@ resource "kubernetes_cron_job_v1" "wealthfolio_sync" {
|
|||
}
|
||||
}
|
||||
|
||||
# Plan-time read of the ESO-created K8s Secret for Grafana datasource password.
|
||||
# First apply: -target=kubernetes_manifest.wealthfolio_sync_db_external_secret first.
|
||||
data "kubernetes_secret" "wealthfolio_sync_db_creds" {
|
||||
metadata {
|
||||
name = "wealthfolio-sync-db-creds"
|
||||
namespace = kubernetes_namespace.wealthfolio.metadata[0].name
|
||||
}
|
||||
depends_on = [kubernetes_manifest.wealthfolio_sync_db_external_secret]
|
||||
}
|
||||
|
||||
# Grafana datasource for wealthfolio_sync PostgreSQL DB.
|
||||
# Lives in the monitoring namespace so the Grafana sidecar (grafana_datasource=1) picks it up.
|
||||
resource "kubernetes_config_map" "grafana_wealth_datasource" {
|
||||
metadata {
|
||||
name = "grafana-wealth-datasource"
|
||||
namespace = "monitoring"
|
||||
labels = {
|
||||
grafana_datasource = "1"
|
||||
}
|
||||
}
|
||||
data = {
|
||||
"wealth-datasource.yaml" = yamlencode({
|
||||
apiVersion = 1
|
||||
datasources = [{
|
||||
name = "Wealth"
|
||||
type = "postgres"
|
||||
access = "proxy"
|
||||
url = "${var.postgresql_host}:5432"
|
||||
user = "wealthfolio_sync"
|
||||
uid = "wealth-pg"
|
||||
# Grafana 11.2+ Postgres plugin reads DB name from jsonData.database
|
||||
# (top-level `database` is silently ignored).
|
||||
jsonData = {
|
||||
database = "wealthfolio_sync"
|
||||
sslmode = "disable"
|
||||
postgresVersion = 1600
|
||||
timescaledb = false
|
||||
}
|
||||
secureJsonData = {
|
||||
password = data.kubernetes_secret.wealthfolio_sync_db_creds.data["PGPASSWORD"]
|
||||
}
|
||||
editable = true
|
||||
}]
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
############################################################################
|
||||
# Backup — sidecar approach
|
||||
#
|
||||
|
|
|
|||
|
|
@ -11,3 +11,8 @@ dependency "vault" {
|
|||
config_path = "../vault"
|
||||
skip_outputs = true
|
||||
}
|
||||
|
||||
dependency "external-secrets" {
|
||||
config_path = "../external-secrets"
|
||||
skip_outputs = true
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue