viktor/infra
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

monitoring(wealth): skew-resilient queries, no more partial-day dips

Browse source 
Bug witnessed 2026-05-01: dashboard "Net worth (current)" showed £88k
instead of £1.03M because at 02:00 UTC an external trigger refreshed
ONE account (Trading212 ISA), creating its 05-01 daily_account_valuation
row. The 5 other accounts still had their last row at 04-30. The panel
SQL `WHERE valuation_date = (SELECT MAX(valuation_date))` then summed
only the single account that had a 05-01 row.

Two new SQL patterns adopted across all 15 affected panels:

  1. Stat / barchart "current snapshot" panels (1, 2, 3, 4, 11, 14, 15,
     16): latest-per-account stitching —
       WITH latest AS (SELECT DISTINCT ON (d.account_id) ...
                       FROM daily_account_valuation d
                       JOIN accounts a ON a.id = d.account_id
                       ORDER BY d.account_id, d.valuation_date DESC)
     gives a coherent "now" snapshot regardless of refresh skew, and
     the inner join filters out orphan/deleted accounts (one such was
     adding a stale £33k from 04-17). 12-month panels add a parallel
     `ago` CTE picking each account's row closest to (d_now - 12mo).

  2. Time-series / yearly panels (5, 6, 7, 8, 9, 12, 13): complete-days-
     only filter —
       WITH active_accounts AS (SELECT COUNT(*) FROM accounts),
            complete_dates AS (SELECT valuation_date
                               FROM daily_account_valuation d
                               JOIN accounts a ON a.id=d.account_id
                               GROUP BY valuation_date
                               HAVING COUNT(*) >= active.n)
     so a partial today never renders as a chart dip. The day rejoins
     the chart automatically once the daily 16:00 UTC sync writes rows
     for every account.

Verified end-to-end against live PG: new queries produce £1,033,734
(matches the 6 active accounts' true latest sum) where the old query
gave £88k.
This commit is contained in:
Viktor Barzin 2026-05-01 16:08:18 +00:00
parent 1d3ae01aac
commit 628f5a0d26
1 changed files with 15 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

30
stacks/monitoring/modules/monitoring/dashboards/wealth.json
View file

@ -47,7 +47,7 @@
"rawQuery": true, "rawQuery": true,
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawSql": "SELECT SUM(total_value) AS net_worth FROM daily_account_valuation WHERE valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation)" "rawSql": "WITH latest AS (SELECT DISTINCT ON (d.account_id) d.account_id, d.total_value, d.net_contribution, d.cash_balance, d.investment_market_value FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id ORDER BY d.account_id, d.valuation_date DESC) SELECT SUM(total_value) AS net_worth FROM latest"
} }
] ]
}, },
@ -81,7 +81,7 @@
"rawQuery": true, "rawQuery": true,
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawSql": "SELECT SUM(net_contribution) AS contribution FROM daily_account_valuation WHERE valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation)" "rawSql": "WITH latest AS (SELECT DISTINCT ON (d.account_id) d.account_id, d.total_value, d.net_contribution, d.cash_balance, d.investment_market_value FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id ORDER BY d.account_id, d.valuation_date DESC) SELECT SUM(net_contribution) AS contribution FROM latest"
} }
] ]
}, },
@ -122,7 +122,7 @@
"rawQuery": true, "rawQuery": true,
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawSql": "SELECT (SUM(total_value) - SUM(net_contribution)) AS growth FROM daily_account_valuation WHERE valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation)" "rawSql": "WITH latest AS (SELECT DISTINCT ON (d.account_id) d.account_id, d.total_value, d.net_contribution, d.cash_balance, d.investment_market_value FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id ORDER BY d.account_id, d.valuation_date DESC) SELECT (SUM(total_value) - SUM(net_contribution)) AS growth FROM latest"
} }
] ]
}, },
@ -164,7 +164,7 @@
"rawQuery": true, "rawQuery": true,
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawSql": "WITH latest AS (SELECT * FROM daily_account_valuation WHERE valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation) AND net_contribution > 0) SELECT (SUM(total_value - net_contribution) / NULLIF(SUM(net_contribution), 0) * 100) AS roi_pct FROM latest" "rawSql": "WITH latest AS (SELECT DISTINCT ON (d.account_id) d.account_id, d.total_value, d.net_contribution, d.cash_balance, d.investment_market_value FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id ORDER BY d.account_id, d.valuation_date DESC) SELECT (SUM(total_value - net_contribution) / NULLIF(SUM(net_contribution), 0) * 100) AS roi_pct FROM latest WHERE net_contribution > 0"
} }
] ]
}, },
@ -208,7 +208,7 @@
"rawQuery": true, "rawQuery": true,
"editorMode": "code", "editorMode": "code",
"format": "time_series", "format": "time_series",
"rawSql": "SELECT valuation_date::timestamp AS \"time\", SUM(total_value) AS net_worth FROM daily_account_valuation WHERE $__timeFilter(valuation_date) GROUP BY valuation_date ORDER BY valuation_date" "rawSql": "WITH active_accounts AS (SELECT COUNT(*) AS n FROM accounts), complete_dates AS (SELECT d.valuation_date FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id GROUP BY d.valuation_date HAVING COUNT(*) >= (SELECT n FROM active_accounts)) SELECT valuation_date::timestamp AS \\\"time\\\", SUM(total_value) AS net_worth FROM daily_account_valuation WHERE $__timeFilter(valuation_date) AND valuation_date IN (SELECT valuation_date FROM complete_dates) GROUP BY valuation_date ORDER BY valuation_date"
} }
] ]
}, },
@ -262,7 +262,7 @@
"rawQuery": true, "rawQuery": true,
"editorMode": "code", "editorMode": "code",
"format": "time_series", "format": "time_series",
"rawSql": "SELECT valuation_date::timestamp AS \"time\", SUM(net_contribution) AS net_contribution, SUM(total_value) AS market_value FROM daily_account_valuation WHERE $__timeFilter(valuation_date) GROUP BY valuation_date ORDER BY valuation_date" "rawSql": "WITH active_accounts AS (SELECT COUNT(*) AS n FROM accounts), complete_dates AS (SELECT d.valuation_date FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id GROUP BY d.valuation_date HAVING COUNT(*) >= (SELECT n FROM active_accounts)) SELECT valuation_date::timestamp AS \\\"time\\\", SUM(net_contribution) AS net_contribution, SUM(total_value) AS market_value FROM daily_account_valuation WHERE $__timeFilter(valuation_date) AND valuation_date IN (SELECT valuation_date FROM complete_dates) GROUP BY valuation_date ORDER BY valuation_date"
} }
] ]
}, },
@ -307,7 +307,7 @@
"rawQuery": true, "rawQuery": true,
"editorMode": "code", "editorMode": "code",
"format": "time_series", "format": "time_series",
"rawSql": "SELECT valuation_date::timestamp AS \"time\", (SUM(total_value) - SUM(net_contribution)) AS growth FROM daily_account_valuation WHERE $__timeFilter(valuation_date) GROUP BY valuation_date ORDER BY valuation_date" "rawSql": "WITH active_accounts AS (SELECT COUNT(*) AS n FROM accounts), complete_dates AS (SELECT d.valuation_date FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id GROUP BY d.valuation_date HAVING COUNT(*) >= (SELECT n FROM active_accounts)) SELECT valuation_date::timestamp AS \\\"time\\\", (SUM(total_value) - SUM(net_contribution)) AS growth FROM daily_account_valuation WHERE $__timeFilter(valuation_date) AND valuation_date IN (SELECT valuation_date FROM complete_dates) GROUP BY valuation_date ORDER BY valuation_date"
} }
] ]
}, },
@ -346,7 +346,7 @@
"rawQuery": true, "rawQuery": true,
"editorMode": "code", "editorMode": "code",
"format": "time_series", "format": "time_series",
"rawSql": "SELECT d.valuation_date::timestamp AS \"time\", a.name AS metric, d.total_value AS value FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id WHERE $__timeFilter(d.valuation_date) ORDER BY d.valuation_date, a.name" "rawSql": "WITH active_accounts AS (SELECT COUNT(*) AS n FROM accounts), complete_dates AS (SELECT d.valuation_date FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id GROUP BY d.valuation_date HAVING COUNT(*) >= (SELECT n FROM active_accounts)) SELECT d.valuation_date::timestamp AS \\\"time\\\", a.name AS metric, d.total_value AS value FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id WHERE $__timeFilter(d.valuation_date) AND d.valuation_date IN (SELECT valuation_date FROM complete_dates) ORDER BY d.valuation_date, a.name"
} }
] ]
}, },
@ -400,7 +400,7 @@
"rawQuery": true, "rawQuery": true,
"editorMode": "code", "editorMode": "code",
"format": "time_series", "format": "time_series",
"rawSql": "SELECT d.valuation_date::timestamp AS \"time\", SUM(CASE WHEN a.account_type = 'WORKPLACE_PENSION' THEN 0 ELSE d.cash_balance END) AS cash, SUM(CASE WHEN a.account_type = 'WORKPLACE_PENSION' THEN d.cash_balance + d.investment_market_value ELSE d.investment_market_value END) AS invested FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id WHERE $__timeFilter(d.valuation_date) GROUP BY d.valuation_date ORDER BY d.valuation_date" "rawSql": "WITH active_accounts AS (SELECT COUNT(*) AS n FROM accounts), complete_dates AS (SELECT d.valuation_date FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id GROUP BY d.valuation_date HAVING COUNT(*) >= (SELECT n FROM active_accounts)) SELECT d.valuation_date::timestamp AS \\\"time\\\", SUM(CASE WHEN a.account_type = 'WORKPLACE_PENSION' THEN 0 ELSE d.cash_balance END) AS cash, SUM(CASE WHEN a.account_type = 'WORKPLACE_PENSION' THEN d.cash_balance + d.investment_market_value ELSE d.investment_market_value END) AS invested FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id WHERE $__timeFilter(d.valuation_date) AND d.valuation_date IN (SELECT valuation_date FROM complete_dates) GROUP BY d.valuation_date ORDER BY d.valuation_date"
} }
] ]
}, },
@ -475,7 +475,7 @@
"rawQuery": true, "rawQuery": true,
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawSql": "WITH bounds AS (SELECT (SELECT MAX(valuation_date) FROM daily_account_valuation) AS d_now, (SELECT MIN(valuation_date) FROM daily_account_valuation WHERE valuation_date >= (SELECT MAX(valuation_date) - INTERVAL '12 months' FROM daily_account_valuation)) AS d_ago), agg AS (SELECT (SELECT SUM(total_value) FROM daily_account_valuation WHERE valuation_date = b.d_now) AS nw_now, (SELECT SUM(net_contribution) FROM daily_account_valuation WHERE valuation_date = b.d_now) AS contrib_now, (SELECT SUM(total_value) FROM daily_account_valuation WHERE valuation_date = b.d_ago) AS nw_ago, (SELECT SUM(net_contribution) FROM daily_account_valuation WHERE valuation_date = b.d_ago) AS contrib_ago FROM bounds b) SELECT ROUND((((nw_now - nw_ago - (contrib_now - contrib_ago)) / NULLIF(nw_ago + 0.5 * (contrib_now - contrib_ago), 0)) * 100)::numeric, 2) AS pct_12mo FROM agg" "rawSql": "WITH latest AS (SELECT DISTINCT ON (d.account_id) d.account_id, d.valuation_date AS d_now, d.total_value AS nw_now, d.net_contribution AS contrib_now FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id ORDER BY d.account_id, d.valuation_date DESC), ago AS (SELECT DISTINCT ON (l.account_id) l.account_id, d.total_value AS nw_ago, d.net_contribution AS contrib_ago FROM latest l JOIN daily_account_valuation d ON d.account_id = l.account_id AND d.valuation_date <= l.d_now - INTERVAL '12 months' ORDER BY l.account_id, d.valuation_date DESC), agg AS (SELECT (SELECT SUM(nw_now) FROM latest) AS nw_now, (SELECT SUM(contrib_now) FROM latest) AS contrib_now, (SELECT SUM(nw_ago) FROM ago) AS nw_ago, (SELECT SUM(contrib_ago) FROM ago) AS contrib_ago) SELECT ROUND((((nw_now - nw_ago - (contrib_now - contrib_ago)) / NULLIF(nw_ago + 0.5 * (contrib_now - contrib_ago), 0)) * 100)::numeric, 2) AS pct_12mo FROM agg"
} }
] ]
}, },
@ -509,7 +509,7 @@
"rawQuery": true, "rawQuery": true,
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawSql": "WITH bounds AS (SELECT (SELECT MAX(valuation_date) FROM daily_account_valuation) AS d_now, (SELECT MIN(valuation_date) FROM daily_account_valuation WHERE valuation_date >= (SELECT MAX(valuation_date) - INTERVAL '12 months' FROM daily_account_valuation)) AS d_ago) SELECT ((SELECT SUM(net_contribution) FROM daily_account_valuation WHERE valuation_date = b.d_now) - (SELECT SUM(net_contribution) FROM daily_account_valuation WHERE valuation_date = b.d_ago)) AS contrib_12mo FROM bounds b" "rawSql": "WITH latest AS (SELECT DISTINCT ON (d.account_id) d.account_id, d.valuation_date AS d_now, d.total_value AS nw_now, d.net_contribution AS contrib_now FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id ORDER BY d.account_id, d.valuation_date DESC), ago AS (SELECT DISTINCT ON (l.account_id) l.account_id, d.total_value AS nw_ago, d.net_contribution AS contrib_ago FROM latest l JOIN daily_account_valuation d ON d.account_id = l.account_id AND d.valuation_date <= l.d_now - INTERVAL '12 months' ORDER BY l.account_id, d.valuation_date DESC), agg AS (SELECT (SELECT SUM(nw_now) FROM latest) AS nw_now, (SELECT SUM(contrib_now) FROM latest) AS contrib_now, (SELECT SUM(nw_ago) FROM ago) AS nw_ago, (SELECT SUM(contrib_ago) FROM ago) AS contrib_ago) SELECT (contrib_now - contrib_ago) AS contrib_12mo FROM agg"
} }
] ]
}, },
@ -550,7 +550,7 @@
"rawQuery": true, "rawQuery": true,
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawSql": "WITH bounds AS (SELECT (SELECT MAX(valuation_date) FROM daily_account_valuation) AS d_now, (SELECT MIN(valuation_date) FROM daily_account_valuation WHERE valuation_date >= (SELECT MAX(valuation_date) - INTERVAL '12 months' FROM daily_account_valuation)) AS d_ago), agg AS (SELECT (SELECT SUM(total_value) FROM daily_account_valuation WHERE valuation_date = b.d_now) AS nw_now, (SELECT SUM(net_contribution) FROM daily_account_valuation WHERE valuation_date = b.d_now) AS contrib_now, (SELECT SUM(total_value) FROM daily_account_valuation WHERE valuation_date = b.d_ago) AS nw_ago, (SELECT SUM(net_contribution) FROM daily_account_valuation WHERE valuation_date = b.d_ago) AS contrib_ago FROM bounds b) SELECT ((nw_now - nw_ago) - (contrib_now - contrib_ago)) AS gain_12mo FROM agg" "rawSql": "WITH latest AS (SELECT DISTINCT ON (d.account_id) d.account_id, d.valuation_date AS d_now, d.total_value AS nw_now, d.net_contribution AS contrib_now FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id ORDER BY d.account_id, d.valuation_date DESC), ago AS (SELECT DISTINCT ON (l.account_id) l.account_id, d.total_value AS nw_ago, d.net_contribution AS contrib_ago FROM latest l JOIN daily_account_valuation d ON d.account_id = l.account_id AND d.valuation_date <= l.d_now - INTERVAL '12 months' ORDER BY l.account_id, d.valuation_date DESC), agg AS (SELECT (SELECT SUM(nw_now) FROM latest) AS nw_now, (SELECT SUM(contrib_now) FROM latest) AS contrib_now, (SELECT SUM(nw_ago) FROM ago) AS nw_ago, (SELECT SUM(contrib_ago) FROM ago) AS contrib_ago) SELECT ((nw_now - nw_ago) - (contrib_now - contrib_ago)) AS gain_12mo FROM agg"
} }
] ]
}, },
@ -610,7 +610,7 @@
"rawQuery": true, "rawQuery": true,
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawSql": "WITH yearly AS (SELECT EXTRACT(YEAR FROM valuation_date)::int AS yr, valuation_date, SUM(total_value) AS nw, SUM(net_contribution) AS contrib FROM daily_account_valuation GROUP BY valuation_date), endpoints AS (SELECT yr, (array_agg(nw ORDER BY valuation_date ASC))[1] AS nw_start, (array_agg(nw ORDER BY valuation_date DESC))[1] AS nw_end, (array_agg(contrib ORDER BY valuation_date ASC))[1] AS contrib_start, (array_agg(contrib ORDER BY valuation_date DESC))[1] AS contrib_end FROM yearly GROUP BY yr) SELECT yr::text AS year, ROUND((((nw_end - nw_start - (contrib_end - contrib_start)) / NULLIF(nw_start + 0.5 * (contrib_end - contrib_start), 0)) * 100)::numeric, 2) AS return_pct FROM endpoints WHERE (nw_start + 0.5 * (contrib_end - contrib_start)) > 0 ORDER BY yr" "rawSql": "WITH active_accounts AS (SELECT COUNT(*) AS n FROM accounts), complete_dates AS (SELECT d.valuation_date FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id GROUP BY d.valuation_date HAVING COUNT(*) >= (SELECT n FROM active_accounts)), yearly AS (SELECT EXTRACT(YEAR FROM valuation_date)::int AS yr, valuation_date, SUM(total_value) AS nw, SUM(net_contribution) AS contrib FROM daily_account_valuation WHERE valuation_date IN (SELECT valuation_date FROM complete_dates) GROUP BY valuation_date), endpoints AS (SELECT yr, (array_agg(nw ORDER BY valuation_date ASC))[1] AS nw_start, (array_agg(nw ORDER BY valuation_date DESC))[1] AS nw_end, (array_agg(contrib ORDER BY valuation_date ASC))[1] AS contrib_start, (array_agg(contrib ORDER BY valuation_date DESC))[1] AS contrib_end FROM yearly GROUP BY yr) SELECT yr::text AS year, ROUND((((nw_end - nw_start - (contrib_end - contrib_start)) / NULLIF(nw_start + 0.5 * (contrib_end - contrib_start), 0)) * 100)::numeric, 2) AS return_pct FROM endpoints WHERE (nw_start + 0.5 * (contrib_end - contrib_start)) > 0 ORDER BY yr"
} }
] ]
}, },
@ -676,7 +676,7 @@
"rawQuery": true, "rawQuery": true,
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawSql": "WITH yearly AS (SELECT EXTRACT(YEAR FROM valuation_date)::int AS yr, valuation_date, SUM(total_value) AS nw, SUM(net_contribution) AS contrib FROM daily_account_valuation GROUP BY valuation_date), endpoints AS (SELECT yr, (array_agg(nw ORDER BY valuation_date ASC))[1] AS nw_start, (array_agg(nw ORDER BY valuation_date DESC))[1] AS nw_end, (array_agg(contrib ORDER BY valuation_date ASC))[1] AS contrib_start, (array_agg(contrib ORDER BY valuation_date DESC))[1] AS contrib_end FROM yearly GROUP BY yr) SELECT yr::text AS year, ROUND((contrib_end - contrib_start)::numeric, 0) AS contributions, ROUND((nw_end - nw_start - (contrib_end - contrib_start))::numeric, 0) AS market_gain FROM endpoints ORDER BY yr" "rawSql": "WITH active_accounts AS (SELECT COUNT(*) AS n FROM accounts), complete_dates AS (SELECT d.valuation_date FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id GROUP BY d.valuation_date HAVING COUNT(*) >= (SELECT n FROM active_accounts)), yearly AS (SELECT EXTRACT(YEAR FROM valuation_date)::int AS yr, valuation_date, SUM(total_value) AS nw, SUM(net_contribution) AS contrib FROM daily_account_valuation WHERE valuation_date IN (SELECT valuation_date FROM complete_dates) GROUP BY valuation_date), endpoints AS (SELECT yr, (array_agg(nw ORDER BY valuation_date ASC))[1] AS nw_start, (array_agg(nw ORDER BY valuation_date DESC))[1] AS nw_end, (array_agg(contrib ORDER BY valuation_date ASC))[1] AS contrib_start, (array_agg(contrib ORDER BY valuation_date DESC))[1] AS contrib_end FROM yearly GROUP BY yr) SELECT yr::text AS year, ROUND((contrib_end - contrib_start)::numeric, 0) AS contributions, ROUND((nw_end - nw_start - (contrib_end - contrib_start))::numeric, 0) AS market_gain FROM endpoints ORDER BY yr"
} }
] ]
}, },
@ -728,7 +728,7 @@
"rawQuery": true, "rawQuery": true,
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawSql": "SELECT a.name AS account, ROUND(((d.total_value - d.net_contribution) / NULLIF(d.net_contribution, 0) * 100)::numeric, 2) AS roi_pct FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id WHERE d.valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation) AND d.net_contribution > 0 ORDER BY roi_pct DESC" "rawSql": "WITH latest AS (SELECT DISTINCT ON (d.account_id) a.name, d.total_value, d.net_contribution FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id ORDER BY d.account_id, d.valuation_date DESC) SELECT name AS account, ROUND(((total_value - net_contribution) / NULLIF(net_contribution, 0) * 100)::numeric, 2) AS roi_pct FROM latest WHERE net_contribution > 0 ORDER BY roi_pct DESC"
} }
] ]
} }