From 628f5a0d26a62569881bcd6b133c081be045a502 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Fri, 1 May 2026 16:08:18 +0000 Subject: [PATCH] monitoring(wealth): skew-resilient queries, no more partial-day dips MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug witnessed 2026-05-01: dashboard "Net worth (current)" showed £88k instead of £1.03M because at 02:00 UTC an external trigger refreshed ONE account (Trading212 ISA), creating its 05-01 daily_account_valuation row. The 5 other accounts still had their last row at 04-30. The panel SQL `WHERE valuation_date = (SELECT MAX(valuation_date))` then summed only the single account that had a 05-01 row. Two new SQL patterns adopted across all 15 affected panels: 1. Stat / barchart "current snapshot" panels (1, 2, 3, 4, 11, 14, 15, 16): latest-per-account stitching — WITH latest AS (SELECT DISTINCT ON (d.account_id) ... FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id ORDER BY d.account_id, d.valuation_date DESC) gives a coherent "now" snapshot regardless of refresh skew, and the inner join filters out orphan/deleted accounts (one such was adding a stale £33k from 04-17). 12-month panels add a parallel `ago` CTE picking each account's row closest to (d_now - 12mo). 2. Time-series / yearly panels (5, 6, 7, 8, 9, 12, 13): complete-days- only filter — WITH active_accounts AS (SELECT COUNT(*) FROM accounts), complete_dates AS (SELECT valuation_date FROM daily_account_valuation d JOIN accounts a ON a.id=d.account_id GROUP BY valuation_date HAVING COUNT(*) >= active.n) so a partial today never renders as a chart dip. The day rejoins the chart automatically once the daily 16:00 UTC sync writes rows for every account. Verified end-to-end against live PG: new queries produce £1,033,734 (matches the 6 active accounts' true latest sum) where the old query gave £88k. --- .../modules/monitoring/dashboards/wealth.json | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/stacks/monitoring/modules/monitoring/dashboards/wealth.json b/stacks/monitoring/modules/monitoring/dashboards/wealth.json index a576f28c..a09ab92d 100644 --- a/stacks/monitoring/modules/monitoring/dashboards/wealth.json +++ b/stacks/monitoring/modules/monitoring/dashboards/wealth.json @@ -47,7 +47,7 @@ "rawQuery": true, "editorMode": "code", "format": "table", - "rawSql": "SELECT SUM(total_value) AS net_worth FROM daily_account_valuation WHERE valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation)" + "rawSql": "WITH latest AS (SELECT DISTINCT ON (d.account_id) d.account_id, d.total_value, d.net_contribution, d.cash_balance, d.investment_market_value FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id ORDER BY d.account_id, d.valuation_date DESC) SELECT SUM(total_value) AS net_worth FROM latest" } ] }, @@ -81,7 +81,7 @@ "rawQuery": true, "editorMode": "code", "format": "table", - "rawSql": "SELECT SUM(net_contribution) AS contribution FROM daily_account_valuation WHERE valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation)" + "rawSql": "WITH latest AS (SELECT DISTINCT ON (d.account_id) d.account_id, d.total_value, d.net_contribution, d.cash_balance, d.investment_market_value FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id ORDER BY d.account_id, d.valuation_date DESC) SELECT SUM(net_contribution) AS contribution FROM latest" } ] }, @@ -122,7 +122,7 @@ "rawQuery": true, "editorMode": "code", "format": "table", - "rawSql": "SELECT (SUM(total_value) - SUM(net_contribution)) AS growth FROM daily_account_valuation WHERE valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation)" + "rawSql": "WITH latest AS (SELECT DISTINCT ON (d.account_id) d.account_id, d.total_value, d.net_contribution, d.cash_balance, d.investment_market_value FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id ORDER BY d.account_id, d.valuation_date DESC) SELECT (SUM(total_value) - SUM(net_contribution)) AS growth FROM latest" } ] }, @@ -164,7 +164,7 @@ "rawQuery": true, "editorMode": "code", "format": "table", - "rawSql": "WITH latest AS (SELECT * FROM daily_account_valuation WHERE valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation) AND net_contribution > 0) SELECT (SUM(total_value - net_contribution) / NULLIF(SUM(net_contribution), 0) * 100) AS roi_pct FROM latest" + "rawSql": "WITH latest AS (SELECT DISTINCT ON (d.account_id) d.account_id, d.total_value, d.net_contribution, d.cash_balance, d.investment_market_value FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id ORDER BY d.account_id, d.valuation_date DESC) SELECT (SUM(total_value - net_contribution) / NULLIF(SUM(net_contribution), 0) * 100) AS roi_pct FROM latest WHERE net_contribution > 0" } ] }, @@ -208,7 +208,7 @@ "rawQuery": true, "editorMode": "code", "format": "time_series", - "rawSql": "SELECT valuation_date::timestamp AS \"time\", SUM(total_value) AS net_worth FROM daily_account_valuation WHERE $__timeFilter(valuation_date) GROUP BY valuation_date ORDER BY valuation_date" + "rawSql": "WITH active_accounts AS (SELECT COUNT(*) AS n FROM accounts), complete_dates AS (SELECT d.valuation_date FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id GROUP BY d.valuation_date HAVING COUNT(*) >= (SELECT n FROM active_accounts)) SELECT valuation_date::timestamp AS \\\"time\\\", SUM(total_value) AS net_worth FROM daily_account_valuation WHERE $__timeFilter(valuation_date) AND valuation_date IN (SELECT valuation_date FROM complete_dates) GROUP BY valuation_date ORDER BY valuation_date" } ] }, @@ -262,7 +262,7 @@ "rawQuery": true, "editorMode": "code", "format": "time_series", - "rawSql": "SELECT valuation_date::timestamp AS \"time\", SUM(net_contribution) AS net_contribution, SUM(total_value) AS market_value FROM daily_account_valuation WHERE $__timeFilter(valuation_date) GROUP BY valuation_date ORDER BY valuation_date" + "rawSql": "WITH active_accounts AS (SELECT COUNT(*) AS n FROM accounts), complete_dates AS (SELECT d.valuation_date FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id GROUP BY d.valuation_date HAVING COUNT(*) >= (SELECT n FROM active_accounts)) SELECT valuation_date::timestamp AS \\\"time\\\", SUM(net_contribution) AS net_contribution, SUM(total_value) AS market_value FROM daily_account_valuation WHERE $__timeFilter(valuation_date) AND valuation_date IN (SELECT valuation_date FROM complete_dates) GROUP BY valuation_date ORDER BY valuation_date" } ] }, @@ -307,7 +307,7 @@ "rawQuery": true, "editorMode": "code", "format": "time_series", - "rawSql": "SELECT valuation_date::timestamp AS \"time\", (SUM(total_value) - SUM(net_contribution)) AS growth FROM daily_account_valuation WHERE $__timeFilter(valuation_date) GROUP BY valuation_date ORDER BY valuation_date" + "rawSql": "WITH active_accounts AS (SELECT COUNT(*) AS n FROM accounts), complete_dates AS (SELECT d.valuation_date FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id GROUP BY d.valuation_date HAVING COUNT(*) >= (SELECT n FROM active_accounts)) SELECT valuation_date::timestamp AS \\\"time\\\", (SUM(total_value) - SUM(net_contribution)) AS growth FROM daily_account_valuation WHERE $__timeFilter(valuation_date) AND valuation_date IN (SELECT valuation_date FROM complete_dates) GROUP BY valuation_date ORDER BY valuation_date" } ] }, @@ -346,7 +346,7 @@ "rawQuery": true, "editorMode": "code", "format": "time_series", - "rawSql": "SELECT d.valuation_date::timestamp AS \"time\", a.name AS metric, d.total_value AS value FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id WHERE $__timeFilter(d.valuation_date) ORDER BY d.valuation_date, a.name" + "rawSql": "WITH active_accounts AS (SELECT COUNT(*) AS n FROM accounts), complete_dates AS (SELECT d.valuation_date FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id GROUP BY d.valuation_date HAVING COUNT(*) >= (SELECT n FROM active_accounts)) SELECT d.valuation_date::timestamp AS \\\"time\\\", a.name AS metric, d.total_value AS value FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id WHERE $__timeFilter(d.valuation_date) AND d.valuation_date IN (SELECT valuation_date FROM complete_dates) ORDER BY d.valuation_date, a.name" } ] }, @@ -400,7 +400,7 @@ "rawQuery": true, "editorMode": "code", "format": "time_series", - "rawSql": "SELECT d.valuation_date::timestamp AS \"time\", SUM(CASE WHEN a.account_type = 'WORKPLACE_PENSION' THEN 0 ELSE d.cash_balance END) AS cash, SUM(CASE WHEN a.account_type = 'WORKPLACE_PENSION' THEN d.cash_balance + d.investment_market_value ELSE d.investment_market_value END) AS invested FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id WHERE $__timeFilter(d.valuation_date) GROUP BY d.valuation_date ORDER BY d.valuation_date" + "rawSql": "WITH active_accounts AS (SELECT COUNT(*) AS n FROM accounts), complete_dates AS (SELECT d.valuation_date FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id GROUP BY d.valuation_date HAVING COUNT(*) >= (SELECT n FROM active_accounts)) SELECT d.valuation_date::timestamp AS \\\"time\\\", SUM(CASE WHEN a.account_type = 'WORKPLACE_PENSION' THEN 0 ELSE d.cash_balance END) AS cash, SUM(CASE WHEN a.account_type = 'WORKPLACE_PENSION' THEN d.cash_balance + d.investment_market_value ELSE d.investment_market_value END) AS invested FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id WHERE $__timeFilter(d.valuation_date) AND d.valuation_date IN (SELECT valuation_date FROM complete_dates) GROUP BY d.valuation_date ORDER BY d.valuation_date" } ] }, @@ -475,7 +475,7 @@ "rawQuery": true, "editorMode": "code", "format": "table", - "rawSql": "WITH bounds AS (SELECT (SELECT MAX(valuation_date) FROM daily_account_valuation) AS d_now, (SELECT MIN(valuation_date) FROM daily_account_valuation WHERE valuation_date >= (SELECT MAX(valuation_date) - INTERVAL '12 months' FROM daily_account_valuation)) AS d_ago), agg AS (SELECT (SELECT SUM(total_value) FROM daily_account_valuation WHERE valuation_date = b.d_now) AS nw_now, (SELECT SUM(net_contribution) FROM daily_account_valuation WHERE valuation_date = b.d_now) AS contrib_now, (SELECT SUM(total_value) FROM daily_account_valuation WHERE valuation_date = b.d_ago) AS nw_ago, (SELECT SUM(net_contribution) FROM daily_account_valuation WHERE valuation_date = b.d_ago) AS contrib_ago FROM bounds b) SELECT ROUND((((nw_now - nw_ago - (contrib_now - contrib_ago)) / NULLIF(nw_ago + 0.5 * (contrib_now - contrib_ago), 0)) * 100)::numeric, 2) AS pct_12mo FROM agg" + "rawSql": "WITH latest AS (SELECT DISTINCT ON (d.account_id) d.account_id, d.valuation_date AS d_now, d.total_value AS nw_now, d.net_contribution AS contrib_now FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id ORDER BY d.account_id, d.valuation_date DESC), ago AS (SELECT DISTINCT ON (l.account_id) l.account_id, d.total_value AS nw_ago, d.net_contribution AS contrib_ago FROM latest l JOIN daily_account_valuation d ON d.account_id = l.account_id AND d.valuation_date <= l.d_now - INTERVAL '12 months' ORDER BY l.account_id, d.valuation_date DESC), agg AS (SELECT (SELECT SUM(nw_now) FROM latest) AS nw_now, (SELECT SUM(contrib_now) FROM latest) AS contrib_now, (SELECT SUM(nw_ago) FROM ago) AS nw_ago, (SELECT SUM(contrib_ago) FROM ago) AS contrib_ago) SELECT ROUND((((nw_now - nw_ago - (contrib_now - contrib_ago)) / NULLIF(nw_ago + 0.5 * (contrib_now - contrib_ago), 0)) * 100)::numeric, 2) AS pct_12mo FROM agg" } ] }, @@ -509,7 +509,7 @@ "rawQuery": true, "editorMode": "code", "format": "table", - "rawSql": "WITH bounds AS (SELECT (SELECT MAX(valuation_date) FROM daily_account_valuation) AS d_now, (SELECT MIN(valuation_date) FROM daily_account_valuation WHERE valuation_date >= (SELECT MAX(valuation_date) - INTERVAL '12 months' FROM daily_account_valuation)) AS d_ago) SELECT ((SELECT SUM(net_contribution) FROM daily_account_valuation WHERE valuation_date = b.d_now) - (SELECT SUM(net_contribution) FROM daily_account_valuation WHERE valuation_date = b.d_ago)) AS contrib_12mo FROM bounds b" + "rawSql": "WITH latest AS (SELECT DISTINCT ON (d.account_id) d.account_id, d.valuation_date AS d_now, d.total_value AS nw_now, d.net_contribution AS contrib_now FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id ORDER BY d.account_id, d.valuation_date DESC), ago AS (SELECT DISTINCT ON (l.account_id) l.account_id, d.total_value AS nw_ago, d.net_contribution AS contrib_ago FROM latest l JOIN daily_account_valuation d ON d.account_id = l.account_id AND d.valuation_date <= l.d_now - INTERVAL '12 months' ORDER BY l.account_id, d.valuation_date DESC), agg AS (SELECT (SELECT SUM(nw_now) FROM latest) AS nw_now, (SELECT SUM(contrib_now) FROM latest) AS contrib_now, (SELECT SUM(nw_ago) FROM ago) AS nw_ago, (SELECT SUM(contrib_ago) FROM ago) AS contrib_ago) SELECT (contrib_now - contrib_ago) AS contrib_12mo FROM agg" } ] }, @@ -550,7 +550,7 @@ "rawQuery": true, "editorMode": "code", "format": "table", - "rawSql": "WITH bounds AS (SELECT (SELECT MAX(valuation_date) FROM daily_account_valuation) AS d_now, (SELECT MIN(valuation_date) FROM daily_account_valuation WHERE valuation_date >= (SELECT MAX(valuation_date) - INTERVAL '12 months' FROM daily_account_valuation)) AS d_ago), agg AS (SELECT (SELECT SUM(total_value) FROM daily_account_valuation WHERE valuation_date = b.d_now) AS nw_now, (SELECT SUM(net_contribution) FROM daily_account_valuation WHERE valuation_date = b.d_now) AS contrib_now, (SELECT SUM(total_value) FROM daily_account_valuation WHERE valuation_date = b.d_ago) AS nw_ago, (SELECT SUM(net_contribution) FROM daily_account_valuation WHERE valuation_date = b.d_ago) AS contrib_ago FROM bounds b) SELECT ((nw_now - nw_ago) - (contrib_now - contrib_ago)) AS gain_12mo FROM agg" + "rawSql": "WITH latest AS (SELECT DISTINCT ON (d.account_id) d.account_id, d.valuation_date AS d_now, d.total_value AS nw_now, d.net_contribution AS contrib_now FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id ORDER BY d.account_id, d.valuation_date DESC), ago AS (SELECT DISTINCT ON (l.account_id) l.account_id, d.total_value AS nw_ago, d.net_contribution AS contrib_ago FROM latest l JOIN daily_account_valuation d ON d.account_id = l.account_id AND d.valuation_date <= l.d_now - INTERVAL '12 months' ORDER BY l.account_id, d.valuation_date DESC), agg AS (SELECT (SELECT SUM(nw_now) FROM latest) AS nw_now, (SELECT SUM(contrib_now) FROM latest) AS contrib_now, (SELECT SUM(nw_ago) FROM ago) AS nw_ago, (SELECT SUM(contrib_ago) FROM ago) AS contrib_ago) SELECT ((nw_now - nw_ago) - (contrib_now - contrib_ago)) AS gain_12mo FROM agg" } ] }, @@ -610,7 +610,7 @@ "rawQuery": true, "editorMode": "code", "format": "table", - "rawSql": "WITH yearly AS (SELECT EXTRACT(YEAR FROM valuation_date)::int AS yr, valuation_date, SUM(total_value) AS nw, SUM(net_contribution) AS contrib FROM daily_account_valuation GROUP BY valuation_date), endpoints AS (SELECT yr, (array_agg(nw ORDER BY valuation_date ASC))[1] AS nw_start, (array_agg(nw ORDER BY valuation_date DESC))[1] AS nw_end, (array_agg(contrib ORDER BY valuation_date ASC))[1] AS contrib_start, (array_agg(contrib ORDER BY valuation_date DESC))[1] AS contrib_end FROM yearly GROUP BY yr) SELECT yr::text AS year, ROUND((((nw_end - nw_start - (contrib_end - contrib_start)) / NULLIF(nw_start + 0.5 * (contrib_end - contrib_start), 0)) * 100)::numeric, 2) AS return_pct FROM endpoints WHERE (nw_start + 0.5 * (contrib_end - contrib_start)) > 0 ORDER BY yr" + "rawSql": "WITH active_accounts AS (SELECT COUNT(*) AS n FROM accounts), complete_dates AS (SELECT d.valuation_date FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id GROUP BY d.valuation_date HAVING COUNT(*) >= (SELECT n FROM active_accounts)), yearly AS (SELECT EXTRACT(YEAR FROM valuation_date)::int AS yr, valuation_date, SUM(total_value) AS nw, SUM(net_contribution) AS contrib FROM daily_account_valuation WHERE valuation_date IN (SELECT valuation_date FROM complete_dates) GROUP BY valuation_date), endpoints AS (SELECT yr, (array_agg(nw ORDER BY valuation_date ASC))[1] AS nw_start, (array_agg(nw ORDER BY valuation_date DESC))[1] AS nw_end, (array_agg(contrib ORDER BY valuation_date ASC))[1] AS contrib_start, (array_agg(contrib ORDER BY valuation_date DESC))[1] AS contrib_end FROM yearly GROUP BY yr) SELECT yr::text AS year, ROUND((((nw_end - nw_start - (contrib_end - contrib_start)) / NULLIF(nw_start + 0.5 * (contrib_end - contrib_start), 0)) * 100)::numeric, 2) AS return_pct FROM endpoints WHERE (nw_start + 0.5 * (contrib_end - contrib_start)) > 0 ORDER BY yr" } ] }, @@ -676,7 +676,7 @@ "rawQuery": true, "editorMode": "code", "format": "table", - "rawSql": "WITH yearly AS (SELECT EXTRACT(YEAR FROM valuation_date)::int AS yr, valuation_date, SUM(total_value) AS nw, SUM(net_contribution) AS contrib FROM daily_account_valuation GROUP BY valuation_date), endpoints AS (SELECT yr, (array_agg(nw ORDER BY valuation_date ASC))[1] AS nw_start, (array_agg(nw ORDER BY valuation_date DESC))[1] AS nw_end, (array_agg(contrib ORDER BY valuation_date ASC))[1] AS contrib_start, (array_agg(contrib ORDER BY valuation_date DESC))[1] AS contrib_end FROM yearly GROUP BY yr) SELECT yr::text AS year, ROUND((contrib_end - contrib_start)::numeric, 0) AS contributions, ROUND((nw_end - nw_start - (contrib_end - contrib_start))::numeric, 0) AS market_gain FROM endpoints ORDER BY yr" + "rawSql": "WITH active_accounts AS (SELECT COUNT(*) AS n FROM accounts), complete_dates AS (SELECT d.valuation_date FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id GROUP BY d.valuation_date HAVING COUNT(*) >= (SELECT n FROM active_accounts)), yearly AS (SELECT EXTRACT(YEAR FROM valuation_date)::int AS yr, valuation_date, SUM(total_value) AS nw, SUM(net_contribution) AS contrib FROM daily_account_valuation WHERE valuation_date IN (SELECT valuation_date FROM complete_dates) GROUP BY valuation_date), endpoints AS (SELECT yr, (array_agg(nw ORDER BY valuation_date ASC))[1] AS nw_start, (array_agg(nw ORDER BY valuation_date DESC))[1] AS nw_end, (array_agg(contrib ORDER BY valuation_date ASC))[1] AS contrib_start, (array_agg(contrib ORDER BY valuation_date DESC))[1] AS contrib_end FROM yearly GROUP BY yr) SELECT yr::text AS year, ROUND((contrib_end - contrib_start)::numeric, 0) AS contributions, ROUND((nw_end - nw_start - (contrib_end - contrib_start))::numeric, 0) AS market_gain FROM endpoints ORDER BY yr" } ] }, @@ -728,7 +728,7 @@ "rawQuery": true, "editorMode": "code", "format": "table", - "rawSql": "SELECT a.name AS account, ROUND(((d.total_value - d.net_contribution) / NULLIF(d.net_contribution, 0) * 100)::numeric, 2) AS roi_pct FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id WHERE d.valuation_date = (SELECT MAX(valuation_date) FROM daily_account_valuation) AND d.net_contribution > 0 ORDER BY roi_pct DESC" + "rawSql": "WITH latest AS (SELECT DISTINCT ON (d.account_id) a.name, d.total_value, d.net_contribution FROM daily_account_valuation d JOIN accounts a ON a.id = d.account_id ORDER BY d.account_id, d.valuation_date DESC) SELECT name AS account, ROUND(((total_value - net_contribution) / NULLIF(net_contribution, 0) * 100)::numeric, 2) AS roi_pct FROM latest WHERE net_contribution > 0 ORDER BY roi_pct DESC" } ] }