[payslip-ingest] ActualBudget payroll sync CronJob + Panel 14 (Phase C)
Wires the daily ActualBudget deposit sync from the payslip-ingest app into K8s as a CronJob, and adds dashboard Panel 14 to overlay bank deposits against payslip net_pay. CronJob: actualbudget-payroll-sync in payslip-ingest namespace, runs 02:00 UTC. Calls `python -m payslip_ingest sync-meta-deposits`, which hits budget-http-api-viktor in the actualbudget namespace and upserts matching Meta payroll deposits into payslip_ingest.external_meta_deposits. ExternalSecret extended with three new Vault keys: - ACTUALBUDGET_API_KEY (same as actualbudget-http-api-viktor's env API_KEY) - ACTUALBUDGET_ENCRYPTION_PASSWORD (Viktor's budget password) - ACTUALBUDGET_BUDGET_SYNC_ID (Viktor's sync_id) These must be seeded at secret/payslip-ingest in Vault before the CronJob will run — it'll CrashLoop on missing env vars otherwise. First run can be triggered on demand via `kubectl -n payslip-ingest create job --from=cronjob/actualbudget-payroll-sync initial-sync`. Panel 14 plots monthly SUM(external_meta_deposits.amount) vs SUM(payslip.net_pay), plus a delta bar series — |delta| > £50 flags likely parser drift on net_pay. Part of: code-860
This commit is contained in:
parent
ef53053ae6
commit
1c0e1bcdde
2 changed files with 239 additions and 1 deletions
|
|
@ -2033,6 +2033,131 @@
|
||||||
"rawSql": "WITH latest AS (SELECT DISTINCT ON (tax_year, employer_paye_ref) tax_year, employer_paye_ref, snapshot_date, gross_pay, income_tax, ni_contributions FROM hmrc_sync.tax_year_snapshot ORDER BY tax_year, employer_paye_ref, snapshot_date DESC), summed AS (SELECT tax_year, COALESCE(SUM(gross_pay), 0) AS sum_gross, COALESCE(SUM(income_tax), 0) AS sum_tax, COALESCE(SUM(national_insurance), 0) AS sum_ni FROM payslip_ingest.payslip GROUP BY tax_year) SELECT l.tax_year, l.employer_paye_ref, l.snapshot_date, l.gross_pay AS hmrc_gross, s.sum_gross AS computed_gross, (l.gross_pay - s.sum_gross) AS delta_gross, l.income_tax AS hmrc_tax, s.sum_tax AS computed_tax, (l.income_tax - s.sum_tax) AS delta_tax, l.ni_contributions AS hmrc_ni, s.sum_ni AS computed_ni, (l.ni_contributions - s.sum_ni) AS delta_ni FROM latest l LEFT JOIN summed s ON s.tax_year = l.tax_year ORDER BY l.tax_year DESC"
|
"rawSql": "WITH latest AS (SELECT DISTINCT ON (tax_year, employer_paye_ref) tax_year, employer_paye_ref, snapshot_date, gross_pay, income_tax, ni_contributions FROM hmrc_sync.tax_year_snapshot ORDER BY tax_year, employer_paye_ref, snapshot_date DESC), summed AS (SELECT tax_year, COALESCE(SUM(gross_pay), 0) AS sum_gross, COALESCE(SUM(income_tax), 0) AS sum_tax, COALESCE(SUM(national_insurance), 0) AS sum_ni FROM payslip_ingest.payslip GROUP BY tax_year) SELECT l.tax_year, l.employer_paye_ref, l.snapshot_date, l.gross_pay AS hmrc_gross, s.sum_gross AS computed_gross, (l.gross_pay - s.sum_gross) AS delta_gross, l.income_tax AS hmrc_tax, s.sum_tax AS computed_tax, (l.income_tax - s.sum_tax) AS delta_tax, l.ni_contributions AS hmrc_ni, s.sum_ni AS computed_ni, (l.ni_contributions - s.sum_ni) AS delta_ni FROM latest l LEFT JOIN summed s ON s.tax_year = l.tax_year ORDER BY l.tax_year DESC"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 14,
|
||||||
|
"title": "Meta payroll: bank deposit vs payslip net pay",
|
||||||
|
"description": "Cross-check between ActualBudget bank deposits (META/FACEBOOK payee) and payslip net_pay. |delta| > \u00a350 flags likely parser or bank-sync drift. Synced daily 02:00 UTC.",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "grafana-postgresql-datasource",
|
||||||
|
"uid": "payslips-pg"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 9,
|
||||||
|
"w": 24,
|
||||||
|
"x": 0,
|
||||||
|
"y": 160
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": {
|
||||||
|
"mode": "palette-classic"
|
||||||
|
},
|
||||||
|
"unit": "currencyGBP",
|
||||||
|
"custom": {
|
||||||
|
"axisPlacement": "auto",
|
||||||
|
"drawStyle": "line",
|
||||||
|
"fillOpacity": 0,
|
||||||
|
"lineWidth": 2,
|
||||||
|
"pointSize": 6,
|
||||||
|
"showPoints": "always",
|
||||||
|
"spanNulls": false,
|
||||||
|
"thresholdsStyle": {
|
||||||
|
"mode": "off"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": [
|
||||||
|
{
|
||||||
|
"matcher": {
|
||||||
|
"id": "byName",
|
||||||
|
"options": "deposit_sum"
|
||||||
|
},
|
||||||
|
"properties": [
|
||||||
|
{
|
||||||
|
"id": "color",
|
||||||
|
"value": {
|
||||||
|
"mode": "fixed",
|
||||||
|
"fixedColor": "green"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "displayName",
|
||||||
|
"value": "Bank deposit (ActualBudget)"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"matcher": {
|
||||||
|
"id": "byName",
|
||||||
|
"options": "payslip_net_pay"
|
||||||
|
},
|
||||||
|
"properties": [
|
||||||
|
{
|
||||||
|
"id": "color",
|
||||||
|
"value": {
|
||||||
|
"mode": "fixed",
|
||||||
|
"fixedColor": "blue"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "displayName",
|
||||||
|
"value": "Payslip net pay"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"matcher": {
|
||||||
|
"id": "byName",
|
||||||
|
"options": "delta"
|
||||||
|
},
|
||||||
|
"properties": [
|
||||||
|
{
|
||||||
|
"id": "color",
|
||||||
|
"value": {
|
||||||
|
"mode": "fixed",
|
||||||
|
"fixedColor": "red"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "displayName",
|
||||||
|
"value": "Delta (deposit \u2212 payslip)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "custom.drawStyle",
|
||||||
|
"value": "bars"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"calcs": [
|
||||||
|
"last"
|
||||||
|
],
|
||||||
|
"displayMode": "table",
|
||||||
|
"placement": "bottom"
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "desc"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "grafana-postgresql-datasource",
|
||||||
|
"uid": "payslips-pg"
|
||||||
|
},
|
||||||
|
"rawSql": "WITH deposits AS (SELECT DATE_TRUNC('month', deposit_date)::date AS month_start, SUM(amount) AS deposit_sum FROM payslip_ingest.external_meta_deposits GROUP BY 1), payslip_net AS (SELECT DATE_TRUNC('month', pay_date)::date AS month_start, SUM(net_pay) AS payslip_net_pay FROM payslip_ingest.payslip GROUP BY 1) SELECT COALESCE(p.month_start, d.month_start) AS \"time\", d.deposit_sum, p.payslip_net_pay, COALESCE(d.deposit_sum, 0) - COALESCE(p.payslip_net_pay, 0) AS delta FROM deposits d FULL OUTER JOIN payslip_net p ON p.month_start = d.month_start WHERE $__timeFilter(COALESCE(p.month_start, d.month_start)) ORDER BY \"time\"",
|
||||||
|
"format": "time_series",
|
||||||
|
"refId": "A",
|
||||||
|
"rawQuery": true,
|
||||||
|
"editorMode": "code"
|
||||||
|
}
|
||||||
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"refresh": "5m",
|
"refresh": "5m",
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,20 @@ resource "kubernetes_namespace" "payslip_ingest" {
|
||||||
# Seed these manually in Vault before applying:
|
# Seed these manually in Vault before applying:
|
||||||
# secret/paperless-ngx -> property `api_token`
|
# secret/paperless-ngx -> property `api_token`
|
||||||
# secret/claude-agent-service -> property `api_bearer_token`
|
# secret/claude-agent-service -> property `api_bearer_token`
|
||||||
# secret/payslip-ingest -> property `webhook_bearer_token`
|
# secret/payslip-ingest -> properties:
|
||||||
|
# - `webhook_bearer_token`
|
||||||
|
# - `actualbudget_api_key` (same value as
|
||||||
|
# actualbudget-http-api-viktor random
|
||||||
|
# api-key — fetch via `kubectl get pods
|
||||||
|
# -n actualbudget -l
|
||||||
|
# app=actualbudget-http-api-viktor -o
|
||||||
|
# jsonpath={.items[0].spec.containers[0].env}`
|
||||||
|
# and grep API_KEY)
|
||||||
|
# - `actualbudget_encryption_password`
|
||||||
|
# (same as Viktor's budget password in
|
||||||
|
# secret/actualbudget/credentials[viktor])
|
||||||
|
# - `actualbudget_budget_sync_id`
|
||||||
|
# (same as Viktor's sync_id)
|
||||||
resource "kubernetes_manifest" "external_secret" {
|
resource "kubernetes_manifest" "external_secret" {
|
||||||
manifest = {
|
manifest = {
|
||||||
apiVersion = "external-secrets.io/v1beta1"
|
apiVersion = "external-secrets.io/v1beta1"
|
||||||
|
|
@ -79,6 +92,27 @@ resource "kubernetes_manifest" "external_secret" {
|
||||||
property = "webhook_bearer_token"
|
property = "webhook_bearer_token"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
secretKey = "ACTUALBUDGET_API_KEY"
|
||||||
|
remoteRef = {
|
||||||
|
key = "payslip-ingest"
|
||||||
|
property = "actualbudget_api_key"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
secretKey = "ACTUALBUDGET_ENCRYPTION_PASSWORD"
|
||||||
|
remoteRef = {
|
||||||
|
key = "payslip-ingest"
|
||||||
|
property = "actualbudget_encryption_password"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
secretKey = "ACTUALBUDGET_BUDGET_SYNC_ID"
|
||||||
|
remoteRef = {
|
||||||
|
key = "payslip-ingest"
|
||||||
|
property = "actualbudget_budget_sync_id"
|
||||||
|
}
|
||||||
|
},
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -288,6 +322,85 @@ resource "kubernetes_service" "payslip_ingest" {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Daily sync of Meta payroll deposits from ActualBudget's http-api sidecar.
|
||||||
|
# Populates payslip_ingest.external_meta_deposits so Panel 14 can overlay bank
|
||||||
|
# deposits against payslip.net_pay — catches parser drift on net_pay.
|
||||||
|
resource "kubernetes_cron_job_v1" "actualbudget_payroll_sync" {
|
||||||
|
metadata {
|
||||||
|
name = "actualbudget-payroll-sync"
|
||||||
|
namespace = kubernetes_namespace.payslip_ingest.metadata[0].name
|
||||||
|
}
|
||||||
|
spec {
|
||||||
|
schedule = "0 2 * * *"
|
||||||
|
concurrency_policy = "Forbid"
|
||||||
|
successful_jobs_history_limit = 3
|
||||||
|
failed_jobs_history_limit = 5
|
||||||
|
starting_deadline_seconds = 300
|
||||||
|
|
||||||
|
job_template {
|
||||||
|
metadata {
|
||||||
|
labels = local.labels
|
||||||
|
}
|
||||||
|
spec {
|
||||||
|
backoff_limit = 1
|
||||||
|
ttl_seconds_after_finished = 86400
|
||||||
|
template {
|
||||||
|
metadata {
|
||||||
|
labels = local.labels
|
||||||
|
}
|
||||||
|
spec {
|
||||||
|
restart_policy = "OnFailure"
|
||||||
|
image_pull_secrets {
|
||||||
|
name = "registry-credentials"
|
||||||
|
}
|
||||||
|
container {
|
||||||
|
name = "sync"
|
||||||
|
image = local.image
|
||||||
|
command = ["python", "-m", "payslip_ingest", "sync-meta-deposits"]
|
||||||
|
|
||||||
|
env_from {
|
||||||
|
secret_ref {
|
||||||
|
name = "payslip-ingest-secrets"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
env_from {
|
||||||
|
secret_ref {
|
||||||
|
name = "payslip-ingest-db-creds"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
env {
|
||||||
|
name = "ACTUALBUDGET_HTTP_API_URL"
|
||||||
|
value = "http://budget-http-api-viktor.actualbudget.svc.cluster.local"
|
||||||
|
}
|
||||||
|
|
||||||
|
resources {
|
||||||
|
requests = {
|
||||||
|
cpu = "50m"
|
||||||
|
memory = "128Mi"
|
||||||
|
}
|
||||||
|
limits = {
|
||||||
|
memory = "256Mi"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
lifecycle {
|
||||||
|
# KYVERNO_LIFECYCLE_V1
|
||||||
|
ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config]
|
||||||
|
}
|
||||||
|
|
||||||
|
depends_on = [
|
||||||
|
kubernetes_manifest.external_secret,
|
||||||
|
kubernetes_manifest.db_external_secret,
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
# Plan-time read of the ESO-created K8s Secret for Grafana datasource password.
|
# Plan-time read of the ESO-created K8s Secret for Grafana datasource password.
|
||||||
# First apply: -target=kubernetes_manifest.db_external_secret first so the Secret exists.
|
# First apply: -target=kubernetes_manifest.db_external_secret first so the Secret exists.
|
||||||
data "kubernetes_secret" "payslip_ingest_db_creds" {
|
data "kubernetes_secret" "payslip_ingest_db_creds" {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue