[payslip-ingest] ActualBudget payroll sync CronJob + Panel 14 (Phase C)
Wires the daily ActualBudget deposit sync from the payslip-ingest app into K8s as a CronJob, and adds dashboard Panel 14 to overlay bank deposits against payslip net_pay. CronJob: actualbudget-payroll-sync in payslip-ingest namespace, runs 02:00 UTC. Calls `python -m payslip_ingest sync-meta-deposits`, which hits budget-http-api-viktor in the actualbudget namespace and upserts matching Meta payroll deposits into payslip_ingest.external_meta_deposits. ExternalSecret extended with three new Vault keys: - ACTUALBUDGET_API_KEY (same as actualbudget-http-api-viktor's env API_KEY) - ACTUALBUDGET_ENCRYPTION_PASSWORD (Viktor's budget password) - ACTUALBUDGET_BUDGET_SYNC_ID (Viktor's sync_id) These must be seeded at secret/payslip-ingest in Vault before the CronJob will run — it'll CrashLoop on missing env vars otherwise. First run can be triggered on demand via `kubectl -n payslip-ingest create job --from=cronjob/actualbudget-payroll-sync initial-sync`. Panel 14 plots monthly SUM(external_meta_deposits.amount) vs SUM(payslip.net_pay), plus a delta bar series — |delta| > £50 flags likely parser drift on net_pay. Part of: code-860
This commit is contained in:
Viktor Barzin
parent
ef53053ae6
commit
1c0e1bcdde
2 changed files with 239 additions and 1 deletions
|
|
@ -2033,6 +2033,131 @@
|
|||
"rawSql": "WITH latest AS (SELECT DISTINCT ON (tax_year, employer_paye_ref) tax_year, employer_paye_ref, snapshot_date, gross_pay, income_tax, ni_contributions FROM hmrc_sync.tax_year_snapshot ORDER BY tax_year, employer_paye_ref, snapshot_date DESC), summed AS (SELECT tax_year, COALESCE(SUM(gross_pay), 0) AS sum_gross, COALESCE(SUM(income_tax), 0) AS sum_tax, COALESCE(SUM(national_insurance), 0) AS sum_ni FROM payslip_ingest.payslip GROUP BY tax_year) SELECT l.tax_year, l.employer_paye_ref, l.snapshot_date, l.gross_pay AS hmrc_gross, s.sum_gross AS computed_gross, (l.gross_pay - s.sum_gross) AS delta_gross, l.income_tax AS hmrc_tax, s.sum_tax AS computed_tax, (l.income_tax - s.sum_tax) AS delta_tax, l.ni_contributions AS hmrc_ni, s.sum_ni AS computed_ni, (l.ni_contributions - s.sum_ni) AS delta_ni FROM latest l LEFT JOIN summed s ON s.tax_year = l.tax_year ORDER BY l.tax_year DESC"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Meta payroll: bank deposit vs payslip net pay",
|
||||
"description": "Cross-check between ActualBudget bank deposits (META/FACEBOOK payee) and payslip net_pay. |delta| > \u00a350 flags likely parser or bank-sync drift. Synced daily 02:00 UTC.",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "grafana-postgresql-datasource",
|
||||
"uid": "payslips-pg"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 160
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"unit": "currencyGBP",
|
||||
"custom": {
|
||||
"axisPlacement": "auto",
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"lineWidth": 2,
|
||||
"pointSize": 6,
|
||||
"showPoints": "always",
|
||||
"spanNulls": false,
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "deposit_sum"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "green"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "Bank deposit (ActualBudget)"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "payslip_net_pay"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "blue"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "Payslip net pay"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "delta"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "red"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "Delta (deposit \u2212 payslip)"
|
||||
},
|
||||
{
|
||||
"id": "custom.drawStyle",
|
||||
"value": "bars"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"last"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "grafana-postgresql-datasource",
|
||||
"uid": "payslips-pg"
|
||||
},
|
||||
"rawSql": "WITH deposits AS (SELECT DATE_TRUNC('month', deposit_date)::date AS month_start, SUM(amount) AS deposit_sum FROM payslip_ingest.external_meta_deposits GROUP BY 1), payslip_net AS (SELECT DATE_TRUNC('month', pay_date)::date AS month_start, SUM(net_pay) AS payslip_net_pay FROM payslip_ingest.payslip GROUP BY 1) SELECT COALESCE(p.month_start, d.month_start) AS \"time\", d.deposit_sum, p.payslip_net_pay, COALESCE(d.deposit_sum, 0) - COALESCE(p.payslip_net_pay, 0) AS delta FROM deposits d FULL OUTER JOIN payslip_net p ON p.month_start = d.month_start WHERE $__timeFilter(COALESCE(p.month_start, d.month_start)) ORDER BY \"time\"",
|
||||
"format": "time_series",
|
||||
"refId": "A",
|
||||
"rawQuery": true,
|
||||
"editorMode": "code"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"refresh": "5m",
|
||||
|
|
|
|||
|
|
@ -32,7 +32,20 @@ resource "kubernetes_namespace" "payslip_ingest" {
|
|||
# Seed these manually in Vault before applying:
|
||||
# secret/paperless-ngx -> property `api_token`
|
||||
# secret/claude-agent-service -> property `api_bearer_token`
|
||||
# secret/payslip-ingest -> property `webhook_bearer_token`
|
||||
# secret/payslip-ingest -> properties:
|
||||
# - `webhook_bearer_token`
|
||||
# - `actualbudget_api_key` (same value as
|
||||
# actualbudget-http-api-viktor random
|
||||
# api-key — fetch via `kubectl get pods
|
||||
# -n actualbudget -l
|
||||
# app=actualbudget-http-api-viktor -o
|
||||
# jsonpath={.items[0].spec.containers[0].env}`
|
||||
# and grep API_KEY)
|
||||
# - `actualbudget_encryption_password`
|
||||
# (same as Viktor's budget password in
|
||||
# secret/actualbudget/credentials[viktor])
|
||||
# - `actualbudget_budget_sync_id`
|
||||
# (same as Viktor's sync_id)
|
||||
resource "kubernetes_manifest" "external_secret" {
|
||||
manifest = {
|
||||
apiVersion = "external-secrets.io/v1beta1"
|
||||
|
|
@ -79,6 +92,27 @@ resource "kubernetes_manifest" "external_secret" {
|
|||
property = "webhook_bearer_token"
|
||||
}
|
||||
},
|
||||
{
|
||||
secretKey = "ACTUALBUDGET_API_KEY"
|
||||
remoteRef = {
|
||||
key = "payslip-ingest"
|
||||
property = "actualbudget_api_key"
|
||||
}
|
||||
},
|
||||
{
|
||||
secretKey = "ACTUALBUDGET_ENCRYPTION_PASSWORD"
|
||||
remoteRef = {
|
||||
key = "payslip-ingest"
|
||||
property = "actualbudget_encryption_password"
|
||||
}
|
||||
},
|
||||
{
|
||||
secretKey = "ACTUALBUDGET_BUDGET_SYNC_ID"
|
||||
remoteRef = {
|
||||
key = "payslip-ingest"
|
||||
property = "actualbudget_budget_sync_id"
|
||||
}
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -288,6 +322,85 @@ resource "kubernetes_service" "payslip_ingest" {
|
|||
}
|
||||
}
|
||||
|
||||
# Daily sync of Meta payroll deposits from ActualBudget's http-api sidecar.
|
||||
# Populates payslip_ingest.external_meta_deposits so Panel 14 can overlay bank
|
||||
# deposits against payslip.net_pay — catches parser drift on net_pay.
|
||||
resource "kubernetes_cron_job_v1" "actualbudget_payroll_sync" {
|
||||
metadata {
|
||||
name = "actualbudget-payroll-sync"
|
||||
namespace = kubernetes_namespace.payslip_ingest.metadata[0].name
|
||||
}
|
||||
spec {
|
||||
schedule = "0 2 * * *"
|
||||
concurrency_policy = "Forbid"
|
||||
successful_jobs_history_limit = 3
|
||||
failed_jobs_history_limit = 5
|
||||
starting_deadline_seconds = 300
|
||||
|
||||
job_template {
|
||||
metadata {
|
||||
labels = local.labels
|
||||
}
|
||||
spec {
|
||||
backoff_limit = 1
|
||||
ttl_seconds_after_finished = 86400
|
||||
template {
|
||||
metadata {
|
||||
labels = local.labels
|
||||
}
|
||||
spec {
|
||||
restart_policy = "OnFailure"
|
||||
image_pull_secrets {
|
||||
name = "registry-credentials"
|
||||
}
|
||||
container {
|
||||
name = "sync"
|
||||
image = local.image
|
||||
command = ["python", "-m", "payslip_ingest", "sync-meta-deposits"]
|
||||
|
||||
env_from {
|
||||
secret_ref {
|
||||
name = "payslip-ingest-secrets"
|
||||
}
|
||||
}
|
||||
env_from {
|
||||
secret_ref {
|
||||
name = "payslip-ingest-db-creds"
|
||||
}
|
||||
}
|
||||
|
||||
env {
|
||||
name = "ACTUALBUDGET_HTTP_API_URL"
|
||||
value = "http://budget-http-api-viktor.actualbudget.svc.cluster.local"
|
||||
}
|
||||
|
||||
resources {
|
||||
requests = {
|
||||
cpu = "50m"
|
||||
memory = "128Mi"
|
||||
}
|
||||
limits = {
|
||||
memory = "256Mi"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
lifecycle {
|
||||
# KYVERNO_LIFECYCLE_V1
|
||||
ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config]
|
||||
}
|
||||
|
||||
depends_on = [
|
||||
kubernetes_manifest.external_secret,
|
||||
kubernetes_manifest.db_external_secret,
|
||||
]
|
||||
}
|
||||
|
||||
# Plan-time read of the ESO-created K8s Secret for Grafana datasource password.
|
||||
# First apply: -target=kubernetes_manifest.db_external_secret first so the Secret exists.
|
||||
data "kubernetes_secret" "payslip_ingest_db_creds" {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue