From 9277d71d817b40f619e8ef95bc39c1041696cac6 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Sun, 24 May 2026 15:32:22 +0000 Subject: [PATCH] nfs-mirror: append transferred files to offsite-sync manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 1 of offsite-sync-backup is incremental on non-monthly days, driven by /mnt/backup/.changed-files which only daily-backup wrote to. nfs-mirror's writes were therefore invisible to Step 1 until the next monthly --delete pass — which would *also* wipe data pre-positioned on Synology pve-backup/ (e.g. the in-place btrfs rename we just did to relocate ~160G of NFS subtrees from /Backup/Viki/nfs// to /Backup/Viki/pve-backup//). Fix: snapshot a timestamp before rsync, then after rsync use `find -newer $STAMP -type f -printf '%P\n'` to enumerate every file nfs-mirror created/modified and append to the manifest. Paths are relative to /mnt/backup/ (matches Step 1 --files-from expectation). State files are excluded. The current in-flight first run started before this patch was deployed, so its writes won't auto-populate the manifest — a one-off manual backfill will be done after it completes. --- .claude/CLAUDE.md | 3 ++- scripts/nfs-mirror.sh | 23 ++++++++++++++++++++++- stacks/blog/.terraform.lock.hcl | 8 ++++++++ stacks/blog/backend.tf | 2 +- stacks/blog/providers.tf | 12 ++++++++++++ stacks/forgejo/.terraform.lock.hcl | 23 +++++++++++++++++++++++ stacks/forgejo/backend.tf | 2 +- stacks/forgejo/providers.tf | 12 ++++++++++++ stacks/n8n/.terraform.lock.hcl | 8 ++++++++ stacks/n8n/backend.tf | 2 +- stacks/n8n/providers.tf | 12 ++++++++++++ stacks/openclaw/backend.tf | 2 +- stacks/recruiter-responder/terragrunt.hcl | 2 +- stacks/url/.terraform.lock.hcl | 16 ++++++++++++++++ stacks/url/backend.tf | 2 +- stacks/url/providers.tf | 16 ++++++++++++++++ 16 files changed, 137 insertions(+), 8 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index e2a57542..d100656d 100755 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -175,7 +175,8 @@ Choose storage class based on workload type: **Default for sensitive data is proxmox-lvm-encrypted.** Use plain `proxmox-lvm` only for non-sensitive workloads. Use NFS when you need RWX, backup pipeline integration, or it's a large shared media library. **NFS server:** -- **Proxmox host** (192.168.1.127): Sole NFS for all workloads. HDD at `/srv/nfs` (ext4 thin LV `pve/nfs-data`, 1TB). SSD at `/srv/nfs-ssd` (ext4 LV `ssd/nfs-ssd-data`, 100GB). Exports use `async,insecure` options (`async` — safe with UPS + Vault Raft replication + databases on block storage; `insecure` — pfSense NATs source ports >1024 between VLANs). +- **Proxmox host** (192.168.1.127): Sole NFS for all workloads. HDD at `/srv/nfs` (ext4 thin LV `pve/nfs-data`, 3 TB). SSD at `/srv/nfs-ssd` (ext4 LV `ssd/nfs-ssd-data`, 100GB). Exports use `async,insecure` options (`async` — safe with UPS + Vault Raft replication + databases on block storage; `insecure` — pfSense NATs source ports >1024 between VLANs). +- **Nextcloud as NFS browser**: Nextcloud (`nextcloud.viktorbarzin.me`) mounts the PVE NFS roots (`/srv/nfs`, `/srv/nfs-ssd`) inside the NC pod at `/mnt/pve-nfs` + `/mnt/pve-nfs-ssd`. Surfaced to users via two ACL patterns: (1) admin-only root browsers `PVE NFS Pool` + `PVE NFS-SSD Pool` (scoped to NC group `admin`); (2) per-archive mounts (e.g. `/anca-elements`) with `applicable_users` set to the owners. ACL is at the mount level via `occ files_external:applicable` — Files Access Control is NOT used (NC 30/31's workflow engine lacks FilePath / UserId checks). Manifest lives in `kubernetes_config_map_v1.nextcloud_external_storage_manifest` (`stacks/nextcloud/external_storage.tf`); a one-shot K8s Job applies it idempotently. - **`nfs-truenas` StorageClass**: Historical name retained only because SC names are immutable on PVs (48 bound PVs reference it — renaming would require mass PV churn, not worth it). Now points to the Proxmox host, identical to `nfs-proxmox`. TrueNAS (VM 9000, 10.0.10.15) operationally decommissioned 2026-04-13; VM still exists in stopped state on PVE pending user decision on deletion. **Migration note**: CSI PV `volumeAttributes` are immutable — cannot update NFS server in place. New PV/PVC pairs required (convention: append `-host` to PV name). diff --git a/scripts/nfs-mirror.sh b/scripts/nfs-mirror.sh index 9bbd0496..7d74b9bb 100644 --- a/scripts/nfs-mirror.sh +++ b/scripts/nfs-mirror.sh @@ -34,6 +34,12 @@ SRC=/srv/nfs/ DST=/mnt/backup/ LOG=/var/log/nfs-mirror.log LOCKFILE=/run/nfs-mirror.lock +# Manifest of files changed under /mnt/backup since the last offsite-sync. +# offsite-sync-backup Step 1 reads this and rsyncs the listed files to Synology +# pve-backup/ on its next daily run. Without populating it, nfs-mirror's writes +# would only reach Synology via the monthly full sync (1st-7th of month), and +# the monthly --delete pass would also wipe any pre-positioned data. +MANIFEST=/mnt/backup/.changed-files PUSHGATEWAY="${NFS_MIRROR_PUSHGATEWAY:-http://10.0.20.100:30091}" PUSHGATEWAY_JOB=nfs-mirror @@ -85,8 +91,10 @@ EOF } KILLED="" +STAMP="" cleanup() { rm -f "$LOCKFILE" + [ -n "$STAMP" ] && rm -f "$STAMP" if [ -n "$KILLED" ]; then push_metrics 2 0 # status=2 = aborted fi @@ -105,6 +113,10 @@ mountpoint -q /mnt/backup || { log "FATAL: /mnt/backup not mounted"; push_metric log "=== mirror starting: $SRC → $DST ===" log "skip: immich, frigate, prometheus, loki, ollama, audiblez, *-backup, temp" +# Marker file used to identify files written by this rsync run, so we can append +# their paths to the offsite-sync manifest. Touch BEFORE rsync; `find -newer` AFTER. +STAMP=$(mktemp) + RSYNC_RC=0 rsync \ -rlt --delete -H \ @@ -116,7 +128,16 @@ rsync \ DST_BYTES=$(df -B1 --output=used /mnt/backup | tail -1) if [ "$RSYNC_RC" -eq 0 ]; then - log "=== mirror complete; /mnt/backup used: $(df -h --output=used /mnt/backup | tail -1 | tr -d ' ') ===" + # Capture files that rsync created/modified and feed them to the offsite-sync + # manifest so daily Step 1 incremental picks them up tomorrow morning. + NEW_COUNT=$(find /mnt/backup -newer "$STAMP" -type f \ + ! -path '/mnt/backup/.changed-files' \ + ! -path '/mnt/backup/.lv-pvc-mapping.json' \ + ! -path '/mnt/backup/.nfs-changes.log' \ + ! -path '/mnt/backup/.last-offsite-sync' \ + -printf '%P\n' 2>/dev/null | tee -a "$MANIFEST" | wc -l) + log "=== mirror complete; ${NEW_COUNT} files added to offsite manifest ===" + log "/mnt/backup used: $(df -h --output=used /mnt/backup | tail -1 | tr -d ' ')" push_metrics 0 "$DST_BYTES" else log "=== mirror failed: rsync exited $RSYNC_RC ===" diff --git a/stacks/blog/.terraform.lock.hcl b/stacks/blog/.terraform.lock.hcl index fabbc047..522ec0cc 100644 --- a/stacks/blog/.terraform.lock.hcl +++ b/stacks/blog/.terraform.lock.hcl @@ -24,6 +24,14 @@ provider "registry.terraform.io/cloudflare/cloudflare" { ] } +provider "registry.terraform.io/gavinbunney/kubectl" { + version = "1.19.0" + constraints = "~> 1.14" + hashes = [ + "h1:9QkxPjp0x5FZFfJbE+B7hBOoads9gmdfj9aYu5N4Sfc=", + ] +} + provider "registry.terraform.io/goauthentik/authentik" { version = "2024.12.1" constraints = "~> 2024.10" diff --git a/stacks/blog/backend.tf b/stacks/blog/backend.tf index 20a8e977..90463a32 100644 --- a/stacks/blog/backend.tf +++ b/stacks/blog/backend.tf @@ -1,7 +1,7 @@ # Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa terraform { backend "pg" { - conn_str = "postgres://terraform_state:ts7DGcKmTTY-5ujz4mhh@10.0.20.200:5432/terraform_state?sslmode=disable" + conn_str = "postgres://terraform_state:LicuZK1nVl4ILE5HF-A9@10.0.20.200:5432/terraform_state?sslmode=disable" schema_name = "blog" } } diff --git a/stacks/blog/providers.tf b/stacks/blog/providers.tf index 012af700..d5469984 100644 --- a/stacks/blog/providers.tf +++ b/stacks/blog/providers.tf @@ -13,6 +13,13 @@ terraform { source = "goauthentik/authentik" version = "~> 2024.10" } + # kubectl (gavinbunney) — workaround for hashicorp/kubernetes + # `kubernetes_manifest` panics on Kyverno CRDs. See beads code-e2dp. + # Declared for all stacks but only used where opted-in. + kubectl = { + source = "gavinbunney/kubectl" + version = "~> 1.14" + } } } @@ -35,3 +42,8 @@ provider "vault" { address = "https://vault.viktorbarzin.me" skip_child_token = true } + +provider "kubectl" { + config_path = var.kube_config_path + load_config_file = true +} diff --git a/stacks/forgejo/.terraform.lock.hcl b/stacks/forgejo/.terraform.lock.hcl index fabbc047..7c813e0f 100644 --- a/stacks/forgejo/.terraform.lock.hcl +++ b/stacks/forgejo/.terraform.lock.hcl @@ -24,6 +24,29 @@ provider "registry.terraform.io/cloudflare/cloudflare" { ] } +provider "registry.terraform.io/gavinbunney/kubectl" { + version = "1.19.0" + constraints = "~> 1.14" + hashes = [ + "h1:9QkxPjp0x5FZFfJbE+B7hBOoads9gmdfj9aYu5N4Sfc=", + "zh:1dec8766336ac5b00b3d8f62e3fff6390f5f60699c9299920fc9861a76f00c71", + "zh:43f101b56b58d7fead6a511728b4e09f7c41dc2e3963f59cf1c146c4767c6cb7", + "zh:4c4fbaa44f60e722f25cc05ee11dfaec282893c5c0ffa27bc88c382dbfbaa35c", + "zh:51dd23238b7b677b8a1abbfcc7deec53ffa5ec79e58e3b54d6be334d3d01bc0e", + "zh:5afc2ebc75b9d708730dbabdc8f94dd559d7f2fc5a31c5101358bd8d016916ba", + "zh:6be6e72d4663776390a82a37e34f7359f726d0120df622f4a2b46619338a168e", + "zh:72642d5fcf1e3febb6e5d4ae7b592bb9ff3cb220af041dbda893588e4bf30c0c", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a1da03e3239867b35812ee031a1060fed6e8d8e458e2eaca48b5dd51b35f56f7", + "zh:b98b6a6728fe277fcd133bdfa7237bd733eae233f09653523f14460f608f8ba2", + "zh:bb8b071d0437f4767695c6158a3cb70df9f52e377c67019971d888b99147511f", + "zh:dc89ce4b63bfef708ec29c17e85ad0232a1794336dc54dd88c3ba0b77e764f71", + "zh:dd7dd18f1f8218c6cd19592288fde32dccc743cde05b9feeb2883f37c2ff4b4e", + "zh:ec4bd5ab3872dedb39fe528319b4bba609306e12ee90971495f109e142d66310", + "zh:f610ead42f724c82f5463e0e71fa735a11ffb6101880665d93f48b4a67b9ad82", + ] +} + provider "registry.terraform.io/goauthentik/authentik" { version = "2024.12.1" constraints = "~> 2024.10" diff --git a/stacks/forgejo/backend.tf b/stacks/forgejo/backend.tf index 7498aa14..9e982ee3 100644 --- a/stacks/forgejo/backend.tf +++ b/stacks/forgejo/backend.tf @@ -1,7 +1,7 @@ # Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa terraform { backend "pg" { - conn_str = "postgres://terraform_state:ts7DGcKmTTY-5ujz4mhh@10.0.20.200:5432/terraform_state?sslmode=disable" + conn_str = "postgres://terraform_state:ZCcWMOLCTqb0aV-XyTAZ@10.0.20.200:5432/terraform_state?sslmode=disable" schema_name = "forgejo" } } diff --git a/stacks/forgejo/providers.tf b/stacks/forgejo/providers.tf index 012af700..d5469984 100644 --- a/stacks/forgejo/providers.tf +++ b/stacks/forgejo/providers.tf @@ -13,6 +13,13 @@ terraform { source = "goauthentik/authentik" version = "~> 2024.10" } + # kubectl (gavinbunney) — workaround for hashicorp/kubernetes + # `kubernetes_manifest` panics on Kyverno CRDs. See beads code-e2dp. + # Declared for all stacks but only used where opted-in. + kubectl = { + source = "gavinbunney/kubectl" + version = "~> 1.14" + } } } @@ -35,3 +42,8 @@ provider "vault" { address = "https://vault.viktorbarzin.me" skip_child_token = true } + +provider "kubectl" { + config_path = var.kube_config_path + load_config_file = true +} diff --git a/stacks/n8n/.terraform.lock.hcl b/stacks/n8n/.terraform.lock.hcl index 9fc3ef9c..0fc9b894 100644 --- a/stacks/n8n/.terraform.lock.hcl +++ b/stacks/n8n/.terraform.lock.hcl @@ -24,6 +24,14 @@ provider "registry.terraform.io/cloudflare/cloudflare" { ] } +provider "registry.terraform.io/gavinbunney/kubectl" { + version = "1.19.0" + constraints = "~> 1.14" + hashes = [ + "h1:9QkxPjp0x5FZFfJbE+B7hBOoads9gmdfj9aYu5N4Sfc=", + ] +} + provider "registry.terraform.io/goauthentik/authentik" { version = "2024.12.1" constraints = "~> 2024.10" diff --git a/stacks/n8n/backend.tf b/stacks/n8n/backend.tf index 35237740..fae81b79 100644 --- a/stacks/n8n/backend.tf +++ b/stacks/n8n/backend.tf @@ -1,7 +1,7 @@ # Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa terraform { backend "pg" { - conn_str = "postgres://terraform_state:ts7DGcKmTTY-5ujz4mhh@10.0.20.200:5432/terraform_state?sslmode=disable" + conn_str = "postgres://terraform_state:LicuZK1nVl4ILE5HF-A9@10.0.20.200:5432/terraform_state?sslmode=disable" schema_name = "n8n" } } diff --git a/stacks/n8n/providers.tf b/stacks/n8n/providers.tf index 012af700..d5469984 100644 --- a/stacks/n8n/providers.tf +++ b/stacks/n8n/providers.tf @@ -13,6 +13,13 @@ terraform { source = "goauthentik/authentik" version = "~> 2024.10" } + # kubectl (gavinbunney) — workaround for hashicorp/kubernetes + # `kubernetes_manifest` panics on Kyverno CRDs. See beads code-e2dp. + # Declared for all stacks but only used where opted-in. + kubectl = { + source = "gavinbunney/kubectl" + version = "~> 1.14" + } } } @@ -35,3 +42,8 @@ provider "vault" { address = "https://vault.viktorbarzin.me" skip_child_token = true } + +provider "kubectl" { + config_path = var.kube_config_path + load_config_file = true +} diff --git a/stacks/openclaw/backend.tf b/stacks/openclaw/backend.tf index 2df80c50..834cb915 100644 --- a/stacks/openclaw/backend.tf +++ b/stacks/openclaw/backend.tf @@ -1,7 +1,7 @@ # Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa terraform { backend "pg" { - conn_str = "postgres://terraform_state:ZCcWMOLCTqb0aV-XyTAZ@10.0.20.200:5432/terraform_state?sslmode=disable" + conn_str = "postgres://terraform_state:LicuZK1nVl4ILE5HF-A9@10.0.20.200:5432/terraform_state?sslmode=disable" schema_name = "openclaw" } } diff --git a/stacks/recruiter-responder/terragrunt.hcl b/stacks/recruiter-responder/terragrunt.hcl index 9f09ba90..769db480 100644 --- a/stacks/recruiter-responder/terragrunt.hcl +++ b/stacks/recruiter-responder/terragrunt.hcl @@ -19,5 +19,5 @@ dependency "external-secrets" { inputs = { # Override per-deploy in CI / commit. - image_tag = "83ffd9fa" + image_tag = "2162e09d" } diff --git a/stacks/url/.terraform.lock.hcl b/stacks/url/.terraform.lock.hcl index a1ca7484..522ec0cc 100644 --- a/stacks/url/.terraform.lock.hcl +++ b/stacks/url/.terraform.lock.hcl @@ -24,6 +24,22 @@ provider "registry.terraform.io/cloudflare/cloudflare" { ] } +provider "registry.terraform.io/gavinbunney/kubectl" { + version = "1.19.0" + constraints = "~> 1.14" + hashes = [ + "h1:9QkxPjp0x5FZFfJbE+B7hBOoads9gmdfj9aYu5N4Sfc=", + ] +} + +provider "registry.terraform.io/goauthentik/authentik" { + version = "2024.12.1" + constraints = "~> 2024.10" + hashes = [ + "h1:roBMd+gi+TGgikH/bMzEI8JfvJiMAQWt+8FmokCrQIs=", + ] +} + provider "registry.terraform.io/hashicorp/helm" { version = "3.1.1" hashes = [ diff --git a/stacks/url/backend.tf b/stacks/url/backend.tf index 64796590..4c6aed16 100644 --- a/stacks/url/backend.tf +++ b/stacks/url/backend.tf @@ -1,7 +1,7 @@ # Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa terraform { backend "pg" { - conn_str = "postgres://terraform_state:SBlzGxotNUN6HH9d0S-m@10.0.20.200:5432/terraform_state?sslmode=disable" + conn_str = "postgres://terraform_state:LicuZK1nVl4ILE5HF-A9@10.0.20.200:5432/terraform_state?sslmode=disable" schema_name = "url" } } diff --git a/stacks/url/providers.tf b/stacks/url/providers.tf index b337a2e9..d5469984 100644 --- a/stacks/url/providers.tf +++ b/stacks/url/providers.tf @@ -9,6 +9,17 @@ terraform { source = "cloudflare/cloudflare" version = "~> 4" } + authentik = { + source = "goauthentik/authentik" + version = "~> 2024.10" + } + # kubectl (gavinbunney) — workaround for hashicorp/kubernetes + # `kubernetes_manifest` panics on Kyverno CRDs. See beads code-e2dp. + # Declared for all stacks but only used where opted-in. + kubectl = { + source = "gavinbunney/kubectl" + version = "~> 1.14" + } } } @@ -31,3 +42,8 @@ provider "vault" { address = "https://vault.viktorbarzin.me" skip_child_token = true } + +provider "kubectl" { + config_path = var.kube_config_path + load_config_file = true +}