backup: retire anca-elements-mirror + anca-elements-sync.sh

Both subsumed by nfs-mirror (deployed earlier this session) — see
commit 4d756be4. anca-elements-sync.sh is now dead code because its
upstream (Synology /volume1/Backup/Anca/Elements) was deleted today
once the sda mirror was parity-verified (109,624 files /
827,480,937,976 bytes equal both sides). PVE NFS is the source of
truth for the archive from here on.

Final script inventory on the PVE host (down from 6 to 4):
- /usr/local/bin/daily-backup           (block PVCs + sqlite + pfsense)
- /usr/local/bin/lvm-pvc-snapshot       (snapshot management)
- /usr/local/bin/nfs-mirror             (NFS local mirror to sda)
- /usr/local/bin/offsite-sync-backup    (sda + bypass-list NFS to Synology)

scripts/anca-elements-mirror.service

@@ -1,15 +0,0 @@
-[Unit]
-Description=Mirror /srv/nfs/anca-elements to /mnt/backup (single-disk-failure protection)
-After=network-online.target local-fs.target
-Wants=network-online.target
-
-[Service]
-Type=oneshot
-ExecStart=/usr/local/bin/anca-elements-mirror
-StandardOutput=journal
-StandardError=journal
-SyslogIdentifier=anca-elements-mirror
-# Big sustained IO — don't compete with foreground services.
-Nice=10
-IOSchedulingClass=idle
-TimeoutStartSec=18000

scripts/anca-elements-mirror.sh

@@ -1,82 +0,0 @@
-#!/usr/bin/env bash
-# anca-elements-mirror — single-disk-failure mirror of /srv/nfs/anca-elements → /mnt/backup
-#
-# Deploy to PVE host at /usr/local/bin/anca-elements-mirror.
-# Schedule: weekly Mon 04:00 via systemd timer (anca-elements-mirror.timer).
-#
-# WHY: /srv/nfs/anca-elements lives on the sdc thin pool. Synology no longer
-# holds the original (deleted after this mirror was verified). sda /mnt/backup
-# is the only other local disk with room (~770G) — this gives us a single-
-# disk-failure copy. No offsite for this archive (intentional, see backup-dr.md).
-#
-# Idempotent: `rsync -aH --delete` makes destination match source exactly.
-# Re-runs only transfer changed files.
-
-set -euo pipefail
-
-SRC=/srv/nfs/anca-elements
-DST=/mnt/backup/anca-elements
-LOG=/var/log/anca-elements-mirror.log
-LOCKFILE=/run/anca-elements-mirror.lock
-PUSHGATEWAY="${ANCA_MIRROR_PUSHGATEWAY:-http://10.0.20.100:30091}"
-PUSHGATEWAY_JOB=anca-elements-mirror
-
-log()  { echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] $*" | tee -a "$LOG"; }
-warn() { log "WARN: $*"; }
-
-push_metrics() {
-    local status="${1:-0}" bytes="${2:-0}"
-    cat <<EOF | curl -s --connect-timeout 5 --max-time 10 --data-binary @- "${PUSHGATEWAY}/metrics/job/${PUSHGATEWAY_JOB}" 2>/dev/null || true
-anca_elements_mirror_last_run_timestamp $(date +%s)
-anca_elements_mirror_last_status ${status}
-anca_elements_mirror_bytes ${bytes}
-EOF
-}
-
-KILLED=""
-cleanup() {
-    rm -f "$LOCKFILE"
-    if [ -n "$KILLED" ]; then
-        push_metrics 2 0  # status=2 → aborted (matches lvm-pvc-snapshot convention)
-    fi
-}
-trap cleanup EXIT
-trap 'KILLED=1; exit 143' TERM INT
-
-if ! ( set -o noclobber; echo $$ > "$LOCKFILE" ) 2>/dev/null; then
-    log "FATAL: another instance running (pid $(cat "$LOCKFILE" 2>/dev/null || echo unknown))"
-    exit 1
-fi
-
-mountpoint -q /mnt/backup || { log "FATAL: /mnt/backup not mounted"; push_metrics 1 0; exit 1; }
-[ -d "$SRC" ]              || { log "FATAL: source $SRC missing"; push_metrics 1 0; exit 1; }
-
-mkdir -p "$DST"
-
-log "=== mirror starting: $SRC → $DST ==="
-SRC_SIZE_GB=$(du -sBG "$SRC" 2>/dev/null | awk '{print $1}')
-log "source size: $SRC_SIZE_GB"
-
-# -aH preserves hardlinks (probably none here, cheap insurance).
-# --info=stats2 emits a final transfer summary into the log.
-# --no-perms / --no-owner / --no-group: source has root:www-data 2775 and
-# we don't need to perfectly preserve those on the mirror copy — dest will
-# inherit /mnt/backup's defaults. (Symmetric with anca-elements-sync.sh's
-# choice when copying FROM Synology.)
-RSYNC_RC=0
-rsync \
-    -rlt --delete -H \
-    --no-perms --no-owner --no-group \
-    --info=stats2 \
-    "$SRC/" "$DST/" 2>&1 | tee -a "$LOG" || RSYNC_RC=${PIPESTATUS[0]}
-
-DST_BYTES=$(du -sb "$DST" 2>/dev/null | awk '{print $1}')
-
-if [ "$RSYNC_RC" -eq 0 ]; then
-    log "=== mirror complete; dest size: $(du -sh "$DST" | cut -f1) ==="
-    push_metrics 0 "$DST_BYTES"
-else
-    log "=== mirror failed: rsync exited $RSYNC_RC ==="
-    push_metrics 1 "$DST_BYTES"
-    exit "$RSYNC_RC"
-fi

scripts/anca-elements-mirror.timer

@@ -1,10 +0,0 @@
-[Unit]
-Description=Weekly anca-elements mirror to /mnt/backup
-
-[Timer]
-OnCalendar=Mon *-*-* 04:00:00
-Persistent=true
-RandomizedDelaySec=15min
-
-[Install]
-WantedBy=timers.target

scripts/anca-elements-sync.sh

@@ -1,112 +0,0 @@
-#!/usr/bin/env bash
-# anca-elements-sync.sh — copy Anca's WD-Elements backup from Synology to PVE NFS
-#
-# Usage:
-#   /usr/local/bin/anca-elements-sync.sh
-#
-# Idempotent: re-running after a successful sync is a no-op (only the dry-run
-# verification runs, which reports "sync verified clean" immediately).
-#
-# Resumable: if fpsync was interrupted, resume with:
-#   fpsync -r /var/tmp/fpsync \
-#     -n 4 -s 4G \
-#     -o "-lptgoD -H --no-perms --no-owner --no-group --exclude=@eaDir/ --exclude=*@synoeastream --exclude=.DS_Store --exclude=Thumbs.db" \
-#     /mnt/synology-backup/Anca/Elements/ /srv/nfs/anca-elements/
-#
-# NOTE: fpsync -o = rsync options override (what we want)
-#       fpsync -O = fpart partition options override (NOT rsync)
-# NOTE: Do NOT use -a or -r in fpsync rsync options — fpsync handles
-#       recursion via fpart; -r causes fpsync to warn and skip the slab.
-#
-# Log: /var/log/anca-elements-sync.log
-
-set -euo pipefail
-
-LOG=/var/log/anca-elements-sync.log
-SRC_HOST=192.168.1.13
-SRC_EXPORT=/volume1/Backup
-SRC_SUBPATH=Anca/Elements
-MOUNT_POINT=/mnt/synology-backup
-DEST=/srv/nfs/anca-elements
-
-log() {
-    echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] $*" | tee -a "$LOG"
-}
-
-# ── 1. Ensure destination + mount-point directories exist ────────────────────
-log "Step 1: ensuring directories"
-mkdir -p "$DEST" "$MOUNT_POINT"
-
-# ── 2. NFS-mount Synology read-only (skip if already mounted) ───────────────
-MOUNTED_HERE=0
-if mountpoint -q "$MOUNT_POINT"; then
-    log "Step 2: $MOUNT_POINT already mounted — skipping"
-else
-    log "Step 2: mounting ${SRC_HOST}:${SRC_EXPORT} at $MOUNT_POINT (read-only)"
-    mount -t nfs \
-        -o ro,vers=4,nolock,soft,timeo=300,retrans=2 \
-        "${SRC_HOST}:${SRC_EXPORT}" \
-        "$MOUNT_POINT"
-    MOUNTED_HERE=1
-    log "Step 2: mount successful"
-fi
-
-# ── 3. Ensure fpsync (from fpart package) is available ──────────────────────
-log "Step 3: checking for fpsync"
-if ! command -v fpsync >/dev/null 2>&1; then
-    log "Step 3: fpsync not found — installing fpart"
-    apt-get install -y fpart
-    log "Step 3: fpart installed"
-else
-    log "Step 3: fpsync already available"
-fi
-
-# ── 4. Run fpsync (4-way parallel, no compression — source is already-compressed media) ──
-log "Step 4: starting fpsync"
-log "  source : ${MOUNT_POINT}/${SRC_SUBPATH}/"
-log "  dest   : ${DEST}/"
-log "  workers: 4, slab: 4G"
-fpsync \
-    -n 4 \
-    -s 4G \
-    -o "-lptgoD -H --no-perms --no-owner --no-group --exclude=@eaDir/ --exclude=*@synoeastream --exclude=.DS_Store --exclude=Thumbs.db" \
-    "${MOUNT_POINT}/${SRC_SUBPATH}/" \
-    "${DEST}/" \
-    2>&1 | tee -a "$LOG"
-log "Step 4: fpsync completed"
-
-# ── 5. Verification dry-run ──────────────────────────────────────────────────
-log "Step 5: running dry-run verification rsync"
-VERIFY_OUT=$(rsync \
-    -rlptgoD -H --no-perms --no-owner --no-group \
-    --exclude='@eaDir/' --exclude='*@synoeastream' \
-    --exclude='.DS_Store' --exclude='Thumbs.db' \
-    -n --delete \
-    --info=progress2 \
-    --out-format='%o %f' \
-    "${MOUNT_POINT}/${SRC_SUBPATH}/" \
-    "${DEST}/" \
-    2>&1 || true)
-
-# Count lines that represent actual file changes (send / del. operations)
-CHANGE_COUNT=$(echo "$VERIFY_OUT" | grep -cE '^(send|del\.)' || true)
-
-if [ "$CHANGE_COUNT" -eq 0 ]; then
-    log "Step 5: sync verified clean — no pending changes"
-else
-    log "Step 5: WARNING — verification found ${CHANGE_COUNT} pending change(s). First 50 lines:"
-    # Use printf to avoid SIGPIPE from head closing the pipe early (set -o pipefail)
-    { echo "$VERIFY_OUT" | head -50; } >> "$LOG" 2>&1 || true
-fi
-
-# ── 6. Unmount (only if we mounted it) ──────────────────────────────────────
-if [ "$MOUNTED_HERE" -eq 1 ]; then
-    log "Step 6: unmounting $MOUNT_POINT"
-    umount "$MOUNT_POINT"
-    rmdir "$MOUNT_POINT"
-    log "Step 6: unmounted"
-else
-    log "Step 6: mount was pre-existing — leaving in place"
-fi
-
-log "Done. Final size: $(du -sh "${DEST}" | cut -f1)"

stacks/immich/.terraform.lock.hcl

@@ -24,11 +24,48 @@ provider "registry.terraform.io/cloudflare/cloudflare" {
   ]
 }
 
+provider "registry.terraform.io/gavinbunney/kubectl" {
+  version     = "1.19.0"
+  constraints = "~> 1.14"
+  hashes = [
+    "h1:9QkxPjp0x5FZFfJbE+B7hBOoads9gmdfj9aYu5N4Sfc=",
+    "zh:1dec8766336ac5b00b3d8f62e3fff6390f5f60699c9299920fc9861a76f00c71",
+    "zh:43f101b56b58d7fead6a511728b4e09f7c41dc2e3963f59cf1c146c4767c6cb7",
+    "zh:4c4fbaa44f60e722f25cc05ee11dfaec282893c5c0ffa27bc88c382dbfbaa35c",
+    "zh:51dd23238b7b677b8a1abbfcc7deec53ffa5ec79e58e3b54d6be334d3d01bc0e",
+    "zh:5afc2ebc75b9d708730dbabdc8f94dd559d7f2fc5a31c5101358bd8d016916ba",
+    "zh:6be6e72d4663776390a82a37e34f7359f726d0120df622f4a2b46619338a168e",
+    "zh:72642d5fcf1e3febb6e5d4ae7b592bb9ff3cb220af041dbda893588e4bf30c0c",
+    "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
+    "zh:a1da03e3239867b35812ee031a1060fed6e8d8e458e2eaca48b5dd51b35f56f7",
+    "zh:b98b6a6728fe277fcd133bdfa7237bd733eae233f09653523f14460f608f8ba2",
+    "zh:bb8b071d0437f4767695c6158a3cb70df9f52e377c67019971d888b99147511f",
+    "zh:dc89ce4b63bfef708ec29c17e85ad0232a1794336dc54dd88c3ba0b77e764f71",
+    "zh:dd7dd18f1f8218c6cd19592288fde32dccc743cde05b9feeb2883f37c2ff4b4e",
+    "zh:ec4bd5ab3872dedb39fe528319b4bba609306e12ee90971495f109e142d66310",
+    "zh:f610ead42f724c82f5463e0e71fa735a11ffb6101880665d93f48b4a67b9ad82",
+  ]
+}
+
 provider "registry.terraform.io/goauthentik/authentik" {
   version     = "2024.12.1"
   constraints = "~> 2024.10"
   hashes = [
     "h1:roBMd+gi+TGgikH/bMzEI8JfvJiMAQWt+8FmokCrQIs=",
+    "zh:090260dc7889ea822ec1d899344e1ee23eba5290461989c0796149c9511f2316",
+    "zh:13c2655ff824b0dc4b9bb832b5ca6d41dba97cb280330258c5fef4115e236209",
+    "zh:166a73c3a810c9c895d68a8ff968158f339f8a2c1c03e20ec9fc5ed99cc64e20",
+    "zh:203777eae1cdc711233315499643180604cff2324411b186b7cf07fdbe16f655",
+    "zh:3b2f18c9a8d28dac74dc6bbf168c946855ab9c68f053578d4630c50d5eaf30a0",
+    "zh:4822275985f6b74b6196c47112316a4252db22cf4ceaef7c9ab4c66d488abf2f",
+    "zh:53ea97562666c8a5a2f6d63d418a302a7f8ee4b7bb7da35dedaa89aa5708b7f0",
+    "zh:56b8a230901e3550c92a1d3f58ee9dafe9853f30fe4315af3ab28ae63262e15d",
+    "zh:6293ab7b1fd8206a0c853591f50186aca4a1eff117b2a773e10760a23a2c83e9",
+    "zh:9433970f79fb92d8aae3ee436db5630ab312c78b6dc9df9c1db3273a18f8aaa1",
+    "zh:95df406214f79b3b98222d7c7fe8fc319a3d90b7a9d53e1d5abbda5dfb8b9436",
+    "zh:a85880da0552a42c8f449390fbd7d8b03541d1a13e04bba9f1404fa658754260",
+    "zh:a95f6e9bd62c67e70eba1b1a14728856b9a6a28cd1e5e3be54a7718882c87e7f",
+    "zh:dd599b51c5beb34a4c6feece244fde07d2558d69929449ab1fd39a5ebe738781",
   ]
 }
 
@@ -55,6 +92,18 @@ provider "registry.terraform.io/hashicorp/kubernetes" {
   version = "3.1.0"
   hashes = [
     "h1:oodIAuFMikXNmEtil5MQgP4dfSctUBYQiGJfjbsF3NY=",
+    "zh:0215c5c60be62028c09a2f22458e89cda3ef5830a632299f1d401eb3538874b0",
+    "zh:09ebb9f442431e278a310a9423f32caf467cb4b3cad3fe59573ca71fa7b14e20",
+    "zh:0c4e5912f83bb35846ae0a9ae54fc320706ee61894cd21cc6b4181b1c5a2fa5c",
+    "zh:1678c982853ad461e65ccb5e79d585e13ed109dd47dab2a66d3a7a304faeef65",
+    "zh:1c050a5c15e330457a9c18caacf61a923c59d663e13f2962e4b32f04fef523a0",
+    "zh:2c55bcec83be58ec132c7cb0a1ac644758b800d794fdc636d53a0eada0358a3a",
+    "zh:a062bb0aa316c08d8460c66a5d68da71da40de5d3bc3b31abcf3a1a9a19650f1",
+    "zh:a26fdea0afaa9b247c73c0b42843ca51ba7db0ac2571f9d3d50dcabd20ca1b98",
+    "zh:c872c9385a78d502bf5823d61cd3bb0f9a0585030e025eb12585c83451beeaa1",
+    "zh:f180879af931182beee4c8c0d9dab62b81d86f17ddcbe3786ef4c7cec9163a4e",
+    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+    "zh:f70f5789264069e0eef06f9b5d5fde955ef7206f7d446d1ce51a4c37a3f3e02f",
   ]
 }
 

stacks/immich/backend.tf

@@ -1,7 +1,7 @@
 # Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa
 terraform {
   backend "pg" {
-    conn_str    = "postgres://terraform_state:tOvxJ-7fxdWq0p3jKeYB@10.0.20.200:5432/terraform_state?sslmode=disable"
+    conn_str    = "postgres://terraform_state:LicuZK1nVl4ILE5HF-A9@10.0.20.200:5432/terraform_state?sslmode=disable"
     schema_name = "immich"
   }
 }

stacks/immich/providers.tf

@@ -13,6 +13,13 @@ terraform {
       source  = "goauthentik/authentik"
       version = "~> 2024.10"
     }
+    # kubectl (gavinbunney) — workaround for hashicorp/kubernetes
+    # `kubernetes_manifest` panics on Kyverno CRDs. See beads code-e2dp.
+    # Declared for all stacks but only used where opted-in.
+    kubectl = {
+      source  = "gavinbunney/kubectl"
+      version = "~> 1.14"
+    }
   }
 }
 
@@ -35,3 +42,8 @@ provider "vault" {
   address          = "https://vault.viktorbarzin.me"
   skip_child_token = true
 }
+
+provider "kubectl" {
+  config_path      = var.kube_config_path
+  load_config_file = true
+}