From aa4c125f9c683618e341297f6e4057eed735b8d0 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Mon, 13 Apr 2026 15:47:56 +0000 Subject: [PATCH] improve 3-2-1 backup: auto-discover dirs, Immich offsite sync, SQLite backup [ci skip] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - weekly-backup.sh: replace hardcoded BACKUP_DIRS with glob auto-discovery (catches nextcloud-backup, council-complaints-backup, future dirs) - weekly-backup.sh: add auto SQLite backup from PVC snapshots (magic number check, ?mode=ro URI, fallback to raw copy) - offsite-sync-backup.sh: add NFS media direct-to-Synology sync (Immich, calibre, audiobookshelf — reuses existing TrueNAS Cloud Sync paths) - Cleaned up 9 orphaned LVs + 38 snapshots on PVE host (101GB reclaimed) Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/offsite-sync-backup.sh | 54 ++++++++++++++++++++++++++++++++++ scripts/weekly-backup.sh | 41 ++++++++++++++++++-------- 2 files changed, 83 insertions(+), 12 deletions(-) diff --git a/scripts/offsite-sync-backup.sh b/scripts/offsite-sync-backup.sh index cbd85439..a55570c2 100644 --- a/scripts/offsite-sync-backup.sh +++ b/scripts/offsite-sync-backup.sh @@ -12,6 +12,11 @@ PUSHGATEWAY="${OFFSITE_SYNC_PUSHGATEWAY:-http://10.0.20.100:30091}" PUSHGATEWAY_JOB="offsite-backup-sync" LOCKFILE="/run/offsite-sync-backup.lock" +# NFS media — synced directly to Synology (bypasses sda, too large to fit) +NFS_BASE="/srv/nfs" +NFS_SSD_BASE="/srv/nfs-ssd" +SYNOLOGY_NFS_DEST="Administrator@192.168.1.13:/volume1/Backup/Viki/truenas" + # --- Logging --- log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } warn() { log "WARN: $*" >&2; } @@ -60,6 +65,55 @@ else log "No changed files in manifest, nothing to sync" fi +# ============================================================ +# STEP 2: NFS media direct to Synology (bypasses sda — too large) +# Reuses existing TrueNAS Cloud Sync paths on Synology +# ============================================================ +log "--- Step 2: NFS media direct to Synology ---" + +# Immich (map Proxmox paths to existing Synology layout) +for subdir in backups encoded-video library profile upload; do + if [ -d "${NFS_BASE}/immich/${subdir}" ]; then + rsync -rltz --delete \ + "${NFS_BASE}/immich/${subdir}/" \ + "${SYNOLOGY_NFS_DEST}/immich/immich/${subdir}/" 2>&1 \ + && log " OK: immich/${subdir}" \ + || { warn "Failed: immich/${subdir}"; STATUS=1; } + fi +done +# Immich PG data + dumps +if [ -d "${NFS_BASE}/immich/postgresql" ]; then + rsync -rltz --delete "${NFS_BASE}/immich/postgresql/" \ + "${SYNOLOGY_NFS_DEST}/immich/data-immich-postgresql/" 2>&1 \ + && log " OK: immich/postgresql" \ + || { warn "Failed: immich/postgresql"; STATUS=1; } +fi +# Immich SSD (thumbs, ML cache) +if [ -d "${NFS_SSD_BASE}/immich/thumbs" ]; then + rsync -rltz --delete "${NFS_SSD_BASE}/immich/thumbs/" \ + "${SYNOLOGY_NFS_DEST}/immich/immich/thumbs/" 2>&1 \ + && log " OK: immich/thumbs" \ + || { warn "Failed: immich/thumbs"; STATUS=1; } +fi +if [ -d "${NFS_SSD_BASE}/immich/machine-learning" ]; then + rsync -rltz --delete "${NFS_SSD_BASE}/immich/machine-learning/" \ + "${SYNOLOGY_NFS_DEST}/immich/machine-learning/" 2>&1 \ + && log " OK: immich/machine-learning" \ + || { warn "Failed: immich/machine-learning"; STATUS=1; } +fi +# Calibre + Audiobookshelf +for media_dir in calibre audiobookshelf; do + if [ -d "${NFS_BASE}/${media_dir}" ]; then + rsync -rltz --delete "${NFS_BASE}/${media_dir}/" \ + "${SYNOLOGY_NFS_DEST}/${media_dir}/" 2>&1 \ + && log " OK: ${media_dir}" \ + || { warn "Failed: ${media_dir}"; STATUS=1; } + fi +done + +# ============================================================ +# Finish +# ============================================================ if [ "${STATUS}" -eq 0 ]; then # Only clear manifest + update timestamp on SUCCESS touch "${BACKUP_ROOT}/.last-offsite-sync" diff --git a/scripts/weekly-backup.sh b/scripts/weekly-backup.sh index 108c9f87..5ba63581 100644 --- a/scripts/weekly-backup.sh +++ b/scripts/weekly-backup.sh @@ -18,18 +18,8 @@ MAPPING_CACHE="${BACKUP_ROOT}/.lv-pvc-mapping.json" KUBECONFIG="${KUBECONFIG:-/root/.kube/config}" export KUBECONFIG -# NFS backup directories to mirror -BACKUP_DIRS=( - mysql-backup - postgresql-backup - vault-backup - vaultwarden-backup - redis-backup - etcd-backup - headscale-backup - prometheus-backup - plotting-book-backup -) +# NFS backup directories — auto-discovered after NFS mount (all *-backup dirs) +BACKUP_DIRS=() # --- Logging --- log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } @@ -93,6 +83,12 @@ if ! mountpoint -q "${NFS_MOUNT}"; then fi if mountpoint -q "${NFS_MOUNT}"; then + # Auto-discover all *-backup directories (no hardcoded list) + for d in "${NFS_MOUNT}"/*-backup/; do + [ -d "$d" ] && BACKUP_DIRS+=("$(basename "$d")") + done + log " Discovered ${#BACKUP_DIRS[@]} backup dirs: ${BACKUP_DIRS[*]}" + mkdir -p "${BACKUP_ROOT}/nfs-mirror" for dir in "${BACKUP_DIRS[@]}"; do src="${NFS_MOUNT}/${dir}/" @@ -161,6 +157,26 @@ else warn "rsync failed for ${ns_pvc}" PVC_FAIL=$((PVC_FAIL + 1)) fi + + # Auto-detect and safely backup SQLite databases from snapshot + if command -v sqlite3 &>/dev/null; then + find "${PVC_MOUNT}" -maxdepth 3 \ + \( -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' \) \ + -size +0 -type f 2>/dev/null | while read -r dbfile; do + # Verify it's actually SQLite (magic number check) + if head -c 15 "$dbfile" 2>/dev/null | grep -q 'SQLite format 3'; then + relpath="${dbfile#${PVC_MOUNT}/}" + dest_file="${BACKUP_ROOT}/sqlite-backup/${WEEK}/${ns_pvc}/${relpath}" + mkdir -p "$(dirname "${dest_file}")" + if sqlite3 "file://${dbfile}?mode=ro" ".backup '${dest_file}'" 2>/dev/null; then + log " SQLite: ${ns_pvc}/${relpath}" + else + cp "${dbfile}" "${dest_file}" 2>/dev/null || true + fi + fi + done + fi + umount "${PVC_MOUNT}" 2>/dev/null || umount -l "${PVC_MOUNT}" 2>/dev/null || true else warn "Failed to mount snapshot ${snap}" @@ -179,6 +195,7 @@ else # Prune old weekly versions (keep 4) ls -1d "${BACKUP_ROOT}/pvc-data"/????-?? 2>/dev/null | head -n -4 | xargs rm -rf 2>/dev/null || true + ls -1d "${BACKUP_ROOT}/sqlite-backup"/????-?? 2>/dev/null | head -n -4 | xargs rm -rf 2>/dev/null || true PVC_BYTES=$(du -sb "${BACKUP_ROOT}/pvc-data/${WEEK}" 2>/dev/null | cut -f1 || true) TOTAL_BYTES=$((TOTAL_BYTES + ${PVC_BYTES:-0}))