improve 3-2-1 backup: auto-discover dirs, Immich offsite sync, SQLite backup [ci skip]
- weekly-backup.sh: replace hardcoded BACKUP_DIRS with glob auto-discovery (catches nextcloud-backup, council-complaints-backup, future dirs) - weekly-backup.sh: add auto SQLite backup from PVC snapshots (magic number check, ?mode=ro URI, fallback to raw copy) - offsite-sync-backup.sh: add NFS media direct-to-Synology sync (Immich, calibre, audiobookshelf — reuses existing TrueNAS Cloud Sync paths) - Cleaned up 9 orphaned LVs + 38 snapshots on PVE host (101GB reclaimed) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Viktor Barzin
parent
38d51ab0af
commit
aa4c125f9c
2 changed files with 83 additions and 12 deletions
|
|
@ -12,6 +12,11 @@ PUSHGATEWAY="${OFFSITE_SYNC_PUSHGATEWAY:-http://10.0.20.100:30091}"
|
|||
PUSHGATEWAY_JOB="offsite-backup-sync"
|
||||
LOCKFILE="/run/offsite-sync-backup.lock"
|
||||
|
||||
# NFS media — synced directly to Synology (bypasses sda, too large to fit)
|
||||
NFS_BASE="/srv/nfs"
|
||||
NFS_SSD_BASE="/srv/nfs-ssd"
|
||||
SYNOLOGY_NFS_DEST="Administrator@192.168.1.13:/volume1/Backup/Viki/truenas"
|
||||
|
||||
# --- Logging ---
|
||||
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
|
||||
warn() { log "WARN: $*" >&2; }
|
||||
|
|
@ -60,6 +65,55 @@ else
|
|||
log "No changed files in manifest, nothing to sync"
|
||||
fi
|
||||
|
||||
# ============================================================
|
||||
# STEP 2: NFS media direct to Synology (bypasses sda — too large)
|
||||
# Reuses existing TrueNAS Cloud Sync paths on Synology
|
||||
# ============================================================
|
||||
log "--- Step 2: NFS media direct to Synology ---"
|
||||
|
||||
# Immich (map Proxmox paths to existing Synology layout)
|
||||
for subdir in backups encoded-video library profile upload; do
|
||||
if [ -d "${NFS_BASE}/immich/${subdir}" ]; then
|
||||
rsync -rltz --delete \
|
||||
"${NFS_BASE}/immich/${subdir}/" \
|
||||
"${SYNOLOGY_NFS_DEST}/immich/immich/${subdir}/" 2>&1 \
|
||||
&& log " OK: immich/${subdir}" \
|
||||
|| { warn "Failed: immich/${subdir}"; STATUS=1; }
|
||||
fi
|
||||
done
|
||||
# Immich PG data + dumps
|
||||
if [ -d "${NFS_BASE}/immich/postgresql" ]; then
|
||||
rsync -rltz --delete "${NFS_BASE}/immich/postgresql/" \
|
||||
"${SYNOLOGY_NFS_DEST}/immich/data-immich-postgresql/" 2>&1 \
|
||||
&& log " OK: immich/postgresql" \
|
||||
|| { warn "Failed: immich/postgresql"; STATUS=1; }
|
||||
fi
|
||||
# Immich SSD (thumbs, ML cache)
|
||||
if [ -d "${NFS_SSD_BASE}/immich/thumbs" ]; then
|
||||
rsync -rltz --delete "${NFS_SSD_BASE}/immich/thumbs/" \
|
||||
"${SYNOLOGY_NFS_DEST}/immich/immich/thumbs/" 2>&1 \
|
||||
&& log " OK: immich/thumbs" \
|
||||
|| { warn "Failed: immich/thumbs"; STATUS=1; }
|
||||
fi
|
||||
if [ -d "${NFS_SSD_BASE}/immich/machine-learning" ]; then
|
||||
rsync -rltz --delete "${NFS_SSD_BASE}/immich/machine-learning/" \
|
||||
"${SYNOLOGY_NFS_DEST}/immich/machine-learning/" 2>&1 \
|
||||
&& log " OK: immich/machine-learning" \
|
||||
|| { warn "Failed: immich/machine-learning"; STATUS=1; }
|
||||
fi
|
||||
# Calibre + Audiobookshelf
|
||||
for media_dir in calibre audiobookshelf; do
|
||||
if [ -d "${NFS_BASE}/${media_dir}" ]; then
|
||||
rsync -rltz --delete "${NFS_BASE}/${media_dir}/" \
|
||||
"${SYNOLOGY_NFS_DEST}/${media_dir}/" 2>&1 \
|
||||
&& log " OK: ${media_dir}" \
|
||||
|| { warn "Failed: ${media_dir}"; STATUS=1; }
|
||||
fi
|
||||
done
|
||||
|
||||
# ============================================================
|
||||
# Finish
|
||||
# ============================================================
|
||||
if [ "${STATUS}" -eq 0 ]; then
|
||||
# Only clear manifest + update timestamp on SUCCESS
|
||||
touch "${BACKUP_ROOT}/.last-offsite-sync"
|
||||
|
|
|
|||
|
|
@ -18,18 +18,8 @@ MAPPING_CACHE="${BACKUP_ROOT}/.lv-pvc-mapping.json"
|
|||
KUBECONFIG="${KUBECONFIG:-/root/.kube/config}"
|
||||
export KUBECONFIG
|
||||
|
||||
# NFS backup directories to mirror
|
||||
BACKUP_DIRS=(
|
||||
mysql-backup
|
||||
postgresql-backup
|
||||
vault-backup
|
||||
vaultwarden-backup
|
||||
redis-backup
|
||||
etcd-backup
|
||||
headscale-backup
|
||||
prometheus-backup
|
||||
plotting-book-backup
|
||||
)
|
||||
# NFS backup directories — auto-discovered after NFS mount (all *-backup dirs)
|
||||
BACKUP_DIRS=()
|
||||
|
||||
# --- Logging ---
|
||||
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
|
||||
|
|
@ -93,6 +83,12 @@ if ! mountpoint -q "${NFS_MOUNT}"; then
|
|||
fi
|
||||
|
||||
if mountpoint -q "${NFS_MOUNT}"; then
|
||||
# Auto-discover all *-backup directories (no hardcoded list)
|
||||
for d in "${NFS_MOUNT}"/*-backup/; do
|
||||
[ -d "$d" ] && BACKUP_DIRS+=("$(basename "$d")")
|
||||
done
|
||||
log " Discovered ${#BACKUP_DIRS[@]} backup dirs: ${BACKUP_DIRS[*]}"
|
||||
|
||||
mkdir -p "${BACKUP_ROOT}/nfs-mirror"
|
||||
for dir in "${BACKUP_DIRS[@]}"; do
|
||||
src="${NFS_MOUNT}/${dir}/"
|
||||
|
|
@ -161,6 +157,26 @@ else
|
|||
warn "rsync failed for ${ns_pvc}"
|
||||
PVC_FAIL=$((PVC_FAIL + 1))
|
||||
fi
|
||||
|
||||
# Auto-detect and safely backup SQLite databases from snapshot
|
||||
if command -v sqlite3 &>/dev/null; then
|
||||
find "${PVC_MOUNT}" -maxdepth 3 \
|
||||
\( -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' \) \
|
||||
-size +0 -type f 2>/dev/null | while read -r dbfile; do
|
||||
# Verify it's actually SQLite (magic number check)
|
||||
if head -c 15 "$dbfile" 2>/dev/null | grep -q 'SQLite format 3'; then
|
||||
relpath="${dbfile#${PVC_MOUNT}/}"
|
||||
dest_file="${BACKUP_ROOT}/sqlite-backup/${WEEK}/${ns_pvc}/${relpath}"
|
||||
mkdir -p "$(dirname "${dest_file}")"
|
||||
if sqlite3 "file://${dbfile}?mode=ro" ".backup '${dest_file}'" 2>/dev/null; then
|
||||
log " SQLite: ${ns_pvc}/${relpath}"
|
||||
else
|
||||
cp "${dbfile}" "${dest_file}" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
umount "${PVC_MOUNT}" 2>/dev/null || umount -l "${PVC_MOUNT}" 2>/dev/null || true
|
||||
else
|
||||
warn "Failed to mount snapshot ${snap}"
|
||||
|
|
@ -179,6 +195,7 @@ else
|
|||
|
||||
# Prune old weekly versions (keep 4)
|
||||
ls -1d "${BACKUP_ROOT}/pvc-data"/????-?? 2>/dev/null | head -n -4 | xargs rm -rf 2>/dev/null || true
|
||||
ls -1d "${BACKUP_ROOT}/sqlite-backup"/????-?? 2>/dev/null | head -n -4 | xargs rm -rf 2>/dev/null || true
|
||||
|
||||
PVC_BYTES=$(du -sb "${BACKUP_ROOT}/pvc-data/${WEEK}" 2>/dev/null | cut -f1 || true)
|
||||
TOTAL_BYTES=$((TOTAL_BYTES + ${PVC_BYTES:-0}))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue