diff --git a/scripts/daily-backup.sh b/scripts/daily-backup.sh index e5f9fbec..7b896780 100644 --- a/scripts/daily-backup.sh +++ b/scripts/daily-backup.sh @@ -20,6 +20,34 @@ log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } warn() { log "WARN: $*" >&2; } die() { log "FATAL: $*" >&2; push_metrics 1 0; exit 1; } +# --- Manifest append helper --- +# Both daily-backup and nfs-mirror append to /mnt/backup/.changed-files. +# If their runs overlap (e.g. nfs-mirror Mon 04:11 still running when +# daily-backup starts Mon 05:00) the appends can interleave mid-line. +# `flock -x` on a sibling lock file makes appends atomic across processes. +MANIFEST_LOCK="${MANIFEST}.lock" +manifest_append() { + ( + flock -x 200 + cat >> "${MANIFEST}" + ) 200>"${MANIFEST_LOCK}" +} + +# Cap manifest size to prevent unbounded growth (e.g. Synology unreachable +# for many days, every daily-backup keeps appending). At >500k lines, +# `--files-from=` rsync becomes pathological — fall back to a full Step 1 +# sync by signalling offsite-sync to ignore the manifest this round. +MANIFEST_MAX_LINES=500000 +check_manifest_size() { + [ -f "${MANIFEST}" ] || return 0 + local lines + lines=$(wc -l < "${MANIFEST}" 2>/dev/null || echo 0) + if [ "${lines:-0}" -gt "${MANIFEST_MAX_LINES}" ]; then + warn "manifest at ${lines} lines (>${MANIFEST_MAX_LINES}) — flagging next offsite-sync as full" + touch "${BACKUP_ROOT}/.force-full-sync" + fi +} + # --- Locking --- # Track whether we got SIGTERM/SIGINT so cleanup can push a non-success metric. # Without this, a systemd timeout-kill leaves WeeklyBackupFailing alerts blind: @@ -283,10 +311,10 @@ else log " PVC copy: ${PVC_COUNT} OK, ${PVC_FAIL} failed" [ "${PVC_FAIL}" -gt 0 ] && STATUS=1 - # Add PVC files to manifest + # Add PVC files to manifest (locked append) if [ -d "${BACKUP_ROOT}/pvc-data/${WEEK}" ]; then find "${BACKUP_ROOT}/pvc-data/${WEEK}" -type f 2>/dev/null | \ - sed "s|^${BACKUP_ROOT}/||" >> "${MANIFEST}" + sed "s|^${BACKUP_ROOT}/||" | manifest_append fi # Prune old weekly versions (keep 4) @@ -310,7 +338,7 @@ if timeout 10 ssh -o BatchMode=yes -o ConnectTimeout=5 root@10.0.20.1 true 2>/de # config.xml — primary restore artifact if scp -o ConnectTimeout=10 root@10.0.20.1:/cf/conf/config.xml "${PFSENSE_DEST}/config-${DATE}.xml" 2>/dev/null; then log " OK: config.xml" - echo "pfsense/config-${DATE}.xml" >> "${MANIFEST}" + echo "pfsense/config-${DATE}.xml" | manifest_append else warn "Failed to copy pfsense config.xml" STATUS=1 @@ -327,7 +355,7 @@ if timeout 10 ssh -o BatchMode=yes -o ConnectTimeout=5 root@10.0.20.1 true 2>/de "tar czf - --exclude=/dev --exclude=/proc --exclude=/tmp --exclude=/var/run /" \ > "${PFSENSE_DEST}/pfsense-full-${DATE}.tar.gz" 2>/dev/null; then log " OK: weekly full tar ($(du -sh "${PFSENSE_DEST}/pfsense-full-${DATE}.tar.gz" | cut -f1))" - echo "pfsense/pfsense-full-${DATE}.tar.gz" >> "${MANIFEST}" + echo "pfsense/pfsense-full-${DATE}.tar.gz" | manifest_append else warn "Failed to tar pfsense filesystem" STATUS=1 @@ -365,9 +393,11 @@ timeout 300 rsync -a --delete /etc/pve/ "${BACKUP_ROOT}/pve-config/etc-pve/" 2>& for script in /usr/local/bin/lvm-pvc-snapshot /usr/local/bin/daily-backup /usr/local/bin/offsite-sync-backup; do [ -f "${script}" ] && cp "${script}" "${BACKUP_ROOT}/pve-config/scripts/" 2>/dev/null || true done -find "${BACKUP_ROOT}/pve-config" -type f 2>/dev/null | sed "s|^${BACKUP_ROOT}/||" >> "${MANIFEST}" +find "${BACKUP_ROOT}/pve-config" -type f 2>/dev/null | sed "s|^${BACKUP_ROOT}/||" | manifest_append log " OK: PVE config" +check_manifest_size + # ============================================================ # STEP 5: Prune LVM snapshots older than 7 days # ============================================================ diff --git a/scripts/nfs-mirror.sh b/scripts/nfs-mirror.sh index 7d74b9bb..034f3317 100644 --- a/scripts/nfs-mirror.sh +++ b/scripts/nfs-mirror.sh @@ -81,6 +81,17 @@ EXCLUDES=( log() { echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] $*" | tee -a "$LOG"; } warn() { log "WARN: $*"; } +# Locked manifest append (shared with daily-backup) — see daily-backup.sh +# for the rationale. flock prevents interleaved appends when nfs-mirror +# (Mon 04:11) overruns into daily-backup (Mon 05:00). +MANIFEST_LOCK="${MANIFEST}.lock" +manifest_append() { + ( + flock -x 200 + cat >> "${MANIFEST}" + ) 200>"${MANIFEST_LOCK}" +} + push_metrics() { local status="${1:-0}" bytes="${2:-0}" cat </dev/null || true @@ -132,10 +143,12 @@ if [ "$RSYNC_RC" -eq 0 ]; then # manifest so daily Step 1 incremental picks them up tomorrow morning. NEW_COUNT=$(find /mnt/backup -newer "$STAMP" -type f \ ! -path '/mnt/backup/.changed-files' \ + ! -path '/mnt/backup/.changed-files.lock' \ ! -path '/mnt/backup/.lv-pvc-mapping.json' \ ! -path '/mnt/backup/.nfs-changes.log' \ ! -path '/mnt/backup/.last-offsite-sync' \ - -printf '%P\n' 2>/dev/null | tee -a "$MANIFEST" | wc -l) + ! -path '/mnt/backup/.force-full-sync' \ + -printf '%P\n' 2>/dev/null | tee >(manifest_append) | wc -l) log "=== mirror complete; ${NEW_COUNT} files added to offsite manifest ===" log "/mnt/backup used: $(df -h --output=used /mnt/backup | tail -1 | tr -d ' ')" push_metrics 0 "$DST_BYTES" diff --git a/scripts/offsite-sync-backup.sh b/scripts/offsite-sync-backup.sh index c286cf58..1f3f7cf7 100644 --- a/scripts/offsite-sync-backup.sh +++ b/scripts/offsite-sync-backup.sh @@ -54,18 +54,28 @@ DAY_OF_MONTH=$(date +%d) # ============================================================ log "--- Step 1: sda → Synology pve-backup/ ---" -if [ "${DAY_OF_MONTH}" -le 7 ]; then - log "Monthly full sync (1st Sunday)..." - rsync -rltz --delete --chmod=Du=rwx,Dgo=rx,Fu=rw,Fog=r \ +# Trigger: monthly cleanup window OR daily-backup signalled the manifest grew +# past its cap (Synology was unreachable too long for incremental to keep up). +FORCE_FULL_FLAG="${BACKUP_ROOT}/.force-full-sync" +FORCE_FULL="" +[ -f "${FORCE_FULL_FLAG}" ] && FORCE_FULL=1 +if [ "${DAY_OF_MONTH}" -le 7 ] || [ -n "${FORCE_FULL}" ]; then + [ -n "${FORCE_FULL}" ] && log "Forced full sync (manifest size cap tripped)..." || log "Monthly full sync (1st Sunday)..." + # No -z on LAN: gigabit hop to 192.168.1.13 doesn't benefit from compression + # and burns CPU on the PVE host that's already busy with cluster IO. + rsync -rlt --delete --chmod=Du=rwx,Dgo=rx,Fu=rw,Fog=r \ --exclude='.changed-files' \ + --exclude='.changed-files.lock' \ --exclude='.last-offsite-sync' \ --exclude='.lv-pvc-mapping.json' \ --exclude='.nfs-changes.log' \ + --exclude='.force-full-sync' \ "${BACKUP_ROOT}/" "${PVE_BACKUP_DEST}/" 2>&1 || STATUS=1 + rm -f "${FORCE_FULL_FLAG}" elif [ -s "${MANIFEST}" ]; then MANIFEST_LINES=$(wc -l < "${MANIFEST}") log "Incremental sync (${MANIFEST_LINES} files from manifest)..." - rsync -rltz --chmod=Du=rwx,Dgo=rx,Fu=rw,Fog=r --files-from="${MANIFEST}" \ + rsync -rlt --chmod=Du=rwx,Dgo=rx,Fu=rw,Fog=r --files-from="${MANIFEST}" \ "${BACKUP_ROOT}/" "${PVE_BACKUP_DEST}/" 2>&1 || STATUS=1 else log "No changed files in manifest, nothing to sync" @@ -110,11 +120,11 @@ NFS_FULL_INCLUDES=( if [ "${DAY_OF_MONTH}" -le 7 ]; then # Monthly: full sync with --delete for cleanup, restricted to bypass-list. log "Monthly full NFS sync (sda-bypass paths only)..." - rsync -rltz --delete "${NFS_FULL_INCLUDES[@]}" /srv/nfs/ "${NFS_DEST}/" 2>&1 \ + rsync -rlt --delete "${NFS_FULL_INCLUDES[@]}" /srv/nfs/ "${NFS_DEST}/" 2>&1 \ && log " OK: nfs/ full sync (bypass-list)" || { warn "nfs/ full sync failed"; STATUS=1; } # nfs-ssd: every dir under it (immich/ollama/llamacpp) is in the bypass list, # so a plain --delete still applies cleanly. - rsync -rltz --delete /srv/nfs-ssd/ "${NFS_SSD_DEST}/" 2>&1 \ + rsync -rlt --delete /srv/nfs-ssd/ "${NFS_SSD_DEST}/" 2>&1 \ && log " OK: nfs-ssd/ full sync" || { warn "nfs-ssd/ full sync failed"; STATUS=1; } > "${NFS_CHANGE_LOG}" elif [ -s "${NFS_CHANGE_LOG}" ]; then @@ -127,7 +137,7 @@ elif [ -s "${NFS_CHANGE_LOG}" ]; then > /tmp/sync-nfs.list 2>/dev/null NFS_COUNT=$(wc -l < /tmp/sync-nfs.list 2>/dev/null || echo 0) if [ "${NFS_COUNT:-0}" -gt 0 ]; then - rsync -rltz --files-from=/tmp/sync-nfs.list /srv/nfs/ "${NFS_DEST}/" 2>&1 \ + rsync -rlt --files-from=/tmp/sync-nfs.list /srv/nfs/ "${NFS_DEST}/" 2>&1 \ && log " OK: nfs/ (${NFS_COUNT} bypass files)" \ || { warn "nfs/ incremental failed"; STATUS=1; } fi @@ -138,7 +148,7 @@ elif [ -s "${NFS_CHANGE_LOG}" ]; then > /tmp/sync-nfs-ssd.list 2>/dev/null || true SSD_COUNT=$(wc -l < /tmp/sync-nfs-ssd.list 2>/dev/null || echo 0) if [ "${SSD_COUNT:-0}" -gt 0 ]; then - rsync -rltz --files-from=/tmp/sync-nfs-ssd.list /srv/nfs-ssd/ "${NFS_SSD_DEST}/" 2>&1 \ + rsync -rlt --files-from=/tmp/sync-nfs-ssd.list /srv/nfs-ssd/ "${NFS_SSD_DEST}/" 2>&1 \ && log " OK: nfs-ssd/ (${SSD_COUNT} files)" \ || { warn "nfs-ssd/ incremental failed"; STATUS=1; } fi