backup pipeline: flock manifest + cap + drop LAN -z
Three more audit fixes from the 2026-05-24 backup-pipeline review: #5 (S1 race) — manifest flock daily-backup and nfs-mirror both append to /mnt/backup/.changed-files. If they overlap (nfs-mirror Mon 04:11 running long, daily-backup starting Mon 05:00), concurrent appends from `find | tee` and `find | sed >>` could interleave mid-line — partial paths would slip past rsync's --files-from. Both scripts now share a manifest_append() helper using `flock -x` on /mnt/backup/.changed-files.lock. The 4 daily-backup call sites + the 1 nfs-mirror call site all pipe through it instead of redirecting directly. #7 (S2 unbounded manifest) daily-backup gains check_manifest_size() invoked after the PVE-config append (the last manifest writer of the run). Above MANIFEST_MAX_LINES (500k) it touches /mnt/backup/.force-full-sync — offsite-sync's Step 1 now treats that flag the same as day-of-month ≤ 7 (full sync with --delete) and clears it on success. Catches the "Synology unreachable for many days" edge case where the manifest would grow unbounded. #9 (wear — drop -z on LAN hops) offsite-sync rsync calls to Synology over the same 192.168.1.0/24 gigabit LAN had `-rltz`. Compression burns CPU on the PVE host (already IO-busy) and gives nothing on a saturated GigE link. Dropped to `-rlt` on all 5 offsite rsync invocations (Step 1 full + Step 1 incremental + Step 2 full nfs + Step 2 full nfs-ssd + Step 2 incremental). Other adjustments: - nfs-mirror's find-after-rsync now also excludes the new state files (.changed-files.lock, .force-full-sync) when populating the manifest. - offsite-sync Step 1 full-sync excludes the same .force-full-sync flag so it doesn't ship to Synology. Deployed to PVE host (/usr/local/bin/{daily-backup,nfs-mirror, offsite-sync-backup}). Currently in-flight nfs-mirror run is unaffected (bash loaded the old script into memory at start). Next runs use the new behaviour. Refs: 2026-05-24 audit Section 2 items #1 (manifest race), #4 (unbounded manifest), #6 (LAN -z wear).
This commit is contained in:
Viktor Barzin
parent
4798583db7
commit
c948dc0dbe
3 changed files with 67 additions and 14 deletions
|
|
@ -20,6 +20,34 @@ log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
|
|||
warn() { log "WARN: $*" >&2; }
|
||||
die() { log "FATAL: $*" >&2; push_metrics 1 0; exit 1; }
|
||||
|
||||
# --- Manifest append helper ---
|
||||
# Both daily-backup and nfs-mirror append to /mnt/backup/.changed-files.
|
||||
# If their runs overlap (e.g. nfs-mirror Mon 04:11 still running when
|
||||
# daily-backup starts Mon 05:00) the appends can interleave mid-line.
|
||||
# `flock -x` on a sibling lock file makes appends atomic across processes.
|
||||
MANIFEST_LOCK="${MANIFEST}.lock"
|
||||
manifest_append() {
|
||||
(
|
||||
flock -x 200
|
||||
cat >> "${MANIFEST}"
|
||||
) 200>"${MANIFEST_LOCK}"
|
||||
}
|
||||
|
||||
# Cap manifest size to prevent unbounded growth (e.g. Synology unreachable
|
||||
# for many days, every daily-backup keeps appending). At >500k lines,
|
||||
# `--files-from=` rsync becomes pathological — fall back to a full Step 1
|
||||
# sync by signalling offsite-sync to ignore the manifest this round.
|
||||
MANIFEST_MAX_LINES=500000
|
||||
check_manifest_size() {
|
||||
[ -f "${MANIFEST}" ] || return 0
|
||||
local lines
|
||||
lines=$(wc -l < "${MANIFEST}" 2>/dev/null || echo 0)
|
||||
if [ "${lines:-0}" -gt "${MANIFEST_MAX_LINES}" ]; then
|
||||
warn "manifest at ${lines} lines (>${MANIFEST_MAX_LINES}) — flagging next offsite-sync as full"
|
||||
touch "${BACKUP_ROOT}/.force-full-sync"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Locking ---
|
||||
# Track whether we got SIGTERM/SIGINT so cleanup can push a non-success metric.
|
||||
# Without this, a systemd timeout-kill leaves WeeklyBackupFailing alerts blind:
|
||||
|
|
@ -283,10 +311,10 @@ else
|
|||
log " PVC copy: ${PVC_COUNT} OK, ${PVC_FAIL} failed"
|
||||
[ "${PVC_FAIL}" -gt 0 ] && STATUS=1
|
||||
|
||||
# Add PVC files to manifest
|
||||
# Add PVC files to manifest (locked append)
|
||||
if [ -d "${BACKUP_ROOT}/pvc-data/${WEEK}" ]; then
|
||||
find "${BACKUP_ROOT}/pvc-data/${WEEK}" -type f 2>/dev/null | \
|
||||
sed "s|^${BACKUP_ROOT}/||" >> "${MANIFEST}"
|
||||
sed "s|^${BACKUP_ROOT}/||" | manifest_append
|
||||
fi
|
||||
|
||||
# Prune old weekly versions (keep 4)
|
||||
|
|
@ -310,7 +338,7 @@ if timeout 10 ssh -o BatchMode=yes -o ConnectTimeout=5 root@10.0.20.1 true 2>/de
|
|||
# config.xml — primary restore artifact
|
||||
if scp -o ConnectTimeout=10 root@10.0.20.1:/cf/conf/config.xml "${PFSENSE_DEST}/config-${DATE}.xml" 2>/dev/null; then
|
||||
log " OK: config.xml"
|
||||
echo "pfsense/config-${DATE}.xml" >> "${MANIFEST}"
|
||||
echo "pfsense/config-${DATE}.xml" | manifest_append
|
||||
else
|
||||
warn "Failed to copy pfsense config.xml"
|
||||
STATUS=1
|
||||
|
|
@ -327,7 +355,7 @@ if timeout 10 ssh -o BatchMode=yes -o ConnectTimeout=5 root@10.0.20.1 true 2>/de
|
|||
"tar czf - --exclude=/dev --exclude=/proc --exclude=/tmp --exclude=/var/run /" \
|
||||
> "${PFSENSE_DEST}/pfsense-full-${DATE}.tar.gz" 2>/dev/null; then
|
||||
log " OK: weekly full tar ($(du -sh "${PFSENSE_DEST}/pfsense-full-${DATE}.tar.gz" | cut -f1))"
|
||||
echo "pfsense/pfsense-full-${DATE}.tar.gz" >> "${MANIFEST}"
|
||||
echo "pfsense/pfsense-full-${DATE}.tar.gz" | manifest_append
|
||||
else
|
||||
warn "Failed to tar pfsense filesystem"
|
||||
STATUS=1
|
||||
|
|
@ -365,9 +393,11 @@ timeout 300 rsync -a --delete /etc/pve/ "${BACKUP_ROOT}/pve-config/etc-pve/" 2>&
|
|||
for script in /usr/local/bin/lvm-pvc-snapshot /usr/local/bin/daily-backup /usr/local/bin/offsite-sync-backup; do
|
||||
[ -f "${script}" ] && cp "${script}" "${BACKUP_ROOT}/pve-config/scripts/" 2>/dev/null || true
|
||||
done
|
||||
find "${BACKUP_ROOT}/pve-config" -type f 2>/dev/null | sed "s|^${BACKUP_ROOT}/||" >> "${MANIFEST}"
|
||||
find "${BACKUP_ROOT}/pve-config" -type f 2>/dev/null | sed "s|^${BACKUP_ROOT}/||" | manifest_append
|
||||
log " OK: PVE config"
|
||||
|
||||
check_manifest_size
|
||||
|
||||
# ============================================================
|
||||
# STEP 5: Prune LVM snapshots older than 7 days
|
||||
# ============================================================
|
||||
|
|
|
|||
|
|
@ -81,6 +81,17 @@ EXCLUDES=(
|
|||
log() { echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] $*" | tee -a "$LOG"; }
|
||||
warn() { log "WARN: $*"; }
|
||||
|
||||
# Locked manifest append (shared with daily-backup) — see daily-backup.sh
|
||||
# for the rationale. flock prevents interleaved appends when nfs-mirror
|
||||
# (Mon 04:11) overruns into daily-backup (Mon 05:00).
|
||||
MANIFEST_LOCK="${MANIFEST}.lock"
|
||||
manifest_append() {
|
||||
(
|
||||
flock -x 200
|
||||
cat >> "${MANIFEST}"
|
||||
) 200>"${MANIFEST_LOCK}"
|
||||
}
|
||||
|
||||
push_metrics() {
|
||||
local status="${1:-0}" bytes="${2:-0}"
|
||||
cat <<EOF | curl -s --connect-timeout 5 --max-time 10 --data-binary @- "${PUSHGATEWAY}/metrics/job/${PUSHGATEWAY_JOB}" 2>/dev/null || true
|
||||
|
|
@ -132,10 +143,12 @@ if [ "$RSYNC_RC" -eq 0 ]; then
|
|||
# manifest so daily Step 1 incremental picks them up tomorrow morning.
|
||||
NEW_COUNT=$(find /mnt/backup -newer "$STAMP" -type f \
|
||||
! -path '/mnt/backup/.changed-files' \
|
||||
! -path '/mnt/backup/.changed-files.lock' \
|
||||
! -path '/mnt/backup/.lv-pvc-mapping.json' \
|
||||
! -path '/mnt/backup/.nfs-changes.log' \
|
||||
! -path '/mnt/backup/.last-offsite-sync' \
|
||||
-printf '%P\n' 2>/dev/null | tee -a "$MANIFEST" | wc -l)
|
||||
! -path '/mnt/backup/.force-full-sync' \
|
||||
-printf '%P\n' 2>/dev/null | tee >(manifest_append) | wc -l)
|
||||
log "=== mirror complete; ${NEW_COUNT} files added to offsite manifest ==="
|
||||
log "/mnt/backup used: $(df -h --output=used /mnt/backup | tail -1 | tr -d ' ')"
|
||||
push_metrics 0 "$DST_BYTES"
|
||||
|
|
|
|||
|
|
@ -54,18 +54,28 @@ DAY_OF_MONTH=$(date +%d)
|
|||
# ============================================================
|
||||
log "--- Step 1: sda → Synology pve-backup/ ---"
|
||||
|
||||
if [ "${DAY_OF_MONTH}" -le 7 ]; then
|
||||
log "Monthly full sync (1st Sunday)..."
|
||||
rsync -rltz --delete --chmod=Du=rwx,Dgo=rx,Fu=rw,Fog=r \
|
||||
# Trigger: monthly cleanup window OR daily-backup signalled the manifest grew
|
||||
# past its cap (Synology was unreachable too long for incremental to keep up).
|
||||
FORCE_FULL_FLAG="${BACKUP_ROOT}/.force-full-sync"
|
||||
FORCE_FULL=""
|
||||
[ -f "${FORCE_FULL_FLAG}" ] && FORCE_FULL=1
|
||||
if [ "${DAY_OF_MONTH}" -le 7 ] || [ -n "${FORCE_FULL}" ]; then
|
||||
[ -n "${FORCE_FULL}" ] && log "Forced full sync (manifest size cap tripped)..." || log "Monthly full sync (1st Sunday)..."
|
||||
# No -z on LAN: gigabit hop to 192.168.1.13 doesn't benefit from compression
|
||||
# and burns CPU on the PVE host that's already busy with cluster IO.
|
||||
rsync -rlt --delete --chmod=Du=rwx,Dgo=rx,Fu=rw,Fog=r \
|
||||
--exclude='.changed-files' \
|
||||
--exclude='.changed-files.lock' \
|
||||
--exclude='.last-offsite-sync' \
|
||||
--exclude='.lv-pvc-mapping.json' \
|
||||
--exclude='.nfs-changes.log' \
|
||||
--exclude='.force-full-sync' \
|
||||
"${BACKUP_ROOT}/" "${PVE_BACKUP_DEST}/" 2>&1 || STATUS=1
|
||||
rm -f "${FORCE_FULL_FLAG}"
|
||||
elif [ -s "${MANIFEST}" ]; then
|
||||
MANIFEST_LINES=$(wc -l < "${MANIFEST}")
|
||||
log "Incremental sync (${MANIFEST_LINES} files from manifest)..."
|
||||
rsync -rltz --chmod=Du=rwx,Dgo=rx,Fu=rw,Fog=r --files-from="${MANIFEST}" \
|
||||
rsync -rlt --chmod=Du=rwx,Dgo=rx,Fu=rw,Fog=r --files-from="${MANIFEST}" \
|
||||
"${BACKUP_ROOT}/" "${PVE_BACKUP_DEST}/" 2>&1 || STATUS=1
|
||||
else
|
||||
log "No changed files in manifest, nothing to sync"
|
||||
|
|
@ -110,11 +120,11 @@ NFS_FULL_INCLUDES=(
|
|||
if [ "${DAY_OF_MONTH}" -le 7 ]; then
|
||||
# Monthly: full sync with --delete for cleanup, restricted to bypass-list.
|
||||
log "Monthly full NFS sync (sda-bypass paths only)..."
|
||||
rsync -rltz --delete "${NFS_FULL_INCLUDES[@]}" /srv/nfs/ "${NFS_DEST}/" 2>&1 \
|
||||
rsync -rlt --delete "${NFS_FULL_INCLUDES[@]}" /srv/nfs/ "${NFS_DEST}/" 2>&1 \
|
||||
&& log " OK: nfs/ full sync (bypass-list)" || { warn "nfs/ full sync failed"; STATUS=1; }
|
||||
# nfs-ssd: every dir under it (immich/ollama/llamacpp) is in the bypass list,
|
||||
# so a plain --delete still applies cleanly.
|
||||
rsync -rltz --delete /srv/nfs-ssd/ "${NFS_SSD_DEST}/" 2>&1 \
|
||||
rsync -rlt --delete /srv/nfs-ssd/ "${NFS_SSD_DEST}/" 2>&1 \
|
||||
&& log " OK: nfs-ssd/ full sync" || { warn "nfs-ssd/ full sync failed"; STATUS=1; }
|
||||
> "${NFS_CHANGE_LOG}"
|
||||
elif [ -s "${NFS_CHANGE_LOG}" ]; then
|
||||
|
|
@ -127,7 +137,7 @@ elif [ -s "${NFS_CHANGE_LOG}" ]; then
|
|||
> /tmp/sync-nfs.list 2>/dev/null
|
||||
NFS_COUNT=$(wc -l < /tmp/sync-nfs.list 2>/dev/null || echo 0)
|
||||
if [ "${NFS_COUNT:-0}" -gt 0 ]; then
|
||||
rsync -rltz --files-from=/tmp/sync-nfs.list /srv/nfs/ "${NFS_DEST}/" 2>&1 \
|
||||
rsync -rlt --files-from=/tmp/sync-nfs.list /srv/nfs/ "${NFS_DEST}/" 2>&1 \
|
||||
&& log " OK: nfs/ (${NFS_COUNT} bypass files)" \
|
||||
|| { warn "nfs/ incremental failed"; STATUS=1; }
|
||||
fi
|
||||
|
|
@ -138,7 +148,7 @@ elif [ -s "${NFS_CHANGE_LOG}" ]; then
|
|||
> /tmp/sync-nfs-ssd.list 2>/dev/null || true
|
||||
SSD_COUNT=$(wc -l < /tmp/sync-nfs-ssd.list 2>/dev/null || echo 0)
|
||||
if [ "${SSD_COUNT:-0}" -gt 0 ]; then
|
||||
rsync -rltz --files-from=/tmp/sync-nfs-ssd.list /srv/nfs-ssd/ "${NFS_SSD_DEST}/" 2>&1 \
|
||||
rsync -rlt --files-from=/tmp/sync-nfs-ssd.list /srv/nfs-ssd/ "${NFS_SSD_DEST}/" 2>&1 \
|
||||
&& log " OK: nfs-ssd/ (${SSD_COUNT} files)" \
|
||||
|| { warn "nfs-ssd/ incremental failed"; STATUS=1; }
|
||||
fi
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue