backup pipeline: prune sda-bypass list to immich-only
Previously /srv/nfs/{ollama,audiblez,ebook2audiobook,*-backup} took
the sdc → Synology direct leg. They now ride sdc → sda → Synology
pve-backup/ via nfs-mirror like every other NFS subtree, so sda
becomes the single canonical mirror and Synology only has to ingest
one feed for the bulk of cluster state.
frigate + temp dropped from BOTH legs (no backup anywhere) per
explicit user ask — frigate is a 14d camera ring, temp is scratch.
prometheus/loki/alertmanager dropped as no-op (orphan dirs that
no longer exist on /srv/nfs).
Also: nfs-mirror's manifest collection switched from find -newer
(mtime) to find -cnewer (ctime) — rsync -t preserves source mtime
on dest, so freshly-written files looked "older than \$STAMP" and
the 2026-05-26 full mirror run captured only 2 of 800k transferred
files. Hit during this session, recovered via .force-full-sync.
Operational result post-rollout:
- sda 87% → 70% (anca-elements 423G deleted, +260G new dirs)
- /Viki/nfs/ on Synology: was 24 stale dirs (~430G), now immich only
- Synology free: ~300G → ~430G+ once btrfs reclaim catches up
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Viktor Barzin
parent
b3dcccfc41
commit
41fb7c4a76
3 changed files with 98 additions and 92 deletions
|
|
@ -76,8 +76,8 @@ if [ "${DAY_OF_MONTH}" -le 7 ] || [ -n "${FORCE_FULL}" ]; then
|
|||
elif [ -s "${MANIFEST}" ]; then
|
||||
MANIFEST_LINES=$(wc -l < "${MANIFEST}")
|
||||
log "Incremental sync (${MANIFEST_LINES} files from manifest)..."
|
||||
# /anca-elements is being ingested into Immich (Immich becomes canonical) —
|
||||
# skip the redundant copy in /mnt/backup/anca-elements/ until manual cleanup.
|
||||
# anca-elements: now in Immich (canonical); /mnt/backup copy deleted
|
||||
# 2026-05-26. Exclude retained as a safety belt in case it re-appears.
|
||||
rsync -rlt --chmod=Du=rwx,Dgo=rx,Fu=rw,Fog=r --files-from="${MANIFEST}" \
|
||||
--exclude='anca-elements/' \
|
||||
"${BACKUP_ROOT}/" "${PVE_BACKUP_DEST}/" 2>&1 || STATUS=1
|
||||
|
|
@ -89,64 +89,60 @@ fi
|
|||
# STEP 2: NFS → Synology nfs/ + nfs-ssd/ (inotify change-tracked, FILTERED)
|
||||
# ============================================================
|
||||
#
|
||||
# DESIGN: Step 2 only carries paths that BYPASS the sda mirror. Paths that ARE
|
||||
# mirrored to sda by nfs-mirror reach Synology via Step 1 (sda → Synology
|
||||
# pve-backup/) and must NOT also flow through Step 2 — that would duplicate
|
||||
# every byte and double Synology consumption.
|
||||
# DESIGN: Step 2 only carries paths that BYPASS the sda mirror. As of
|
||||
# 2026-05-26 that's just /srv/nfs/immich/ (1.5T, doesn't fit on sda).
|
||||
# Everything else under /srv/nfs/ now flows through sda via nfs-mirror,
|
||||
# reaching Synology via Step 1 (sda → pve-backup/). frigate and temp are
|
||||
# excluded from both legs — intentionally NOT backed up.
|
||||
#
|
||||
# The skip-list below MUST stay in sync with EXCLUDES in
|
||||
# /usr/local/bin/nfs-mirror (which defines what nfs-mirror does NOT copy to
|
||||
# sda). The two are complementary: nfs-mirror EXCLUDES = offsite-sync Step 2
|
||||
# INCLUDES. Failing to keep them aligned creates either gaps (data missing
|
||||
# from Synology) or duplication (data on Synology via both paths).
|
||||
log "--- Step 2: NFS → Synology (skip-list paths only — sda-bypass leg) ---"
|
||||
# nfs-ssd is handled separately below: its three dirs (immich, ollama,
|
||||
# llamacpp) all go direct to Synology since /srv/nfs-ssd is not mirrored
|
||||
# to sda. ollama+llamacpp are small enough (~85G total) that the direct
|
||||
# leg is fine and we don't need to extend nfs-mirror to cover the SSD.
|
||||
#
|
||||
# Keep this aligned with /usr/local/bin/nfs-mirror's EXCLUDES — the
|
||||
# excludes there are { immich (this leg), frigate (no backup), temp
|
||||
# (no backup), anca-elements (deleted), pvc-data and friends (owned by
|
||||
# daily-backup) }. Only the bypass-leg subset matters here: { immich }.
|
||||
log "--- Step 2: NFS → Synology (immich-only direct leg + nfs-ssd) ---"
|
||||
|
||||
# Regex matching paths NOT on sda (must reach Synology directly).
|
||||
# Top-level dirs under /srv/nfs/ — anchored, no nesting allowed.
|
||||
NFS_SDA_BYPASS_RE='^/srv/nfs/(immich|frigate|prometheus|loki|temp|alertmanager|ollama|audiblez|ebook2audiobook|[^/]+-backup)/'
|
||||
NFS_SDA_BYPASS_RE='^/srv/nfs/immich/'
|
||||
|
||||
# rsync include/exclude args for the monthly full sync (HDD).
|
||||
# Order matters: --include patterns first, --exclude '*' last.
|
||||
NFS_FULL_INCLUDES=(
|
||||
--include='/immich/' --include='/immich/***'
|
||||
--include='/frigate/' --include='/frigate/***'
|
||||
--include='/prometheus/' --include='/prometheus/***'
|
||||
--include='/loki/' --include='/loki/***'
|
||||
--include='/temp/' --include='/temp/***'
|
||||
--include='/alertmanager/' --include='/alertmanager/***'
|
||||
--include='/ollama/' --include='/ollama/***'
|
||||
--include='/audiblez/' --include='/audiblez/***'
|
||||
--include='/ebook2audiobook/' --include='/ebook2audiobook/***'
|
||||
--include='/*-backup/' --include='/*-backup/***'
|
||||
--include='/immich/' --include='/immich/***'
|
||||
--exclude='*'
|
||||
)
|
||||
|
||||
if [ "${DAY_OF_MONTH}" -le 7 ]; then
|
||||
# Monthly: full sync with --delete for cleanup, restricted to bypass-list.
|
||||
log "Monthly full NFS sync (sda-bypass paths only)..."
|
||||
# --delete here will reap legacy dirs on Synology (frigate, ollama,
|
||||
# audiblez, ebook2audiobook, *-backup, prometheus, loki, temp,
|
||||
# alertmanager) since they're no longer in NFS_FULL_INCLUDES.
|
||||
log "Monthly full NFS sync (immich-only — reaps legacy bypass dirs)..."
|
||||
rsync -rlt --delete "${NFS_FULL_INCLUDES[@]}" /srv/nfs/ "${NFS_DEST}/" 2>&1 \
|
||||
&& log " OK: nfs/ full sync (bypass-list)" || { warn "nfs/ full sync failed"; STATUS=1; }
|
||||
# nfs-ssd: every dir under it (immich/ollama/llamacpp) is in the bypass list,
|
||||
# so a plain --delete still applies cleanly.
|
||||
&& log " OK: nfs/ full sync (immich-only)" || { warn "nfs/ full sync failed"; STATUS=1; }
|
||||
# nfs-ssd: full sync of all three dirs (immich, ollama, llamacpp).
|
||||
rsync -rlt --delete /srv/nfs-ssd/ "${NFS_SSD_DEST}/" 2>&1 \
|
||||
&& log " OK: nfs-ssd/ full sync" || { warn "nfs-ssd/ full sync failed"; STATUS=1; }
|
||||
> "${NFS_CHANGE_LOG}"
|
||||
elif [ -s "${NFS_CHANGE_LOG}" ]; then
|
||||
# Incremental: only sync changed files in bypass-list paths.
|
||||
# Incremental: only sync changed files matching the bypass leg (immich).
|
||||
sort -u "${NFS_CHANGE_LOG}" > /tmp/nfs-changes-deduped
|
||||
|
||||
# HDD NFS — include only sda-bypass paths.
|
||||
# HDD NFS — include only /srv/nfs/immich/ paths.
|
||||
grep -E "${NFS_SDA_BYPASS_RE}" /tmp/nfs-changes-deduped | \
|
||||
while IFS= read -r f; do [ -f "$f" ] && echo "${f#/srv/nfs/}"; done \
|
||||
> /tmp/sync-nfs.list 2>/dev/null
|
||||
NFS_COUNT=$(wc -l < /tmp/sync-nfs.list 2>/dev/null || echo 0)
|
||||
if [ "${NFS_COUNT:-0}" -gt 0 ]; then
|
||||
rsync -rlt --files-from=/tmp/sync-nfs.list /srv/nfs/ "${NFS_DEST}/" 2>&1 \
|
||||
&& log " OK: nfs/ (${NFS_COUNT} bypass files)" \
|
||||
&& log " OK: nfs/ (${NFS_COUNT} immich files)" \
|
||||
|| { warn "nfs/ incremental failed"; STATUS=1; }
|
||||
fi
|
||||
|
||||
# SSD NFS — every nfs-ssd path (immich/ollama/llamacpp) is in the bypass list.
|
||||
# SSD NFS — every nfs-ssd path (immich/ollama/llamacpp) ships direct.
|
||||
grep '^/srv/nfs-ssd/' /tmp/nfs-changes-deduped | \
|
||||
while IFS= read -r f; do [ -f "$f" ] && echo "${f#/srv/nfs-ssd/}"; done \
|
||||
> /tmp/sync-nfs-ssd.list 2>/dev/null || true
|
||||
|
|
@ -158,7 +154,7 @@ elif [ -s "${NFS_CHANGE_LOG}" ]; then
|
|||
fi
|
||||
|
||||
TOTAL=$(wc -l < /tmp/nfs-changes-deduped)
|
||||
log " Processed ${TOTAL} change events (${NFS_COUNT} nfs + ${SSD_COUNT} nfs-ssd bypass-list files synced)"
|
||||
log " Processed ${TOTAL} change events (${NFS_COUNT} nfs/immich + ${SSD_COUNT} nfs-ssd files synced)"
|
||||
> "${NFS_CHANGE_LOG}"
|
||||
rm -f /tmp/nfs-changes-deduped /tmp/sync-nfs.list /tmp/sync-nfs-ssd.list
|
||||
else
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue