From 38d51ab0afab174fecccf2fcf9675d3dbb322dfa Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Mon, 13 Apr 2026 14:41:15 +0000 Subject: [PATCH] deprecate TrueNAS: migrate Immich NFS to Proxmox, remove all 10.0.10.15 references [ci skip] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Migrate Immich (8 NFS PVs, 1.1TB) from TrueNAS to Proxmox host NFS - Update config.tfvars nfs_server to 192.168.1.127 (Proxmox) - Update nfs-csi StorageClass share to /srv/nfs - Update scripts (weekly-backup, cluster-healthcheck) to Proxmox IP - Delete obsolete TrueNAS scripts (nfs_exports.sh, truenas-status.sh) - Rewrite nfs-health.sh for Proxmox NFS monitoring - Update Freedify nfs_music_server default to Proxmox - Mark CloudSync monitor CronJob as deprecated - Update Prometheus alert summaries - Update all architecture docs, AGENTS.md, and reference docs - Zero PVs remain on TrueNAS — VM ready for decommission Co-Authored-By: Claude Opus 4.6 (1M context) --- .claude/cluster-health.sh | 8 +- .claude/reference/patterns.md | 21 +- .claude/reference/proxmox-inventory.md | 30 +-- .claude/scripts/nfs-health.sh | 61 +++-- .claude/scripts/truenas-status.sh | 186 -------------- AGENTS.md | 18 +- config.tfvars | Bin 4772 -> 10275 bytes docs/architecture/backup-dr.md | 75 ++---- docs/architecture/overview.md | 12 +- docs/architecture/storage.md | 230 +++++++----------- scripts/cluster_healthcheck.sh | 10 +- scripts/weekly-backup.sh | 6 +- secrets/nfs_exports.sh | Bin 809 -> 0 bytes stacks/freedify/factory/main.tf | 4 +- stacks/immich/.terraform.lock.hcl | 3 + stacks/immich/backend.tf | 2 +- stacks/immich/main.tf | 89 ++++--- stacks/monitoring/modules/monitoring/main.tf | 10 +- .../monitoring/prometheus_chart_values.tpl | 2 +- stacks/nfs-csi/modules/nfs-csi/main.tf | 2 +- 20 files changed, 245 insertions(+), 524 deletions(-) delete mode 100755 .claude/scripts/truenas-status.sh delete mode 100755 secrets/nfs_exports.sh diff --git a/.claude/cluster-health.sh b/.claude/cluster-health.sh index 299b7eba..94d51abe 100755 --- a/.claude/cluster-health.sh +++ b/.claude/cluster-health.sh @@ -1002,16 +1002,16 @@ check_nfs() { # Try native tools first (available locally), fall back to kubectl-based check (pod environment) if command -v showmount &>/dev/null; then - if showmount -e 10.0.10.15 &>/dev/null; then - pass "NFS server 10.0.10.15 reachable (exports listed)" + if showmount -e 192.168.1.127 &>/dev/null; then + pass "NFS server 192.168.1.127 reachable (exports listed)" json_add "nfs" "PASS" "NFS reachable" return 0 fi fi if command -v nc &>/dev/null; then - if nc -z -G 3 10.0.10.15 2049 &>/dev/null; then - pass "NFS server 10.0.10.15 port 2049 open" + if nc -z -G 3 192.168.1.127 2049 &>/dev/null; then + pass "NFS server 192.168.1.127 port 2049 open" json_add "nfs" "PASS" "NFS port open" return 0 fi diff --git a/.claude/reference/patterns.md b/.claude/reference/patterns.md index 7427b73c..6035b764 100644 --- a/.claude/reference/patterns.md +++ b/.claude/reference/patterns.md @@ -3,29 +3,28 @@ Reference file for patterns, procedures, and tables. Read on demand when the specific topic comes up. ## NFS Volume Pattern -Use the `nfs_volume` shared module for all NFS volumes (CSI-backed, `soft,timeo=30,retrans=3`): +Use the `nfs_volume` shared module for all NFS volumes (creates static PVs, CSI-backed, `soft,timeo=30,retrans=3`): ```hcl module "nfs_data" { source = "../../modules/kubernetes/nfs_volume" # ../../../../ for platform modules, ../../../ for sub-stacks name = "-data" # Must be globally unique (PV is cluster-scoped) namespace = kubernetes_namespace..metadata[0].name - nfs_server = var.nfs_server - nfs_path = "/mnt/main/" + nfs_server = var.nfs_server # 192.168.1.127 (Proxmox host) + nfs_path = "/srv/nfs/" # HDD NFS, or "/srv/nfs-ssd/" for SSD } # In pod spec: persistent_volume_claim { claim_name = module.nfs_data.claim_name } ``` +**Note**: Some legacy PVs still reference `/mnt/main/` paths (from the TrueNAS era). These work via compatibility on the Proxmox host. New PVs should use `/srv/nfs/` or `/srv/nfs-ssd/`. **DO NOT use inline `nfs {}` blocks** — they mount with `hard,timeo=600` defaults which hang forever. ## Adding NFS Exports -1. Create dir on TrueNAS: `ssh root@10.0.10.15 "mkdir -p /mnt/main/ && chmod 777 /mnt/main/"` -2. Edit `secrets/nfs_directories.txt` — add path, keep sorted -3. Run `secrets/nfs_exports.sh` from `secrets/` -4. If any path doesn't exist on TrueNAS, the API rejects the entire update. +1. Create dir on Proxmox host: `ssh root@192.168.1.127 "mkdir -p /srv/nfs/ && chmod 777 /srv/nfs/"` +2. Edit `/etc/exports` on the Proxmox host — add the export entry +3. Reload exports: `ssh root@192.168.1.127 "exportfs -ra"` +4. Verify: `showmount -e 192.168.1.127` -## iSCSI Storage (Databases) -**StorageClass**: `iscsi-truenas` (democratic-csi, `freenas-iscsi` SSH driver — NOT `freenas-api-iscsi`). -Used by: PostgreSQL (CNPG), MySQL (InnoDB Cluster). ZFS: `main/iscsi` (zvols), `main/iscsi-snaps`. -All K8s nodes have `open-iscsi` + `iscsid` running. +## ~~iSCSI Storage~~ (REMOVED — replaced by proxmox-lvm) +> iSCSI via democratic-csi and TrueNAS has been fully removed (2026-04). All database storage now uses `StorageClass: proxmox-lvm` (Proxmox CSI, LVM-thin hotplug). TrueNAS has been decommissioned. ## Anti-AI Scraping (5-Layer Defense) Default `anti_ai_scraping = true` in ingress_factory. Disable per-service: `anti_ai_scraping = false`. diff --git a/.claude/reference/proxmox-inventory.md b/.claude/reference/proxmox-inventory.md index 4ad08e8f..1d9abb68 100644 --- a/.claude/reference/proxmox-inventory.md +++ b/.claude/reference/proxmox-inventory.md @@ -8,30 +8,10 @@ - **RAM**: 272 GB DDR4-2400 ECC RDIMM (10 DIMMs, see Memory Layout below) - **GPU**: NVIDIA Tesla T4 (PCIe passthrough to k8s-node1) - **iDRAC**: 192.168.1.4 (root/calvin) -- **Disks**: 1.1TB RAID1 SAS (unused) + 931GB Samsung SSD + 10.7TB RAID1 HDD +- **Disks**: 1.1TB RAID1 SAS (backup) + 931GB Samsung SSD + 10.7TB RAID1 HDD +- **NFS server**: Proxmox host serves NFS directly. HDD NFS: `/srv/nfs` on ext4 LV `pve/nfs-data` (2TB). SSD NFS: `/srv/nfs-ssd` on ext4 LV `ssd/nfs-ssd-data` (100GB). TrueNAS (10.0.10.15) decommissioned. - **Proxmox access**: `ssh root@192.168.1.127` -## NFS Exports (Proxmox Host) - -The Proxmox host serves NFS for all workloads except Immich (which remains on TrueNAS). - -### HDD NFS -- **LV**: `pve/nfs-data` (thin LV, 1TB) -- **Filesystem**: ext4 (chosen over btrfs — btrfs CoW on LVM thin = double-CoW problem) -- **Mount**: `/srv/nfs` with `noatime,commit=30` -- **Export**: `/srv/nfs *(rw,no_subtree_check,no_root_squash,insecure,fsid=0)` - -### SSD NFS -- **LV**: `ssd/nfs-ssd-data` (100GB) -- **Filesystem**: ext4 -- **Mount**: `/srv/nfs-ssd` with `noatime,commit=30` -- **Export**: `/srv/nfs-ssd *(rw,no_subtree_check,no_root_squash,insecure,fsid=1)` -- **Current users**: Ollama (migrated from TrueNAS SSD `/mnt/ssd/ollama`) - -### Notes -- `insecure` option required: pfSense NATs source ports >1024 when routing between VLANs -- 21 stacks migrated from TrueNAS, only Immich (8 PVCs) remains on TrueNAS - ## Memory Layout (updated 2026-04-01) ### Physical DIMM Slot Map @@ -97,10 +77,10 @@ Channel 3: A4 [32G] ──── A8 [32G] ──── A12[ 8G ] = 72 GB ## Network Topology ``` -10.0.10.0/24 - Management: Wizard (10.0.10.10), TrueNAS NFS (10.0.10.15) +10.0.10.0/24 - Management: Wizard (10.0.10.10) 10.0.20.0/24 - Kubernetes: pfSense GW (10.0.20.1), Registry (10.0.20.10), k8s-master (10.0.20.100), DNS (10.0.20.101), MetalLB (10.0.20.102-200) -192.168.1.0/24 - Physical: Proxmox (192.168.1.127, NFS server for k8s) +192.168.1.0/24 - Physical: Proxmox (192.168.1.127) ``` ## Network Bridges @@ -122,7 +102,7 @@ Channel 3: A4 [32G] ──── A8 [32G] ──── A12[ 8G ] = 72 GB | 204 | k8s-node4 | running | 8 | 24GB | vmbr1:vlan20 | 256G | Worker | | 220 | docker-registry | running | 4 | 4GB | vmbr1:vlan20 | 64G | MAC DE:AD:BE:EF:22:22 (10.0.20.10) | | 300 | Windows10 | running | 16 | 8GB | vmbr0 | 100G | Windows VM | -| 9000 | truenas | running | 16 | 8GB | vmbr1:vlan10 | 32G+7x256G+1T | NFS (10.0.10.15) — Immich only | +| ~~9000~~ | ~~truenas~~ | **stopped/decommissioned** | — | — | — | — | NFS migrated to Proxmox host (192.168.1.127) at `/srv/nfs` and `/srv/nfs-ssd` | **Total VM RAM allocated**: 180 GB of 272 GB (66%) — 92 GB free for future VMs diff --git a/.claude/scripts/nfs-health.sh b/.claude/scripts/nfs-health.sh index dc933868..d540893a 100755 --- a/.claude/scripts/nfs-health.sh +++ b/.claude/scripts/nfs-health.sh @@ -3,7 +3,7 @@ set -euo pipefail AGENT="nfs-health" KUBECTL="kubectl --kubeconfig /Users/viktorbarzin/code/infra/config" -TRUENAS_HOST="10.0.10.15" +NFS_HOST="192.168.1.127" NODES=("k8s-master:10.0.20.100" "k8s-node1:10.0.20.101" "k8s-node2:10.0.20.102" "k8s-node3:10.0.20.103" "k8s-node4:10.0.20.104") SSH_USER="wizard" DRY_RUN=false @@ -21,33 +21,61 @@ add_check() { checks+=("{\"name\": \"$name\", \"status\": \"$status\", \"message\": \"$message\"}") } -check_truenas_reachable() { +check_nfs_reachable() { if $DRY_RUN; then - add_check "truenas-reachable" "ok" "dry-run: would ping $TRUENAS_HOST" + add_check "nfs-reachable" "ok" "dry-run: would ping $NFS_HOST" return fi - if timeout 5 ping -c 1 "$TRUENAS_HOST" &>/dev/null; then - add_check "truenas-reachable" "ok" "TrueNAS at $TRUENAS_HOST is reachable" + if timeout 5 ping -c 1 "$NFS_HOST" &>/dev/null; then + add_check "nfs-reachable" "ok" "Proxmox NFS at $NFS_HOST is reachable" else - add_check "truenas-reachable" "fail" "TrueNAS at $TRUENAS_HOST is unreachable" + add_check "nfs-reachable" "fail" "Proxmox NFS at $NFS_HOST is unreachable" fi } -check_truenas_nfs_service() { +check_nfs_exports() { if $DRY_RUN; then - add_check "truenas-nfs-service" "ok" "dry-run: would check NFS service on TrueNAS" + add_check "nfs-exports" "ok" "dry-run: would check NFS exports on Proxmox" return fi local result - if result=$(timeout 10 ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "root@$TRUENAS_HOST" \ - "service nfs-server status 2>/dev/null || systemctl is-active nfs-server 2>/dev/null || echo 'unknown'" 2>/dev/null); then - if echo "$result" | grep -qiE "running|active|is running"; then - add_check "truenas-nfs-service" "ok" "NFS service is running on TrueNAS" + if result=$(timeout 10 ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "root@$NFS_HOST" \ + "exportfs -v 2>/dev/null || cat /etc/exports 2>/dev/null" 2>/dev/null); then + local export_count + export_count=$(echo "$result" | grep -c '/' || echo 0) + if [ "$export_count" -gt 0 ]; then + add_check "nfs-exports" "ok" "$export_count NFS exports active on Proxmox" else - add_check "truenas-nfs-service" "warn" "NFS service status unclear: $(echo "$result" | head -1 | tr '"' "'")" + add_check "nfs-exports" "warn" "No NFS exports found on Proxmox" fi else - add_check "truenas-nfs-service" "fail" "Could not check NFS service on TrueNAS via SSH" + add_check "nfs-exports" "fail" "Could not check NFS exports on Proxmox via SSH" + fi +} + +check_nfs_disk_usage() { + if $DRY_RUN; then + add_check "nfs-disk" "ok" "dry-run: would check NFS disk usage" + return + fi + local result + if result=$(timeout 10 ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "root@$NFS_HOST" \ + "df -h /srv/nfs /srv/nfs-ssd 2>/dev/null" 2>/dev/null); then + while IFS= read -r line; do + local mount pct + mount=$(echo "$line" | awk '{print $6}') + pct=$(echo "$line" | awk '{print $5}' | tr -d '%') + [ -z "$pct" ] || ! [[ "$pct" =~ ^[0-9]+$ ]] && continue + if [ "$pct" -ge 90 ]; then + add_check "nfs-disk-$mount" "fail" "$mount is ${pct}% full" + elif [ "$pct" -ge 80 ]; then + add_check "nfs-disk-$mount" "warn" "$mount is ${pct}% full" + else + add_check "nfs-disk-$mount" "ok" "$mount is ${pct}% full" + fi + done <<< "$result" + else + add_check "nfs-disk" "warn" "Could not check NFS disk usage" fi } @@ -116,8 +144,9 @@ check_nfs_pvcs() { } # Run checks -check_truenas_reachable -check_truenas_nfs_service +check_nfs_reachable +check_nfs_exports +check_nfs_disk_usage for node_entry in "${NODES[@]}"; do node_name="${node_entry%%:*}" diff --git a/.claude/scripts/truenas-status.sh b/.claude/scripts/truenas-status.sh deleted file mode 100755 index 055fe2e7..00000000 --- a/.claude/scripts/truenas-status.sh +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -AGENT="truenas-status" -TRUENAS_HOST="root@10.0.10.15" -DRY_RUN=false - -for arg in "$@"; do - case "$arg" in - --dry-run) DRY_RUN=true ;; - esac -done - -checks=() - -add_check() { - local name="$1" status="$2" message="$3" - checks+=("{\"name\": \"$name\", \"status\": \"$status\", \"message\": \"$message\"}") -} - -ssh_cmd() { - timeout 15 ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$TRUENAS_HOST" "$@" 2>/dev/null -} - -check_zfs_pools() { - if $DRY_RUN; then - add_check "zfs-pools" "ok" "dry-run: would check ZFS pool status" - return - fi - - local pool_status - if ! pool_status=$(ssh_cmd "zpool status -x" 2>/dev/null); then - add_check "zfs-pools" "fail" "Could not retrieve ZFS pool status via SSH" - return - fi - - if echo "$pool_status" | grep -q "all pools are healthy"; then - add_check "zfs-pools" "ok" "All ZFS pools are healthy" - else - local degraded_pools - degraded_pools=$(echo "$pool_status" | grep "pool:" | awk '{print $2}' | tr '\n' ', ' | sed 's/,$//') - if [ -n "$degraded_pools" ]; then - add_check "zfs-pools" "fail" "Degraded ZFS pools: $degraded_pools" - else - add_check "zfs-pools" "warn" "ZFS pool status unclear: $(echo "$pool_status" | head -1 | tr '"' "'")" - fi - fi - - # Check pool capacity - local pool_list - if pool_list=$(ssh_cmd "zpool list -H -o name,cap" 2>/dev/null); then - while IFS=$'\t' read -r pool_name cap_pct; do - local cap_num - cap_num=$(echo "$cap_pct" | tr -d '%') - if [ -n "$cap_num" ] && [ "$cap_num" -ge 90 ]; then - add_check "zfs-capacity-$pool_name" "fail" "Pool $pool_name is ${cap_pct} full" - elif [ -n "$cap_num" ] && [ "$cap_num" -ge 80 ]; then - add_check "zfs-capacity-$pool_name" "warn" "Pool $pool_name is ${cap_pct} full" - else - add_check "zfs-capacity-$pool_name" "ok" "Pool $pool_name is ${cap_pct} full" - fi - done <<< "$pool_list" - fi -} - -check_smart_health() { - if $DRY_RUN; then - add_check "smart-health" "ok" "dry-run: would check SMART disk health" - return - fi - - local disk_list - if ! disk_list=$(ssh_cmd "smartctl --scan" 2>/dev/null); then - add_check "smart-health" "warn" "Could not scan disks for SMART status" - return - fi - - local fail_count=0 - local total_count=0 - local failed_disks="" - - while IFS= read -r line; do - local dev - dev=$(echo "$line" | awk '{print $1}') - [ -z "$dev" ] && continue - total_count=$((total_count + 1)) - - local health - if health=$(ssh_cmd "smartctl -H '$dev'" 2>/dev/null); then - if ! echo "$health" | grep -qiE "PASSED|OK"; then - fail_count=$((fail_count + 1)) - failed_disks="$failed_disks $dev" - fi - fi - done <<< "$disk_list" - - if [ "$fail_count" -gt 0 ]; then - add_check "smart-health" "fail" "$fail_count/$total_count disks failing SMART:$failed_disks" - elif [ "$total_count" -gt 0 ]; then - add_check "smart-health" "ok" "All $total_count disks pass SMART health checks" - else - add_check "smart-health" "warn" "No disks found for SMART check" - fi -} - -check_replication() { - if $DRY_RUN; then - add_check "replication" "ok" "dry-run: would check replication task status" - return - fi - - # Check for any running/failed replication tasks via midclt if available - local repl_status - if repl_status=$(ssh_cmd "midclt call replication.query 2>/dev/null" 2>/dev/null); then - local failed - failed=$(echo "$repl_status" | python3 -c " -import sys, json -try: - tasks = json.load(sys.stdin) - failed = [t.get('name','unknown') for t in tasks if t.get('state',{}).get('state','') == 'ERROR'] - print(len(failed)) -except: print('error') -" 2>/dev/null || echo "error") - - if [ "$failed" = "error" ]; then - add_check "replication" "warn" "Could not parse replication task status" - elif [ "$failed" = "0" ]; then - add_check "replication" "ok" "All replication tasks healthy" - else - add_check "replication" "fail" "$failed replication tasks in ERROR state" - fi - else - # Fallback: check if zfs send/recv processes are stuck - local send_procs - send_procs=$(ssh_cmd "pgrep -c 'zfs send' 2>/dev/null || echo 0") - add_check "replication" "warn" "midclt unavailable; $send_procs active zfs send processes" - fi -} - -check_iscsi() { - if $DRY_RUN; then - add_check "iscsi-targets" "ok" "dry-run: would check iSCSI target status" - return - fi - - local target_status - if target_status=$(ssh_cmd "ctladm islist 2>/dev/null || targetcli ls 2>/dev/null" 2>/dev/null); then - local target_count - target_count=$(echo "$target_status" | wc -l | tr -d ' ') - if [ "$target_count" -gt 0 ]; then - add_check "iscsi-targets" "ok" "iSCSI service active with $target_count entries" - else - add_check "iscsi-targets" "warn" "iSCSI service active but no targets listed" - fi - else - # Try checking if the service is at least running - if ssh_cmd "midclt call iscsi.global.config" &>/dev/null; then - add_check "iscsi-targets" "ok" "iSCSI service is configured and running" - else - add_check "iscsi-targets" "warn" "Could not query iSCSI target status" - fi - fi -} - -# Run checks -check_zfs_pools -check_smart_health -check_replication -check_iscsi - -# Determine overall status -overall="ok" -for c in "${checks[@]}"; do - if echo "$c" | grep -q '"status": "fail"'; then - overall="fail" - break - elif echo "$c" | grep -q '"status": "warn"'; then - overall="warn" - fi -done - -# Output JSON -checks_json=$(IFS=,; echo "${checks[*]}") -cat <1024 when routing between VLANs, which NFS `secure` mode rejects. -- **CSI PV volumeAttributes are immutable**: Can't update NFS server in place. Must create new PV/PVC pairs (pattern: append `-host` to PV name). +- **NFS export directory must exist** on the Proxmox host before Terraform can create the PV. ## Shared Variables (never hardcode) -`var.nfs_server` (192.168.1.127 — Proxmox host), `var.nfs_server_truenas` (10.0.10.15 — Immich only), `var.redis_host`, `var.postgresql_host`, `var.mysql_host`, `var.ollama_host`, `var.mail_host` +`var.nfs_server` (192.168.1.127), `var.redis_host`, `var.postgresql_host`, `var.mysql_host`, `var.ollama_host`, `var.mail_host` ## Tier System `0-core` | `1-cluster` | `2-gpu` | `3-edge` | `4-aux` — Kyverno auto-generates LimitRange + ResourceQuota per namespace based on tier label. @@ -100,7 +96,7 @@ Terragrunt-based homelab managing a Kubernetes cluster (5 nodes, v1.34.2) on Pro - **Fix crashed pods**: Run healthcheck first. Safe to delete evicted/failed pods and CrashLoopBackOff pods with >10 restarts. - **OOMKilled**: Check `kubectl describe limitrange tier-defaults -n `. Increase `resources.limits.memory` in the stack's main.tf. - **Add a secret**: `sops set secrets.sops.json '["key"]' '"value"'` then commit. -- **NFS exports**: Create dir on Proxmox host (`/srv/nfs/` or `/srv/nfs-ssd/`). For Immich (TrueNAS only): add to `secrets/nfs_directories.txt`, run `secrets/nfs_exports.sh`. +- **NFS exports**: Create dir on Proxmox host (`ssh root@192.168.1.127 "mkdir -p /srv/nfs/"`), add to `/etc/exports`, run `exportfs -ra`. ## Detailed Reference See `.claude/reference/patterns.md` for: NFS volume code examples, iSCSI details, Kyverno governance tables, anti-AI scraping layers, Terragrunt architecture, node rebuild procedure, archived troubleshooting runbooks index. diff --git a/config.tfvars b/config.tfvars index 3824c21fd8b9ab23a08f966f7b20a363f78ed391..950c9fc0a67c6eb849d97e96932d16964ead97c5 100644 GIT binary patch literal 10275 zcmV+;DBRZoM@dveQdv+`0650u`Oj#+_}o7!Tl=F!2CmI>a?8Cy3{8EvPV0`eC;}*c z?{nfxf556vhy|nFy5m86{C=X7p_xM1e)P5{GggV3bnndpyQk%`_ehPX?;M9{`I0J( z&I=iC${2bXagrw9Dzu#q zx&@n&y@N1qE5^2uA@>&~B|&DPCmv$!&rz|cYUL3PP2sim)WwH?Pw6jmot%HCF|S}< zXK??I6vV#mD&}V`rXAtc)D>XQbcc&Y7~=heOGeU`IF?y@vCpBR&`d?PD{7LNTdbr!326s+SHVyRwoo7B?*H+N*|ImkE7~Y z+-$RfwN)WZ{{LWx_;couZeV=+{zIVdD)^4wFjwqg#Rh3n z#1G0W`&-*UT>n$Y=EgdUqr{Wuv11n>1yFr^`K|(XqvV3NFTYbf{3%o~H4{`JA9JgS zNJPMQX}Z}T%(}2Ci8w_<@)hv1*^S%qF=L?4{sM+NM$!qpR{#nfl;8b^zep<+%O7A3 zqMLC3YolCsy!y*iadiZQ>U&xNnUwsW#LjQ#P1B+yb0vms)HwoL&Lc7@aS|7ZqfNm@ ziL=f}Jz!Vn8HC7|P*CeSM&omtwHl zdIWNfE@D@Dc<-P@6a)qZHUJNm$qYbyYL?!z{4>1q67`bMvWYNP&M{W-?IJ)XF|mR4 zq=WKO-y#?6wbNOs{3;M@$wyDF_FrnXCwR}FbbPJg|HyX9!YV%mCAhKm6iDjc=P)v^;QRQcJY4fc|A$I)Uy6Cw0=$_qTgxb~5o@!9NaqpBWk#Nl zZAiKX>60(=GvidH{=*lYv(HFB!{M5V60zi$yxcbaH~{|8UOIfiV&N^&IJh&EbfO72 z;M^r*v{aB1?W#JVoJwL)I&}ZY4GCW~)3Z8Tspd9pyj12hS&!)i9xp~_W55m^tG~2m zI3l4HTi_soS*W7m$^tS4&7ezp&X^v}nQQD=0o$S>CS10Y_2<1GF4P)(8aDXG^bpJJ z{WOAUMj(dC!DR;YKh<}Hy>|t%_DUuYSfsAbDb)kh+n|-^$tiC%>Lw^Ll93s(xWX0e zS)+uYr}ah>d;Kj77idwL6>}c|!B!no4&cFP6-^n43+eg(bF*oil;ULE>r(EWYV8JE zOJr=77D?4?zKZL&DAa*dzcIV7Y7-?0n;w*&r6PoF!nDbzlj_;EhCdnp=T5YjcUEJG z5^ky!O=c$YS=KxZ)8=4_PB3RpBdbB|vzBR?{rO)oC1-vx?GbqN`yA3%4uru^Km5cU zl@i56y_sCk|H^DdK4ZD(b&tfFV0wd5HyJkWpn-2b;C@~%EvkS>>i^_^QNs?=2Kw5< z*~6E7c5v~)m-CS1VEcboF?rREG>1urD2}0#@;M8RYUTVw9S;fzaw@lV@xLfl{|ua0 z&q);%UvFHiBW2R1W^&<2WNJFE#e~eOGqKbl#ruTQWUg<~hZ17C4_4%JLe-NNz-|WY zXTR8*1R7lxlM{gKAx(2VaKJ)|YXnFP8}xs|{?&)W)eX@dnq){Ng5D;KQ{i(K-Nr{S z!CuV#Wa!E<51bpI>i*#~-*nQ3$57JgTTry_06Ra?gPB#05t{8`WngxO% z2{)c}&=sBC8RvqmbM|UXN~hG&_5U}ZxpbVx(RPA~wA3`wD8a$hNAu3~+M@O;<|`-g zN~u#?YfrTBK188ndW{hF@|E{0>Dtio%NmtGd^NON%*>CaJBP@Wph3RH=ycYk&r+S? zMLH>jjtxLdaukChcNw7V{~PGlzh6iJ+6;17rlD3AZ>$?c|Bo;oH_*Q&xuFZ>LkLuM z%BTvQ(95ZMSKVj|8Uil+1qi9R)qGVxG`4(604wc{!a7;@^-dRVLFLg$ec*vyP)|-G zr3-U|#_t4C0hzo&%XE}_{EgVV$ALp_b4$cUz>ZRqIa(*~ zr>B1xZJqiWbS>rON$fYHS6BFw_7QVQ)x$WoX@m8jQ!ZYS6 z+~r$wG8NcW6*_{}RM{@JMBo|!A`XTw22hr337DV@A9bJ_Cl)!%BnTkQ+rRZb$H?&m zGdidO_$o7aG;U_B+Cqirr&g+C4v~D-$i3_8D$a)ApQh1Z0izHeQ@R37=-^(iawf&u zy~m_97O{p=QWRv{x+ZVygB|9+=18h?c3=iQ1u=(mlM^P(b%5iRg{3jkqmJwRKiW~A z0`#HE8={wsq%jeUKA1h@ei*<*(;_B`hnrx`_E$j{qQ*ksESv!SwuSRVvSLqF)P~sl z&DkCft}q3_kz?DtRt&q^wJ8#XuNDVhXmbMJd(hFq56n-f^r}6g{_&!%6>jHKCu255 z4Gr#0DMJ5KzgD1r7@~bIWb|&1Kh5$)qvR~5;^qJUiuNk@JJVQK&lMY4#VD1Dm1>Qm zLnKy*J(r^+=~o_yobgbH*T9FoOV*26+?jxX7faXB9!o_WKMnkc?0 zlna{t=}UIGds@{o-N5HPA1$*7r&m?yIn02V=NfT->~+GF0m4~^*a-y{pM=(j>K#qFQL~L1H}299x`Tf!Pzj26czWiX~JQs=wjWre=%GYA*HrNL!4ajrlb(~pV_Eb!#EpKE{_^BaS3wMw&Z{XHj>$vW*7=II$_Py`_w=M*ud=Gw_I~bjZ@g1v`)^^pXf1{Y%=KGF3w|$Fq#C zlXv9S)WCJtBsHy1D5xJ}B5;!kW2;(_VC-41BR^v1Y8GU)ej_7mpf}&f+{AK#3)Bwm zB3<#W-L@E6H_LRlx@PQj4RQEzHU(bYgcHZBZ^&KLFA6*MSLy`4L_2n~=q1b%TOY%+vhrp7X;`|DH&Uwh2XmLY#^=&YlA{D_F;xB!p z|G5n}o13kyKiddE$aitHPH1Q^l#|T%oHUL$=*54aF1d}w(XZpoic8?RK@LFYL>w-OY{ev@+-@t^ zueBddCxMk5Uv(sjX8%DA`1g=|7f&7%>Dn5JBig-xsnX6StGwbFy9!D9N|}QbsIklF zdF(>YsGbXCF;xJ%cV)8%*N`{2)A1vxf1G}jv2-ceRFG6Xr5Sw5;_j{OQJ$LZGl+(hTSFS`<`dXYRXRfQc+AmGi!LJL4ibiy|q7ZJ_Ky^sZ zmUYMa>&wdH!H+X5Hv#R2xYMH>i8{Y2AeZ2rvtny)>g%-LvqsXYfeK#n(t7h5WY+$8 z*oP`O81)-sN)J3)gzyI~ZHG%_x2e4tz7(>9Q7;gqeQ*8xCecDMYXkB5vtwie3~Y)% zcnWbP6$wX{N?e8J3ow9Dt8;%BJpB-cf!_IUuZ-hg z<;!9r&f5Rw0o@;1pAdqPHG$`K4dG+39l8P%F^tsQL=2P_K%2zKiJog*3$_Me^X{m; z+ta^=9>UggM*LZ4c$az_wlhF~_CXi*R2_*coFEB|x)-;8efcdCtl`vP&fG+YY+wnWg83T6L{Z{a#LQ!E-eay^onGRZU>G6!`&Q}CWm;-DHX(+t!$LFf8=9mmzzFe;(|}tRrXsBL8xnb z)9S!JN2MjwK0jLnuCQ$K(5}G6%F1uSMTgbhVkI1W1H>yddz<4XA#`gR0t>WwT$ z));&hCDLEPnedqqU^W_?QiDOsI-4ccz(~WfIPHVY#_Ya;X=@GVN%QK_Vf)xrerP+A zh|O(edc^OHRntpIZ2rl{VN?!sy8+lvTWAk3s4LhS@%pzw6S#sc$C0pj$sN~0W8E zeajhO{rO zSE4HFGaR_PDz3D4Giq!Yh@<|69lQvT(x>PQoJf%4zkst`OFpsg?svs?1@rx6(f8%!9r`%G1|)L0j**cdl|!Gy~HUT&Z>coz?ov9Z5C!; z7O3vPx&!psB51N}cCI4+aR}EP--9>bc)029t-T* z>yt@YBs1U}va3un^Ekd1jQ z(q+>E!OeNBbxFWI*uY_pra_z%NLuHp^BMLAPQYZp9h`tG84V9TNLe4UP45Q)0M1L@C6sc2Bu(NwKc3;}0LkI!FI}h!iDagMQYx_2L9N8-St6e8IP8^`-Z?hP;Ln>~ZG(oA z`a!ig4F)%a9aVy+$#C1Ad%SYRh56VH)Tr%xa`m3xI$Cf3`~adDT;6mO`VV`wyX#|j z&P>=o<==CFgb_9j5O#R|Z&dMA5CsO|FnkFm&;GuK(C2y)o>?epG;Qk84DxNLVJwMT zNw-VAs|XfM`TcY_yBpnm9`yFKF6B@26*PVs!?F`G#E@_Wpd4tanR$O%A^$6y@mzh% zYZk^#jw{C7==>)z&OT`G=98y~u97^-Hn)2NIW)855^a?9tJr4dc9CO|QXdv(}Q zqgSM3dMa&B(|z@B1t-YU)75eUSTQx$>YHE6-& z;N#rsA8v2+{@&*&mTiUZARw*1i6$C2ejH(14=!U*VjcVv;BkwOT(VOUzgeR?M4jyj z$jVF7V5M|sn$fVVyw~*o)8H8WxiGpZaND9145r>;vv|dIHCDC8$mB0M>x%oM+#B1V*bqPw&WUGlb#ECyggPu8{(h9D(8Z6JLU|TJjCaB$ z*5zUOrBM=9?zKJg(w2Pv*UesWLQT2RjK84%#Ywu5?iiqLyq+&Yk(bu_*J5pK@kpM6 zgGX$6;_yBV9W>mNkd@+EhA!}ga}#Y8whhxeU2dA#>Ke?!n?q22|23=C?32c?C=(y9 zsM7)!R8Y;AmCMQeGXODpwBYiB{R&31N-Guxbou>OD$#l%pX|n#`h=c@g6#*`(|HwA zL+V_G_TsgIOvA62aHwct^19W>;VcbkD61H9B6uuj01xz0qZ46}lqC;ef*eIj*(0*o zci9R_zxmzXL(-92zxc*sv{_LEu}RMY(V_c8Ydet$-kEU<|Dgwejp8BTj8DI!pOmRackb~UxTkg zMNYD5d5XRMJX!Sqx|2dz+F=>$nrWCP&~#AUwZttB*(uo-4(!D~3Nv3>%I@k>Vl(|f zqpxl*NxV;|_&GE4Z&9|G8yC{hm9^Fi;kJ%(TzPkKaoezKtwnDhLl#^0#1lF>M{Ya( zAA;cYRI%=&f}F~fwz{{bUCt&NS6g@6OH|n0i{$h!jY+qrx6m{Lx0W`reux0YpG5>` z#4@H=TD+80V5#}Q5iE~Y4V@b^7vKx^F4FkqGEZw;yuc~`q!0C%%@)u7A4V1cm_-e> z!dw{u#hEstWJ!!m8ltmTl$h7Y-1sEx(?H{lU}eSS*@@5Lf_^N!eaidr)^|$H?911aOSCbp1E1xuggU@#qqm(=*8~eJk+vG8SP?pm_EiGRDX_-aZT~b z*1l}z;wIq3^mG?<77S(gACZ_9rY2*Cz9}<7D*ryk2o?%tWGXsaD>1XcTb}iwZx2IO z9UU8*ux4dTgwp)R8!KfUW`t=&p}rS7d))8(d_SZhv;+<^GdVq}X4;0I5`etQz{3U6 z-~auk{0OITVU9JzhL7bqYl%`Tpg36QeNAub6_^sezoOq2tSHb9f|DFyH)!tMgjc4Q zw-A*DyZ0nM_JS!4uXXU|0!|2+Ztj2X2n$UG6+hb)3~Cp>vGcONG1 znd|+B#sqpz$E4fOp4Wnk=$YklyfGi1P(UaZ1Bv?^0;Rwz|NU^y4Iw98m?_C-PZgnG zMe2}at}))GAfek)m2oWaN;^dgr3JpkdK!pNl)AF#CV(s2$*;8qX=S*?2 zjl#x!-P62vIv+Dy!Tb>WixKd#DX;KB7OVKSN%>Zgd~weWt>;lSnV_WiRWT>Faqcf^ z=-s+x7XFOFvv-1PE6|??Q%wunuzzm$}Z5N^vXp50(;^m}HeDNBT zeiAd1^e8-Kc4-Au3nzAHkq0MJsjAgE4QlnyR6bMjlD!it%JSu6FuuPp&F#` z7YLpV>)tB>nDNY`wIp4qmuYNmx6bP_SRv0OCv1giExQMY;)|qWW8)t>v5rY}%ssyIFZSo}Jikc|^ zRQ5Vnbgw%CUz_7P-yRok6%GZ|PZYy~pH>{~(A*ZT!=l5;YHjBAsUw`&Zp%uB^qdE8>LuXg>v?tRaU2w?T7n{5Wx`O4F9X`S~^Y}SG2_)R~F*)6Eut_?H|q? z_+Y`MxKSFOVM`E@-?#G)qIB4I46BieYY`%cX_@(E423dRRNzJMjNb zfiPVHd_~1JJqh;mnr7$RqK+ef>s;#xmvbOYR{mKu;}zq)%7AqdE9FS0H;>A05ENQ% zM}rIYew4|1I7{lO5-CfBc*Q;UMyWHu$=2*aNUR-thL(Mi>OglH9cg#x&_{c}mBxQI z0Gk{zx)%mzsIcgh&>kd!mwri8-OHD4Hwj})u4$vLy~C-FpFuQ0=P@%A!)LZXKUMXI zKG*oQm5^pu1M}r~YvKP9ZSzP;9s+|>x22ML$M!$BrgQJ`E1aDXg=>)B5Ji@q-@In% zD&M^@d|dd^M+p7C2yJMIzXk28#-D>s&HqJ2YX0U~1--?*8i5ep0W8ZIH@bR8jp9bfwfum{wmdMPjZWuPP>z$b${x(+K~h~L*6 z8fRmr;S-dU^xU(b`CT;JK-2cOs;Fsz2{c(PDYz3E9>u@Tqy7}@>+cHk;U77KEkb@o z<9#lTQMO=G1J6)S!iSYigX+qLJ3ZghJy82EyJp`@p6)<&(L)YdlD{lX2MK1Je)$lW ziQ-pKy>i=H$UV=7slbg(sfJE=rMW?}j85;2h_Gdg!0vhkbN2zu9$O8k(K#F-Vqj>> z9$*+~-NcglOD&&?<*9S8 z2*6yQftfA3Rl`FltMQ2Z`wh<{MlNAG-nek(Im6CnhDdUpxotKlGu#wkqPdru2OrCT z7Ul;;`Ce^1Ahny$jJMOH!F$5yOmH`C+ZgXX)n1 z{fqNqO^GV*)!Y5Kr)y_i*S|-IWe;`mG`8JH?x}7lep^+V!^!4yK*qsT6NL6H#6p3v z1+Sz+Ctt@>%edmhaYD}NT2y!PZgjF|F4U)c|LANkV4S$>RQTG;^~7wi9lRv16v$wu z_niWeXT;<^{P(x}w>j}K18VF~KBcNZUZ?gg#I-tbEVaGZyM!Y7K)5f}N8XH2%K~)& ztx;@J8uGLVI`x~UZTETr?qkGv3l|zeS+*8$c{W{l)>H;RvbjuDsbmU{!8GT+egODe_-@CU&x5tZBH7rxWe78_0nAij{4hRyk#6 zTYu+^Y1TMm*ZD#3Cd6w9tvsvEM$7b-mp+^{(+8+H zStmK1*al!uBft*qV6SetWF>p*u9BiZ|M3z_AO!0d05hyC=lHDOv~%9`hUwg++<86% zQrNY+p(^5>A+O!RKP6HP76fnh@Ee3c>h0muF0mNDBKN8 zHXh`DQfZA6X61yAMRJjTvt9NwyKsW*aD_0oZ#}=Y-2r=t#^bQxQYogEcFRHs%6fg3Vpe@0 zz&K$JABP5^&=2H~?#1Pu|0Qy|V%m{WoyeWqPPCA8<`$ z=R){aq4dAXDO$sS^AE)=(e=!*PywVP(_txUoy6p z@pak2y=OE6NTu7W>bUo2JNpRb-F0Twj%3v6WPkiDZ`18dAh9A<;$)oZ-~0eC1#8~sH9-|!)~MAz`oRZfK?8nBK)QA$WyO|bb) zQK;?|tuhh=O#abY#<+xOu9PwPUTMpp2PiZ`6NdqK4C8O)yuEDLf&1^d_dRbS%EU8c zayIza$r3P5*tfF(2PVe%RDQ{QESJOB|0~w-&RI{oKKVyUq5^m{nq}X!KvGRFRUlQI z3mMu`M$&tryyts|Z$GGe3atklMd8EydhPwb0`LNulaj1=$w!rbN$u&an7y(|Q94^) zTwc^UHqg;or2eo-6qT+4jqzjQ?zo;g*FbYtw0j)_RE!hY8p{jPDG68yw9=rW`GBIm zIhVzr(cqUQq;GY+}iTgt#tj7!-TK!Oa4o%EDpWOMOWf z8>#CptdHnqDM~liV)AM0v#782A9`ko(1vpuJ)Q~faD{r3Z<1S)l)@4+h)cUGr-qBq$B_;d pYZ(Nc*aQ@w literal 4772 zcmV;V5?k#6M@dveQdv+`0QmV$ZtpTuOAqUm`U9|g<5j`V#6jl_>3;8?w;mXFmO6X^_MK0x-xb z`E__q1d}3@Bo&>TCXSf;#jpKYQj|PAJwoI7Xh8dVt2H|*}1J(;|9*DrZE01u6!~f9LD^m2NVlv ze1nGprDk;BiOuK%2-;#Gm~{M8%p&)qGo*i5D;~O#Bvx)B%ok3*h&st%)!@iAH9x~JMDbIxco-5%l zXdD$Qosj+wB3!(usXu7YfnKO=x14rrvjjIRhUFAP1!OE6_W!vtdx0C1Uf^kYj~!W$ z9fO}wrC$qGDGGW}&LMF++C*B5;eGB*XtJYaKEocfG%S+=foqkVnE+8f#^a%x_nZZ8 zX5vq%b*#Z>YjIXq6KHP`vw`_)PH=KX_G6#Dzyf;j&UaOvS$^0nY8z2zVFT&ct90ju z|AC1(Qvlli#4Wh_YAKUAl1tqyfs%wqCy zYmnzWrhnEB=iIC$1{bKE5iPtIGqkl&fO8EI*@kz8WNL1TdZB3jnK4%8Lejg~n z&AiH-iqZ3%nWS-SC~=Bo8fk4Cja%gkMs?VjgrcX2)MAY+7h18&UerRXal zEWtKz1u={@$<%u%?D0WWG+{KJ` zLjtqo5}N~ns|4)xJw(5qke<*M=YJrHO}T&<)|)KMl<~N{ywN=Yi>1JXy;-4-g5NLe`P*Pe5{aWRv^Wat> zu?&a5ah;n3z94@Q1*&Cmb)1ojG#c)Kw7y+u`W9<<;h~$I%3b35Q;SYEvRbA3rFsL4 zxveKZ#<6!uk|Z*wHs2hA-4~Lab1ny~m2&TlyuYNK!AVxS@PmW)NJ=Fc0IV}ubP-LE zTsag@XW#L1*H{&i=?#HAV@>S?F^MhmQ_X@F zFX@v$sbEa(7z&}++galZ%h7(9*uke{{a28(3dAtei()DY+j`~h7!I$kP zialJxI;qZV4~gsNX5T@cSy&c%5Y9pi_;Q|e`u|)f6duA4Ub*el39<VkAJt$l=pGe1nHi|bt z6cCd>4`vvHIpI#~+yY5>C%GNChcsHvq)0*u(e@ z;z;X~?a$Z=W*#yEY<5W$69~4;_z4rS0YZ%fCOOQgbo8j-Fnwl-pwncKZFQK5&-eN3 zm62y0v6DUnH#L_oGvV5@m7ZFzqZe#mgVDRQxj>2D)%Zx8UDt;_h%fDtQh-QKJiB%F zVzS5E2`(_V8NiI`ZbQ~6(aQf3G1Q}`dXNg&hK#g-&A*``AzYb;dLR+khcR1JEDH@9 zlNO$qdB0{Nrp@>0odfxrK-G3lYojoWsJKN)GN7U6lS)v6jA4c0H;ZCI{Sb7l*DpMX znm%A>oSNmDAn_s75Cc_q_yRPwQz=^5g7b@2J4}QYa+Q^ptH7drjXs{{Sc(~&;x}J4 zS`L6|;NqnAR#8lb1a)WG4&7={(%=t>GHHf)Fskyvb6l?F)2k5>^_2^x4^skl8yx_T zTr~qtapx&`T7ct%l-5$9OAMAE8sQc6kGp4VGQp?o{?Q;=Pjq(JECx!fE%%NO=m_kemnGYq7KsPO>;^-bs`?RaLMdM@OiAGl*o%G7 z^{euieRPlGYioULU1I4oPXU-&`+Fql&I3TU%WB#%ExGq)~-sO0t}$r4$lNzC+9NxzkVJiV9l6-s6&<1a8}f^ z`n?k`ovNJ{Z)Hs>DC6l}P@4iDdA*3bCE=p$+f|B+}oAPd=gtm?#MfkL7} z$^K-_v;YFsU$6aou%OPge`?ulLpClhRt4j&I8|i9@{C1Qq$ecJe>qdbiW4R2?jjv+ zdXt~B)p~HN<%Rea`q+uS5#Sd*YJ-4SV%n@0)Wa#RFF#vx@_;4I`s`+%tJyN0`*WR4#{`4vReruu;z=&dsk-OA7i6ehcwsoJ*9K z+rPycAxpII8$wgOKq(UZ?Pt1WT{}N9VC{bFZ3*5onNm9k#8Itth{(%xbap2eHy2Zi z{IAo69WSDTAD`~q>poH8^rtoE?XFvUrHyrMS%am>_f*!jIZ$OjK<`!~z?&js13MHn zJ-&rALt0qU=Ov-SCwXX?^(>dCIA6q3zmx=_LXlwJ-iE4v zn>$d!l$w|Yo|Pka)o6rK%J>nbOMq>k)g~TljAHN{FPJ8dzXGO^$@}dr^Z79uDy3N5 z6bA$6=?T%QdTXrzJ1G7 z0>Sd)Olr&F-s?EM)5!C3y`jJF$dS)O1Gp(&$eks{*%gA3ViG#RgbAU@TVjHhs1`N* zV0I-s5u4ndod^Cy(okEXO$6%YCHro~=KHd8*D3bu3Ire+up8^SAylRir$?J-qA~1Z zW%SjVmpT@8V>G3O(1DOBWC0_)vR&Yy$*pb)OiVy@_C{`yKC?e$7KG>#pHaMOxjK$y zt-fN`ug%0I-$va?eK^S=cz8X2$e=CKvD`b&OB~PXJN>4;)!C1_WaP!*-fT>fT`HFa{0H&MJCx}nXfwmH1HQ^07ZYsNf4P^s!+g8| z4iUeWVzl4j(U1J94dj(C-?*F>>t(S1-fn43TWFn%D*Hrz|qdaz+o5=3EoN#)I}iQ zvJ->j8?){?FA3 zgeT1OJ4%V2$O{}@h2V0~xbjSC4==J#DZzMa+**oPC%g7B5!>r8wl_)RGK>_d62dLo zCK4u61l&m~!j7%F@9P$A;k}%RGQ7`Y2%4|_i^i)4oC5umU+vL{{p~5d-`gRn+Nr5o zkis|eR!ZR4xg8oK@h~ol0wR}W?EbKFkIGk2mU1Q|RyW(@=xrx#l~9rwiR9k4AZ^%N zE*!RiRz{T1m06#NnY!Z{lgw*XJ_F^cagVjdd072_2LV1 zh|oSTVsT{+lw@)|kco3iiW7|8veKbu!F%JCg~`f45}eT2aB0zTp{ zXX|XKs%3MsG9CG~7JPZRA|O&XF`--nYc&OaF-N&&(E14`j)+`kviqKWN~#4f?!go9 zVbKY&>#|fgkKZ_pY_w=57~4*X=3F&g#mxOy=b6#!p{&MW6=(1E2U(~|lI=6{oeW;Z z*Y5`22{_v%)L-Z>x8N3KRJ}<)?W?-{dobd@Xnxy~!^$l^^@gE+hH?NXbi32b5xnsf z@g7*aEQ;!VIM0Ti!O5UUm?%hR5ol5lw-o#bA*3FQ;7H&?hqg21cvX-lLX<(KkFd7P zz87zTGD)(CN-4I9Ox0jqv!6+;QWhJj;B5by2uZXDM5nIMZf?ZnInw>auRpf&LSoI+ z-u6n?=tV?>53EOVApOKv)9rgXMDSsxw;6MGuO}KCQd_c`@b(nFpsF#flGn0A#zo@t z9>G=hOnoW4YUE3EZ>^;%$`o{gKMWr{us_uqgJ)?W5oMee!-o&VHS5)sjy zd8Q||4pG7Xhtn3wY?260UICp^8OMOt78nEeq8Hkq@cS*bR7QI;Z?@6a))V+Axj9`{ zLo4ps+A`qsYK!(Cvwce9jUJyBXWpF2KoYkAax#!!ztP$XDHP!0f|6lG#F2(@Isbu* z*@8!)EjPc8lrI+&syd_!zeX>*=&%c&a#ORL}H>d>YVuQ}rg{trIlA0ZBdC8t3iAkti zHu;jp>8TWygk2*fgw2RiJ}?$9+Jameg`T^5J{wGetBF-LBWM^Y!stRobfL^5p6!O4 z?r!;}n47G+>eyC>QLPNpqqyXVzU_pArx-ltqx{l`N6?0S@eySk8c@AGY$wzRo$1Dd zkV3mbvx@W*Vfa%ND^O1V&Y7XpG09Z%-q2#ZWsxFM+Ej@#m*x#VjB6ec sda end - subgraph TrueNAS["TrueNAS (10.0.10.15)"] - NFS_Backup["NFS-exported
/mnt/main/*-backup/"] - Media["Media (NFS)
Immich ~800GB
audiobookshelf, servarr, navidrome"] + subgraph NFS_Storage["Proxmox NFS (/srv/nfs)"] + NFS_Backup["NFS dirs
/srv/nfs/*-backup/"] subgraph AppBackups["App-Level Backup CronJobs"] CronDaily["Daily 00:00-00:30
PostgreSQL, MySQL
14d retention"] @@ -57,16 +55,13 @@ graph TB subgraph Layer3["Layer 3: Offsite Sync"] PVEOffsite["PVE → Synology
Sunday 08:00
rsync --files-from
/Backup/Viki/pve-backup/"] - CloudSync["TrueNAS → Synology
Monday 09:00
Cloud Sync (media only)
/Backup/Viki/truenas/"] end sda --> PVEOffsite - Media --> CloudSync Synology["Synology NAS
192.168.1.13
Offsite protection"] PVEOffsite --> Synology - CloudSync --> Synology NFS_Backup -.->|mirrored to sda| NFSMirror @@ -100,14 +95,7 @@ graph LR S01 --> S02 --> S03a --> S03b --> S05 --> S08 - subgraph Monday["Monday"] - M09["09:00 TrueNAS Cloud Sync
Media → Synology"] - end - - S08 -.->|next day| M09 - style Sunday fill:#ffe0b2 - style Monday fill:#e1f5ff ``` ### Physical Disk Layout @@ -159,7 +147,7 @@ graph TB Type -->|"Database"| AppBackup["Use app-level dump
/mnt/backup/nfs-mirror/-backup/
OR Synology/pve-backup/nfs-mirror/
RTO: <10 min"] Type -->|"PVC files"| Proceed["Proceed with
selected restore method"] - Type -->|"Media (NFS)"| CloudSync["Use Synology backup
Synology/truenas//
RTO: varies by size"] + Type -->|"Media (NFS)"| OffsiteMedia["Use Synology backup
Synology/pve-backup/nfs-mirror/
RTO: varies by size"] style Start fill:#ffcdd2 style LVM fill:#c8e6c9 @@ -213,7 +201,7 @@ graph LR | Vault Backup | Weekly Sunday 02:00, 30d | CronJob in `vault` | raft snapshot | | Redis Backup | Weekly Sunday 03:00, 30d | CronJob in `redis` | BGSAVE + copy | | Vaultwarden Integrity Check | Hourly | CronJob in `vaultwarden` | PRAGMA integrity_check → metric | -| TrueNAS Cloud Sync | Monday 09:00 (weekly) | TrueNAS Cloud Sync Task 1 | Media → Synology NAS | +| ~~TrueNAS Cloud Sync~~ | **DECOMMISSIONED** | Was TrueNAS Cloud Sync Task 1 | Replaced by offsite-sync-backup (Path 1) | ## How It Works @@ -251,7 +239,7 @@ Native LVM thin snapshots provide crash-consistent point-in-time recovery for 62 - 4 weekly versions with `--link-dest` hardlink dedup (unchanged files share inodes) **2. NFS Backup Mirror** (`/mnt/backup/nfs-mirror/`): -- Mount TrueNAS NFS ro → rsync DB dump dirs → unmount +- Rsync DB dump dirs from Proxmox NFS (`/srv/nfs/*-backup/`) - Covers: `mysql-backup/`, `postgresql-backup/`, `vault-backup/`, `vaultwarden-backup/`, `redis-backup/`, `etcd-backup/` - Single copy (no rotation) — latest dump only @@ -274,11 +262,11 @@ Native LVM thin snapshots provide crash-consistent point-in-time recovery for 62 ### Layer 2b: Application-Level Backups -K8s CronJobs run inside the cluster, dumping database/state to NFS-exported backup directories. Each service writes to `/mnt/main/-backup/`. +K8s CronJobs run inside the cluster, dumping database/state to NFS-exported backup directories. Each service writes to `/srv/nfs/-backup/` (some legacy paths still use `/mnt/main/-backup/`). **Why needed**: LVM snapshots capture block-level state, but: - Cannot restore individual databases from a PostgreSQL snapshot -- Proxmox CSI LVs are opaque to TrueNAS (raw block devices) +- Proxmox CSI LVs are opaque raw block devices - Need point-in-time recovery for specific apps without full LVM rollback **Daily backups (00:00-00:30)**: @@ -331,28 +319,9 @@ Two independent paths push backups offsite: **Monitoring**: Pushes `offsite_backup_sync_last_success_timestamp` to Pushgateway. Alerts: `OffsiteBackupSyncStale` (>8d), `OffsiteBackupSyncFailing`. -#### Path 2: TrueNAS Media (Cloud Sync) +#### ~~Path 2: TrueNAS Media (Cloud Sync)~~ — DECOMMISSIONED -**Task**: TrueNAS Cloud Sync Task 1 runs `rclone sync` Monday 09:00 -**Source**: `/mnt/main/` (NFS pool on TrueNAS) -**Destination**: `sftp://192.168.1.13/Backup/Viki/truenas` -**Scope**: Media libraries only (Immich ~800GB, audiobookshelf, servarr, navidrome music) - -**Excludes** (Cloud Sync configured to skip): -- `clickhouse/**` (2.47M files, regenerable) -- `loki/**` (68K files, regenerable) -- `prometheus/**` (covered by monthly app backup) -- `frigate/**` (ephemeral recordings) -- `audiblez/**`, `ebook2audiobook/**` (regenerable) -- `ollama/**` (chat history, low value) -- `real-estate-crawler/**` (regenerable) -- `crowdsec/**` (regenerable) -- `servarr/downloads/**` (transient) -- `ytldp/**` (replaceable) -- `iscsi/**`, `iscsi-snaps/**` (raw zvols, backed at app level) -- `*-backup/**` (already mirrored via Path 1) - -**Monitoring**: Existing `CloudSyncStale`, `CloudSyncNeverRun`, `CloudSyncFailing` alerts still apply. +> TrueNAS Cloud Sync was decommissioned along with TrueNAS (2026-04). Media offsite backup is now handled by the Proxmox host `offsite-sync-backup` script (Path 1) which includes NFS media directories in its manifest. The `Synology/Backup/Viki/truenas/` directory on the Synology NAS contains the last Cloud Sync snapshot and is no longer updated. ## Configuration @@ -488,12 +457,12 @@ df -h /mnt/backup **Common causes**: - Backup disk full (check `df -h /mnt/backup`, alert: `BackupDiskFull`) - LV mount failed (check `lvs pve`, `dmesg | grep backup`) -- NFS mount failed (check `showmount -e 10.0.10.15`) +- NFS mount failed (check `showmount -e 192.168.1.127`) **Fix**: 1. If disk full: Clean up old weekly versions manually, adjust retention 2. If LV mount failed: `lvchange -ay backup/data && mount /mnt/backup` -3. If NFS failed: Check TrueNAS availability, verify exports +3. If NFS failed: Check Proxmox NFS availability (`showmount -e 192.168.1.127`), verify exports 4. Manually trigger: `systemctl start weekly-backup.service` ### Offsite Sync Failing @@ -531,12 +500,12 @@ kubectl logs -n dbaas job/postgresql-backup- **Common causes**: - Pod OOMKilled (increase memory limit) -- NFS mount unavailable (check TrueNAS) +- NFS mount unavailable (check Proxmox NFS) - pg_dumpall command failed (check PostgreSQL connectivity) **Fix**: 1. If OOM: Increase `resources.limits.memory` in `stacks/dbaas/backup.tf` -2. If NFS: Verify mount on worker node, restart NFS server if needed +2. If NFS: Verify mount on worker node, restart NFS server on Proxmox host if needed (`systemctl restart nfs-server`) 3. Manually trigger: `kubectl create job --from=cronjob/postgresql-backup manual-backup -n dbaas` ### Vaultwarden Integrity Check Failing @@ -662,7 +631,7 @@ module "nfs_backup" { name = "${var.service_name}-backup" namespace = kubernetes_namespace.service.metadata[0].name nfs_server = var.nfs_server - nfs_path = "/mnt/main/${var.service_name}-backup" + nfs_path = "/srv/nfs/${var.service_name}-backup" } ``` @@ -678,9 +647,9 @@ module "nfs_backup" { │ VaultBackupStale > 8d since last success │ │ VaultwardenBackupStale > 8d since last success │ │ RedisBackupStale > 8d since last success │ -│ CloudSyncStale > 8d since last success │ -│ CloudSyncNeverRun task never completed │ -│ CloudSyncFailing task in error state │ +│ ~~CloudSyncStale~~ REMOVED (TrueNAS decommissioned) │ +│ ~~CloudSyncNeverRun~~ REMOVED (TrueNAS decommissioned) │ +│ ~~CloudSyncFailing~~ REMOVED (TrueNAS decommissioned) │ │ VaultwardenIntegrityFail integrity_ok == 0 │ │ LVMSnapshotStale > 24h since last snapshot │ │ LVMSnapshotFailing snapshot creation failed │ @@ -698,7 +667,7 @@ module "nfs_backup" { - LVM snapshot script: Pushes `lvm_snapshot_last_success_timestamp`, `lvm_snapshot_count`, `lvm_thin_pool_free_percent` - Weekly backup script: Pushes `backup_weekly_last_success_timestamp`, `backup_disk_usage_percent` - Offsite sync script: Pushes `offsite_backup_sync_last_success_timestamp` -- CloudSync monitor: Queries TrueNAS API every 6h, pushes `cloudsync_last_success_timestamp` +- ~~CloudSync monitor~~: Removed (TrueNAS decommissioned) - Vaultwarden integrity: Pushes `vaultwarden_sqlite_integrity_ok` hourly **Alert routing**: @@ -738,7 +707,7 @@ module "nfs_backup" { - — = Not needed (other layers cover it, or data is regenerable/disposable) - excluded = Too large/regenerable, not worth offsite bandwidth -**Note**: All 65 proxmox-lvm PVCs get LVM snapshots (except dbaas+monitoring = 3 PVCs) + file-level backup (except dbaas+monitoring). NFS-backed media relies on TrueNAS Cloud Sync for offsite. +**Note**: All 65 proxmox-lvm PVCs get LVM snapshots (except dbaas+monitoring = 3 PVCs) + file-level backup (except dbaas+monitoring). NFS-backed media is included in the Proxmox host weekly-backup offsite sync. ## Recovery Procedures @@ -761,7 +730,7 @@ Detailed runbooks in `docs/runbooks/`: - Vault: <10 min - Vaultwarden: <5 min - etcd: <20 min (requires cluster rebuild) -- Full cluster from offsite: <4 hours (TrueNAS restore + K8s bootstrap + app deploys) +- Full cluster from offsite: <4 hours (NFS restore + K8s bootstrap + app deploys) ## Related diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 6dd6211b..df06e765 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -23,7 +23,6 @@ graph TB NODE3["k8s-node3
203"] NODE4["k8s-node4
204"] REG["docker-registry
220"] - TN["TrueNAS
9000"] end subgraph Network["Network Bridges"] @@ -48,7 +47,6 @@ graph TB PF --> VMBR1_20 HA --> VMBR0 DEV --> VMBR1_10 - TN --> VMBR1_10 MASTER --> VMBR1_20 NODE1 --> VMBR1_20 @@ -78,7 +76,7 @@ graph TB | Network | VLAN | CIDR | Purpose | |---------|------|------|---------| | Physical | - | 192.168.1.0/24 | Physical devices, Proxmox host (192.168.1.127) | -| Management | 10 | 10.0.10.0/24 | Infrastructure VMs, TrueNAS, devvm | +| Management | 10 | 10.0.10.0/24 | Infrastructure VMs, devvm | | Kubernetes | 20 | 10.0.20.0/24 | K8s cluster nodes and services | ### Virtual Machine Inventory @@ -94,7 +92,7 @@ graph TB | 203 | k8s-node3 | 8 | 32GB | vmbr1:vlan20 | - | Worker node | | 204 | k8s-node4 | 8 | 32GB | vmbr1:vlan20 | - | Worker node | | 220 | docker-registry | 4 | 4GB | vmbr1:vlan20 | 10.0.20.10 | Private Docker registry | -| 9000 | truenas | 16 | 16GB | vmbr1:vlan10 | 10.0.10.15 | NFS storage server | +| ~~9000~~ | ~~truenas~~ | — | — | — | ~~10.0.10.15~~ | **DECOMMISSIONED** — NFS now served by Proxmox host (192.168.1.127) | ### Kubernetes Cluster @@ -103,7 +101,7 @@ graph TB | Version | v1.34.2 | | Nodes | 5 (1 control plane, 4 workers) | | CNI | Calico | -| Storage | NFS (democratic-csi) + Proxmox-LVM (Proxmox CSI) | +| Storage | NFS (Proxmox host, nfs-csi) + Proxmox-LVM (Proxmox CSI) | | Ingress | Traefik v3 | | Total Services | 70+ services across 5 tiers | @@ -164,8 +162,8 @@ Kyverno policies automatically inject namespace labels, LimitRange, ResourceQuot - **Headscale**: Tailscale-compatible mesh VPN control plane **Storage & Security**: -- **TrueNAS**: NFS storage backend (10.0.10.15) -- **democratic-csi**: Dynamic PV provisioning from TrueNAS +- **Proxmox NFS**: NFS storage served directly from Proxmox host (192.168.1.127) at `/srv/nfs` (HDD) and `/srv/nfs-ssd` (SSD) +- **Proxmox CSI**: Block storage via LVM-thin hotplug for databases - **Vaultwarden**: Password manager - **Immich**: Photo management - **CrowdSec**: IPS/IDS with community threat intelligence diff --git a/docs/architecture/storage.md b/docs/architecture/storage.md index 33ea9253..4417145f 100644 --- a/docs/architecture/storage.md +++ b/docs/architecture/storage.md @@ -1,18 +1,24 @@ # Storage Architecture -Last updated: 2026-04-06 +Last updated: 2026-04-13 ## Overview -The cluster uses two storage backends: **Proxmox CSI** for database block storage and **TrueNAS NFS** for application data. +The cluster uses two storage backends: **Proxmox CSI** for database block storage and **Proxmox NFS** for application data. **Block storage (Proxmox CSI)**: 65 PVCs for databases and stateful apps (CNPG PostgreSQL, MySQL InnoDB, Redis, Vaultwarden, Prometheus, Nextcloud, Calibre-Web, Forgejo, FreshRSS, ActualBudget, NovelApp, Headscale, Uptime Kuma, etc.) use `StorageClass: proxmox-lvm`, which provisions thin LVs directly from the Proxmox host's `local-lvm` storage (sdc, 10.7TB RAID1 HDD thin pool). This eliminates the previous double-CoW (ZFS + LVM-thin) path that caused 56 ZFS checksum errors. -**NFS storage (TrueNAS)**: ~100 NFS shares for media libraries (Immich, audiobookshelf, servarr, navidrome), backup targets (`*-backup/` directories), and legacy app data continue to use TrueNAS ZFS at `10.0.10.15` via `StorageClass: nfs-truenas`. +**NFS storage (Proxmox host)**: ~100 NFS shares for media libraries (Immich, audiobookshelf, servarr, navidrome), backup targets (`*-backup/` directories), and app data are served directly from the Proxmox host at `192.168.1.127`. Two NFS export roots exist: +- **HDD NFS**: `/srv/nfs` on ext4 LV `pve/nfs-data` (2TB) — bulk media and backup targets +- **SSD NFS**: `/srv/nfs-ssd` on ext4 LV `ssd/nfs-ssd-data` (100GB) — high-performance data (Immich ML) + +Both `StorageClass: nfs-truenas` (name kept for compatibility) and `StorageClass: nfs-proxmox` (identical) point to the Proxmox host. Migrated from TrueNAS (10.0.10.15) which has been fully decommissioned. **Backup storage (sda)**: 1.1TB RAID1 SAS disk, VG `backup`, LV `data` (ext4), mounted at `/mnt/backup` on PVE host. Dedicated backup disk for weekly PVC file backups, NFS mirrors, pfSense backups, and PVE config. Independent of live storage (sdc). -**Migration (2026-04-02)**: All iSCSI block volumes were migrated from democratic-csi (TrueNAS iSCSI → ZFS → LVM-thin) to Proxmox CSI (direct LVM-thin hotplug). democratic-csi iSCSI driver is deprecated and pending removal. +**Migration (2026-04-02)**: All iSCSI block volumes were migrated from democratic-csi (TrueNAS iSCSI → ZFS → LVM-thin) to Proxmox CSI (direct LVM-thin hotplug). democratic-csi iSCSI driver has been removed. + +**Migration (2026-04)**: TrueNAS (10.0.10.15) fully decommissioned. All NFS storage migrated to the Proxmox host (192.168.1.127). ZFS datasets under `/mnt/main/` and `/mnt/ssd/` moved to ext4 LVs at `/srv/nfs/` and `/srv/nfs-ssd/`. Legacy PVs referencing `/mnt/main/` paths still work (bind-mounted or symlinked on the Proxmox host); new PVs use `/srv/nfs/` and `/srv/nfs-ssd/`. ## Architecture Diagram @@ -21,43 +27,37 @@ graph TB subgraph Proxmox["Proxmox Host (192.168.1.127)"] sdc["sdc: 10.7TB RAID1 HDD
VG pve, LV data (thin pool)
65 proxmox-lvm PVCs"] sda["sda: 1.1TB RAID1 SAS
VG backup, LV data (ext4)
/mnt/backup"] - end - - subgraph TrueNAS["TrueNAS (10.0.10.15)
VMID 9000, 16c/16GB"] - ZFS_Main["ZFS Pool: main
1.64 TiB
32G + 7x256G + 1T disks"] - ZFS_SSD["ZFS Pool: ssd
~256GB SSD
Immich ML, PostgreSQL hot data"] - - ZFS_Main --> NFS_Datasets["NFS Datasets
~100 shares
main/<service>
Media + backup targets"] - - NFS_Datasets --> NFS_Exports["NFS Exports
managed by secrets/nfs_exports.sh"] - - ZFS_SSD --> SSD_Data["Immich ML models"] + NFS_HDD["LV pve/nfs-data (2TB ext4)
/srv/nfs
~100 NFS shares
Media + backup targets"] + NFS_SSD["LV ssd/nfs-ssd-data (100GB ext4)
/srv/nfs-ssd
High-performance data
(Immich ML)"] + NFS_Exports["NFS Exports
managed by /etc/exports"] + NFS_HDD --> NFS_Exports + NFS_SSD --> NFS_Exports end subgraph K8s["Kubernetes Cluster"] - CSI_NFS["democratic-csi-nfs
StorageClass: nfs-truenas
soft,timeo=30,retrans=3"] - CSI_iSCSI["democratic-csi-iscsi
StorageClass: iscsi-truenas
SSH driver"] + CSI_NFS["nfs-csi driver
StorageClass: nfs-truenas / nfs-proxmox
soft,timeo=30,retrans=3"] + CSI_PVE["Proxmox CSI plugin
StorageClass: proxmox-lvm"] NFS_PV["NFS PersistentVolumes
RWX, ~100 volumes"] - iSCSI_PV["iSCSI PersistentVolumes
RWO, ~19 volumes"] + Block_PV["Block PersistentVolumes
RWO, 65 PVCs"] Pods["Application Pods"] DBPods["Database Pods
PostgreSQL CNPG
MySQL InnoDB"] end - NFS_Exports -->|CSI driver| CSI_NFS - iSCSI_Targets -->|CSI driver| CSI_iSCSI + NFS_Exports -->|NFS mount| CSI_NFS + sdc -->|LVM-thin hotplug| CSI_PVE CSI_NFS --> NFS_PV - CSI_iSCSI --> iSCSI_PV + CSI_PVE --> Block_PV NFS_PV --> Pods - iSCSI_PV --> DBPods + Block_PV --> DBPods - style TrueNAS fill:#e1f5ff + style Proxmox fill:#e1f5ff style K8s fill:#fff4e1 - style ZFS_Main fill:#c8e6c9 - style ZFS_SSD fill:#ffe0b2 + style NFS_HDD fill:#c8e6c9 + style NFS_SSD fill:#ffe0b2 ``` ## Components @@ -66,36 +66,38 @@ graph TB |-----------|---------------|----------|---------| | **Proxmox CSI plugin** | Helm chart | Namespace: proxmox-csi | Block storage via LVM-thin hotplug | | **StorageClass `proxmox-lvm`** | RWO, WaitForFirstConsumer | Cluster-wide | Databases and stateful apps | -| TrueNAS VM | VMID 9000, 16c/16GB | Proxmox host (10.0.10.15) | ZFS NFS storage server | -| ZFS pool `main` | 1.64 TiB usable | 32G + 7x256G + 1T disks | NFS data for all services | -| ZFS pool `ssd` | ~256GB SSD | Dedicated SSD | High-performance data (Immich ML) | +| Proxmox NFS (HDD) | LV `pve/nfs-data`, 2TB ext4 | 192.168.1.127:/srv/nfs | Bulk NFS data for all services | +| Proxmox NFS (SSD) | LV `ssd/nfs-ssd-data`, 100GB ext4 | 192.168.1.127:/srv/nfs-ssd | High-performance data (Immich ML) | | nfs-csi | Helm chart | Namespace: nfs-csi | NFS CSI driver | -| StorageClass `nfs-truenas` | RWX, soft mount | Cluster-wide | Default storage for apps | -| ~~democratic-csi-iscsi~~ | **DEPRECATED** | Namespace: iscsi-csi | Replaced by Proxmox CSI (2026-04-02) | -| ~~StorageClass `iscsi-truenas`~~ | **DEPRECATED** | Cluster-wide | Replaced by `proxmox-lvm` | -| TF module `nfs_volume` | `modules/kubernetes/nfs_volume/` | Infra repo | NFS PV/PVC factory | +| StorageClass `nfs-truenas` | RWX, soft mount | Cluster-wide | NFS storage (name kept for compatibility, points to Proxmox) | +| StorageClass `nfs-proxmox` | RWX, soft mount | Cluster-wide | NFS storage (identical to nfs-truenas) | +| TF module `nfs_volume` | `modules/kubernetes/nfs_volume/` | Infra repo | Static NFS PV/PVC factory | +| ~~TrueNAS VM~~ | **DECOMMISSIONED** | Was VMID 9000 at 10.0.10.15 | Replaced by Proxmox NFS (2026-04) | +| ~~democratic-csi-iscsi~~ | **REMOVED** | Was namespace: iscsi-csi | Replaced by Proxmox CSI (2026-04-02) | +| ~~StorageClass `iscsi-truenas`~~ | **REMOVED** | Was cluster-wide | Replaced by `proxmox-lvm` | ## How It Works ### NFS Storage Flow -1. **Dataset creation**: NFS shares are created as ZFS datasets under `main/` (e.g., `main/immich`, `main/nextcloud`) -2. **Export configuration**: `/root/secrets/nfs_exports.sh` on TrueNAS generates `/etc/exports` with per-dataset exports (`/mnt/main/`) -3. **CSI provisioning**: democratic-csi-nfs mounts NFS shares and creates K8s PersistentVolumes -4. **Terraform module**: Stacks use `modules/kubernetes/nfs_volume/` to declaratively create PV + PVC pairs: +1. **Directory creation**: NFS share directories are created under `/srv/nfs/` (HDD) or `/srv/nfs-ssd/` (SSD) on the Proxmox host +2. **Export configuration**: `/etc/exports` on the Proxmox host lists per-directory NFS exports +3. **Terraform module**: Stacks use `modules/kubernetes/nfs_volume/` to declaratively create static PV + PVC pairs: ```hcl module "nfs_data" { source = "../../modules/kubernetes/nfs_volume" name = "immich-data" namespace = kubernetes_namespace.immich.metadata[0].name - nfs_server = var.nfs_server # 10.0.10.15 - nfs_path = "/mnt/main/immich" + nfs_server = var.nfs_server # 192.168.1.127 + nfs_path = "/srv/nfs/immich" } ``` -5. **Pod mount**: Applications reference PVCs in their deployment specs -6. **Mount options**: All NFS mounts use `soft,timeo=30,retrans=3` (set in StorageClass) to prevent indefinite hangs +4. **Pod mount**: Applications reference PVCs in their deployment specs +5. **Mount options**: All NFS mounts use `soft,timeo=30,retrans=3` (set in StorageClass) to prevent indefinite hangs -**CRITICAL**: Never use inline `nfs {}` blocks in pod specs — they default to `hard,timeo=600` which causes 10-minute hangs on network issues. Always use the `nfs-truenas` StorageClass via PVCs. +**Note**: Some legacy PVs still reference `/mnt/main/` paths. These work via compatibility symlinks/bind-mounts on the Proxmox host. New PVs should use `/srv/nfs/` or `/srv/nfs-ssd/`. + +**CRITICAL**: Never use inline `nfs {}` blocks in pod specs — they default to `hard,timeo=600` which causes 10-minute hangs on network issues. Always use the `nfs-truenas` or `nfs-proxmox` StorageClass via PVCs. ### Block Storage Flow (Proxmox CSI) — NEW @@ -127,28 +129,9 @@ SQLite uses `fsync()` to guarantee durability. NFS's soft mount + async semantic **Solution**: Use Proxmox CSI (`proxmox-lvm`) for any SQLite database (Vaultwarden, plotting-book) or local disk (ephemeral). -### Democratic-CSI Sidecar Resources +### ~~Democratic-CSI Sidecar Resources~~ (HISTORICAL — democratic-csi removed) -The Helm chart spawns 17 sidecar containers (driver-registrar, external-provisioner, etc.) across controller + node DaemonSet pods. Each sidecar defaults to `resources: {}`, which gets LimitRange defaults of 256Mi. - -**Fix**: Set explicit resources in `values.yaml`: -```yaml -csiProxy: # TOP-LEVEL key, not nested - resources: - requests: - memory: "32Mi" - limits: - memory: "32Mi" - -controller: - externalProvisioner: - resources: - requests: {memory: "64Mi"} - limits: {memory: "64Mi"} - # ... repeat for all sidecars -``` - -Total footprint: ~1.5Gi → ~400Mi. +> Democratic-csi has been removed along with TrueNAS decommissioning (2026-04). This section is kept for historical reference only. ## Configuration @@ -156,25 +139,23 @@ Total footprint: ~1.5Gi → ~400Mi. | Path | Purpose | |------|---------| -| `/root/secrets/nfs_exports.sh` | TrueNAS: generates `/etc/exports` with all service shares | +| `/etc/exports` (on Proxmox host) | NFS export configuration for all service shares | | `stacks/proxmox-csi/` | Terraform stack for Proxmox CSI plugin + StorageClass | -| `stacks/iscsi-csi/` | **DEPRECATED** — democratic-csi iSCSI driver (pending removal) | -| `stacks/nfs-csi/` | NFS CSI driver | -| `modules/kubernetes/nfs_volume/` | Reusable module for NFS PV/PVC creation | -| `config.tfvars` | Variable `nfs_server = "10.0.10.15"` shared by all stacks | +| `stacks/nfs-csi/` | NFS CSI driver + StorageClasses (`nfs-truenas`, `nfs-proxmox`) | +| `modules/kubernetes/nfs_volume/` | Reusable module for static NFS PV/PVC creation | +| `config.tfvars` | Variable `nfs_server = "192.168.1.127"` shared by all stacks | ### Vault Paths | Path | Contents | |------|----------| -| `secret/viktor/truenas_ssh_key` | SSH private key for democratic-csi SSH driver | -| `secret/viktor/truenas_root_password` | TrueNAS root password (web UI access) | +| ~~`secret/viktor/truenas_ssh_key`~~ | **LEGACY** — was SSH key for democratic-csi SSH driver (TrueNAS decommissioned) | +| ~~`secret/viktor/truenas_root_password`~~ | **LEGACY** — was TrueNAS root password (TrueNAS decommissioned) | ### Terraform Stacks - **`stacks/proxmox-csi/`**: Deploys Proxmox CSI plugin + `proxmox-lvm` StorageClass + node topology labels -- **`stacks/nfs-csi/`**: Deploys NFS CSI driver for TrueNAS -- **`stacks/iscsi-csi/`**: ~~Deploys democratic-csi iSCSI driver~~ — **DEPRECATED**, pending removal +- **`stacks/nfs-csi/`**: Deploys NFS CSI driver + StorageClasses for Proxmox NFS - All application stacks reference NFS volumes via `module "nfs_"` calls - Database PVCs use `storageClass: proxmox-lvm` (CNPG, MySQL Helm VCT, Redis Helm, standalone PVCs) @@ -182,14 +163,11 @@ Total footprint: ~1.5Gi → ~400Mi. NFS exports are NOT managed by Terraform. To add a new service: -1. SSH to TrueNAS: `ssh root@10.0.10.15` -2. Edit `/root/secrets/nfs_exports.sh` -3. Add dataset + export entry: - ```bash - create_nfs_export "main/" "/mnt/main/" - ``` -4. Run the script: `/root/secrets/nfs_exports.sh` -5. Verify: `showmount -e 10.0.10.15` +1. SSH to Proxmox host: `ssh root@192.168.1.127` +2. Create the directory: `mkdir -p /srv/nfs/ && chmod 777 /srv/nfs/` +3. Edit `/etc/exports` — add the export entry +4. Reload exports: `exportfs -ra` +5. Verify: `showmount -e 192.168.1.127` ## Decisions & Rationale @@ -197,26 +175,14 @@ NFS exports are NOT managed by Terraform. To add a new service: - **Simplicity**: No volume provisioning delays, instant mounts - **RWX support**: Multiple pods can share one volume (Nextcloud, Immich) -- **ZFS benefits**: Snapshots, compression, dedup all work at dataset level -- **Good enough**: For SQLite on NFS specifically, we accept the risk for low-value data (logs, caches) but mandate iSCSI for critical DBs +- **Good enough**: For SQLite on NFS specifically, we accept the risk for low-value data (logs, caches) but mandate proxmox-lvm for critical DBs -### Why iSCSI for Databases? +### Why Proxmox CSI for Databases? (formerly iSCSI) - **ACID guarantees**: Block device + local filesystem = real fsync -- **Performance**: No NFS protocol overhead for random I/O -- **Tested**: PostgreSQL CNPG and MySQL InnoDB Cluster both run on iSCSI, zero corruption in 2+ years - -### Why SSH Driver Over API? - -The democratic-csi API driver (`driver: freenas-api-iscsi`) has these issues: -- Requires TrueNAS API credentials in plaintext ConfigMap -- Fails silently when API schema changes between TrueNAS versions -- No retry logic on transient API errors - -SSH driver (`driver: freenas-ssh`) is simpler: -- Direct `zfs` commands, no API translation layer -- SSH key auth (Vault-managed) -- Deterministic error messages +- **Performance**: No NFS protocol overhead for random I/O, no network hop (LVM-thin hotplug direct to VM) +- **Tested**: PostgreSQL CNPG and MySQL InnoDB Cluster both run on proxmox-lvm, zero corruption +- **Single CoW layer**: LVM-thin only, no ZFS double-CoW issues ### Why Soft Mount for NFS? @@ -230,7 +196,7 @@ Soft mount (`soft,timeo=30,retrans=3`) trades availability for responsiveness: - Operations return EIO after timeout → app can handle error - Acceptable for non-critical data paths -**Critical paths**: Databases use iSCSI (not NFS), so soft mount never affects data integrity. +**Critical paths**: Databases use proxmox-lvm (not NFS), so soft mount never affects data integrity. ## Troubleshooting @@ -242,55 +208,23 @@ Soft mount (`soft,timeo=30,retrans=3`) trades availability for responsiveness: ```bash # On K8s node mount | grep nfs -showmount -e 10.0.10.15 +showmount -e 192.168.1.127 -# Check NFS server -ssh root@10.0.10.15 -zfs list | grep main/ +# Check NFS server (Proxmox host) +ssh root@192.168.1.127 +ls -la /srv/nfs/ cat /etc/exports | grep ``` **Fix**: -1. Verify dataset exists: `zfs list main/` +1. Verify directory exists: `ls /srv/nfs/` (or `/srv/nfs-ssd/`) 2. Verify export: `grep /etc/exports` -3. If missing: re-run `/root/secrets/nfs_exports.sh` -4. Restart NFS server: `service nfs-server restart` +3. If missing: add to `/etc/exports` and run `exportfs -ra` +4. Restart NFS server: `systemctl restart nfs-server` -### iSCSI Session Drops +### ~~iSCSI Session Drops~~ (HISTORICAL — iSCSI removed) -**Symptom**: PostgreSQL/MySQL pod restarts, iSCSI reconnection loops - -**Diagnosis**: -```bash -# On K8s node -iscsiadm -m session -dmesg | grep iscsi -journalctl -u iscsid -f -``` - -**Fix**: -1. Check TrueNAS iSCSI service: WebUI → Sharing → iSCSI → Targets -2. Verify hardened timeouts: `iscsiadm -m node -o show | grep timeout` -3. If defaults: re-apply cloud-init or manually update `/etc/iscsi/iscsid.conf` -4. Restart session: - ```bash - iscsiadm -m node -u - iscsiadm -m node -l - ``` - -### Democratic-CSI Sidecar OOMKill - -**Symptom**: `kubectl describe pod` shows sidecar containers OOMKilled - -**Diagnosis**: -```bash -kubectl get events -n democratic-csi | grep OOM -kubectl top pod -n democratic-csi -``` - -**Fix**: -1. Set explicit resources in Helm values (see "Democratic-CSI Sidecar Resources" above) -2. Apply: `terragrunt apply` in `stacks/democratic-csi/` +> iSCSI was replaced by Proxmox CSI (2026-04-02) and TrueNAS has been decommissioned. This section is kept for historical reference only. ### SQLite Corruption on NFS @@ -302,8 +236,8 @@ kubectl top pod -n democratic-csi sqlite3 /data/db.sqlite "PRAGMA integrity_check;" ``` -**Fix**: Migrate to iSCSI -1. Create iSCSI PVC in Terraform stack +**Fix**: Migrate to proxmox-lvm +1. Create proxmox-lvm PVC in Terraform stack 2. Restore from backup to new volume 3. Update deployment to use new PVC 4. Delete old NFS PVC @@ -314,18 +248,18 @@ sqlite3 /data/db.sqlite "PRAGMA integrity_check;" **Diagnosis**: ```bash -# On TrueNAS -zpool iostat -v 5 -arc_summary | grep "Hit Rate" +# On Proxmox host +ssh root@192.168.1.127 +iostat -x 5 +lvs --reportformat json pve/nfs-data ssd/nfs-ssd-data # On K8s node nfsiostat 5 ``` **Optimization**: -1. Check ZFS ARC hit rate (should be >90%) -2. Move hot datasets to SSD pool: `zfs send main/ | zfs recv ssd/` -3. Tune NFS mount: add `rsize=1048576,wsize=1048576` to StorageClass `mountOptions` +1. Move hot data to SSD NFS: relocate from `/srv/nfs/` to `/srv/nfs-ssd/` and update PV path +2. Tune NFS mount: add `rsize=1048576,wsize=1048576` to StorageClass `mountOptions` ## Related @@ -333,5 +267,5 @@ nfsiostat 5 - `docs/runbooks/restore-postgresql.md` - `docs/runbooks/restore-mysql.md` - `docs/runbooks/recover-nfs-mount.md` -- **Architecture**: `docs/architecture/backup-dr.md` (backup strategy using ZFS snapshots) -- **Reference**: `.claude/reference/service-catalog.md` (which services use NFS vs iSCSI) +- **Architecture**: `docs/architecture/backup-dr.md` (backup strategy using LVM snapshots and Proxmox host scripts) +- **Reference**: `.claude/reference/service-catalog.md` (which services use NFS vs proxmox-lvm) diff --git a/scripts/cluster_healthcheck.sh b/scripts/cluster_healthcheck.sh index 546e2a65..9b77b09e 100755 --- a/scripts/cluster_healthcheck.sh +++ b/scripts/cluster_healthcheck.sh @@ -938,15 +938,15 @@ check_kyverno() { check_nfs() { section 20 "NFS Connectivity" - if showmount -e 10.0.10.15 &>/dev/null; then - pass "NFS server 10.0.10.15 reachable (exports listed)" + if showmount -e 192.168.1.127 &>/dev/null; then + pass "NFS server 192.168.1.127 (Proxmox) reachable (exports listed)" json_add "nfs" "PASS" "NFS reachable" - elif nc -z -G 3 10.0.10.15 2049 &>/dev/null; then - pass "NFS server 10.0.10.15 port 2049 open" + elif nc -z -G 3 192.168.1.127 2049 &>/dev/null; then + pass "NFS server 192.168.1.127 port 2049 open" json_add "nfs" "PASS" "NFS port open" else [[ "$QUIET" == true ]] && section_always 20 "NFS Connectivity" - fail "NFS server 10.0.10.15 unreachable — 30+ services depend on NFS" + fail "NFS server 192.168.1.127 (Proxmox) unreachable — 30+ services depend on NFS" json_add "nfs" "FAIL" "NFS unreachable" fi } diff --git a/scripts/weekly-backup.sh b/scripts/weekly-backup.sh index c9077ea4..108c9f87 100644 --- a/scripts/weekly-backup.sh +++ b/scripts/weekly-backup.sh @@ -6,9 +6,9 @@ set -euo pipefail # --- Configuration --- BACKUP_ROOT="/mnt/backup" -NFS_SERVER="10.0.10.15" -NFS_BASE="/mnt/main" -NFS_MOUNT="/mnt/nfs-truenas" +NFS_SERVER="192.168.1.127" +NFS_BASE="/srv/nfs" +NFS_MOUNT="/mnt/nfs-proxmox" PVC_MOUNT="/tmp/pvc-mount" PUSHGATEWAY="${WEEKLY_BACKUP_PUSHGATEWAY:-http://10.0.20.100:30091}" PUSHGATEWAY_JOB="weekly-backup" diff --git a/secrets/nfs_exports.sh b/secrets/nfs_exports.sh deleted file mode 100755 index 2e1cd90c2b6001a57b04b22ad72a444f47b1463c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 809 zcmV+^1J?WiM@dveQdv+`0M%>CM>YyS^z~XXa)>>^i)`jsyQ2mrcWjZt=X@NXo}tXy zTvM3KGrn8al2yr6SsvQIIT`kt4f5+<6oVS-k%YLzBT6%|V^OB(Uj^ouN1}8^UJ*%5 zA3^ISfTt_%@YvKC4Jc-DcdV`ZepsYWovq_nM;3iQ9G1C!*7h$qy<3D4w^lK`3n94JPq?0_?JA8oPSiY-`b;p5b z8%yy#KBGOil@sen+sF>=j*Bv&2><(ACAa<(8m)n`$HWdEWj1$BBhhtDoRAR@r*VrC zxY<})&@E2Cs*K4Exx-A~`}{pdw+-IHeu7zE+C+6|wi8|%36)hiiOJ=S)97<1{zcWK zkqVJh2#3QruA7528b|RRGLkfI{(XI}4TfC3d9|NcF~$6MA=X1j%=7^kB4#75n3kqi z-EdE&;}?U-HDTy{t&qm%C`#sGOXX$(qTw*JRK024O^2*@hU*s{>I4a6JOhFVtJd&p z>*Tb98eKS-n$S~_cSq_uFw*wG30sL;m11ySeC&G0M$Id=o|>ZG!-~mk@eLy~*TEj* zTL~tdX$^c&#}DdoUv)Boh+WQGfqfbRxf4E$vifaHUUj`BMR5F~4s6g5rG;6?5*HM8 zYoHPk%ZVDrwt6<#^{S-Q3;teUU+x*ja<+b(@H%3HW?}~J=O2cMND}Bi<}V8`T)$%) zFFU=aDsecSq4aX3Tby81-7+a9dA@$#v5e_mh>z6~O!) z`DWmPle`BFZl<)MJnk-)HVgQ4Dl&xpUIuNXKfV*=-EIcC@r0*W-fI%3vyM~ZTUBJ* z>QL058p9w1Wr2rbFZ|h%eYAR=&5{Pl=gY@`(NY)WQbV@Sp}GoG$Cxpan~*=!PNtpd n@FZn(mg7121#Zyw2^at)&^>d!g3{k{XK8s;0xi2JMS}V9VwsAd diff --git a/stacks/freedify/factory/main.tf b/stacks/freedify/factory/main.tf index 7e5224ef..fdaff5db 100755 --- a/stacks/freedify/factory/main.tf +++ b/stacks/freedify/factory/main.tf @@ -63,11 +63,11 @@ variable "ha_sofia_token" { } variable "nfs_music_server" { type = string - default = "10.0.10.15" + default = "192.168.1.127" } variable "nfs_music_path" { type = string - default = "/mnt/main/freedify-music" + default = "/srv/nfs/freedify-music" } diff --git a/stacks/immich/.terraform.lock.hcl b/stacks/immich/.terraform.lock.hcl index 8830db04..eca3dfe9 100644 --- a/stacks/immich/.terraform.lock.hcl +++ b/stacks/immich/.terraform.lock.hcl @@ -5,6 +5,7 @@ provider "registry.terraform.io/hashicorp/helm" { version = "3.1.1" hashes = [ "h1:47CqNwkxctJtL/N/JuEj+8QMg8mRNI/NWeKO5/ydfZU=", + "h1:5b2ojWKT0noujHiweCds37ZreRFRQLNaErdJLusJN88=", "zh:1a6d5ce931708aec29d1f3d9e360c2a0c35ba5a54d03eeaff0ce3ca597cd0275", "zh:3411919ba2a5941801e677f0fea08bdd0ae22ba3c9ce3309f55554699e06524a", "zh:81b36138b8f2320dc7f877b50f9e38f4bc614affe68de885d322629dd0d16a29", @@ -24,6 +25,7 @@ provider "registry.terraform.io/hashicorp/kubernetes" { version = "3.0.1" hashes = [ "h1:P0c8knzZnouTNFIRij8IS7+pqd0OKaFDYX0j4GRsiqo=", + "h1:vyHdH0p6bf9xp1NPePObAJkXTJb/I09FQQmmevTzZe0=", "zh:02d55b0b2238fd17ffa12d5464593864e80f402b90b31f6e1bd02249b9727281", "zh:20b93a51bfeed82682b3c12f09bac3031f5bdb4977c47c97a042e4df4fb2f9ba", "zh:6e14486ecfaee38c09ccf33d4fdaf791409f90795c1b66e026c226fad8bc03c7", @@ -44,6 +46,7 @@ provider "registry.terraform.io/hashicorp/vault" { constraints = "~> 4.0" hashes = [ "h1:GPfhH6dr1LY0foPBDYv9bEGifx7eSwYqFcEAOWOUxLk=", + "h1:aHqgWQhDBMeZO9iUKwJYMlh4q+xNMUlMIcjRbF4d02Y=", "zh:269ab13433f67684012ae7e15876532b0312f5d0d2002a9cf9febb1279ce5ea6", "zh:4babc95bf0c40eb85005db1dc2ca403c46be4a71dd3e409db3711a56f7a5ca0e", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", diff --git a/stacks/immich/backend.tf b/stacks/immich/backend.tf index 21d6b261..aa78688f 100644 --- a/stacks/immich/backend.tf +++ b/stacks/immich/backend.tf @@ -1,6 +1,6 @@ # Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa terraform { backend "local" { - path = "/Users/viktorbarzin/code/infra/state/stacks/immich/terraform.tfstate" + path = "/home/wizard/code/infra/state/stacks/immich/terraform.tfstate" } } diff --git a/stacks/immich/main.tf b/stacks/immich/main.tf index f9ff83b6..89a3e823 100644 --- a/stacks/immich/main.tf +++ b/stacks/immich/main.tf @@ -17,7 +17,7 @@ variable "immich_version" { # Change me to upgrade default = "v2.7.4" } -variable "nfs_server" { type = string } +variable "proxmox_host" { type = string } variable "redis_host" { type = string } @@ -27,71 +27,70 @@ module "tls_secret" { tls_secret_name = var.tls_secret_name } -# NFS volumes for immich-server -module "nfs_backups" { +# NFS volumes on Proxmox host (migrated from TrueNAS 2026-04-13) + +module "nfs_backups_host" { source = "../../modules/kubernetes/nfs_volume" - name = "immich-backups" + name = "immich-backups-host" namespace = kubernetes_namespace.immich.metadata[0].name - nfs_server = var.nfs_server - nfs_path = "/mnt/main/immich/immich/backups" + nfs_server = var.proxmox_host + nfs_path = "/srv/nfs/immich/backups" } -module "nfs_encoded_video" { +module "nfs_encoded_video_host" { source = "../../modules/kubernetes/nfs_volume" - name = "immich-encoded-video" + name = "immich-encoded-video-host" namespace = kubernetes_namespace.immich.metadata[0].name - nfs_server = var.nfs_server - nfs_path = "/mnt/main/immich/immich/encoded-video" + nfs_server = var.proxmox_host + nfs_path = "/srv/nfs/immich/encoded-video" } -module "nfs_library" { +module "nfs_library_host" { source = "../../modules/kubernetes/nfs_volume" - name = "immich-library" + name = "immich-library-host" namespace = kubernetes_namespace.immich.metadata[0].name - nfs_server = var.nfs_server - nfs_path = "/mnt/main/immich/immich/library" + nfs_server = var.proxmox_host + nfs_path = "/srv/nfs/immich/library" } -module "nfs_profile" { +module "nfs_profile_host" { source = "../../modules/kubernetes/nfs_volume" - name = "immich-profile" + name = "immich-profile-host" namespace = kubernetes_namespace.immich.metadata[0].name - nfs_server = var.nfs_server - nfs_path = "/mnt/main/immich/immich/profile" + nfs_server = var.proxmox_host + nfs_path = "/srv/nfs/immich/profile" } -module "nfs_thumbs" { +module "nfs_thumbs_host" { source = "../../modules/kubernetes/nfs_volume" - name = "immich-thumbs" + name = "immich-thumbs-host" namespace = kubernetes_namespace.immich.metadata[0].name - nfs_server = var.nfs_server - nfs_path = "/mnt/ssd/immich/thumbs" + nfs_server = var.proxmox_host + nfs_path = "/srv/nfs-ssd/immich/thumbs" } -module "nfs_upload" { +module "nfs_upload_host" { source = "../../modules/kubernetes/nfs_volume" - name = "immich-upload" + name = "immich-upload-host" namespace = kubernetes_namespace.immich.metadata[0].name - nfs_server = var.nfs_server - nfs_path = "/mnt/main/immich/immich/upload" + nfs_server = var.proxmox_host + nfs_path = "/srv/nfs/immich/upload" } -# NFS volume for immich-postgresql (shared with backup cronjob) -module "nfs_postgresql" { +module "nfs_postgresql_host" { source = "../../modules/kubernetes/nfs_volume" - name = "immich-postgresql-data" + name = "immich-postgresql-data-host" namespace = kubernetes_namespace.immich.metadata[0].name - nfs_server = var.nfs_server - nfs_path = "/mnt/main/immich/data-immich-postgresql" + nfs_server = var.proxmox_host + nfs_path = "/srv/nfs/immich/postgresql" } -# NFS volume for immich-machine-learning cache -module "nfs_ml_cache" { +module "nfs_ml_cache_host" { source = "../../modules/kubernetes/nfs_volume" - name = "immich-ml-cache" + name = "immich-ml-cache-host" namespace = kubernetes_namespace.immich.metadata[0].name - nfs_server = var.nfs_server - nfs_path = "/mnt/ssd/immich/machine-learning" + nfs_server = var.proxmox_host + nfs_path = "/srv/nfs-ssd/immich/machine-learning" } resource "kubernetes_namespace" "immich" { @@ -303,37 +302,37 @@ resource "kubernetes_deployment" "immich_server" { volume { name = "backups" persistent_volume_claim { - claim_name = module.nfs_backups.claim_name + claim_name = module.nfs_backups_host.claim_name } } volume { name = "encoded-video" persistent_volume_claim { - claim_name = module.nfs_encoded_video.claim_name + claim_name = module.nfs_encoded_video_host.claim_name } } volume { name = "library" persistent_volume_claim { - claim_name = module.nfs_library.claim_name + claim_name = module.nfs_library_host.claim_name } } volume { name = "profile" persistent_volume_claim { - claim_name = module.nfs_profile.claim_name + claim_name = module.nfs_profile_host.claim_name } } volume { name = "thumbs" persistent_volume_claim { - claim_name = module.nfs_thumbs.claim_name + claim_name = module.nfs_thumbs_host.claim_name } } volume { name = "upload" persistent_volume_claim { - claim_name = module.nfs_upload.claim_name + claim_name = module.nfs_upload_host.claim_name } } } @@ -478,7 +477,7 @@ resource "kubernetes_deployment" "immich-postgres" { volume { name = "postgresql-persistent-storage" persistent_volume_claim { - claim_name = module.nfs_postgresql.claim_name + claim_name = module.nfs_postgresql_host.claim_name } } } @@ -646,7 +645,7 @@ resource "kubernetes_deployment" "immich-machine-learning" { volume { name = "cache" persistent_volume_claim { - claim_name = module.nfs_ml_cache.claim_name + claim_name = module.nfs_ml_cache_host.claim_name } } } @@ -771,7 +770,7 @@ resource "kubernetes_cron_job_v1" "postgresql-backup" { volume { name = "postgresql-backup" persistent_volume_claim { - claim_name = module.nfs_postgresql.claim_name + claim_name = module.nfs_postgresql_host.claim_name } } } diff --git a/stacks/monitoring/modules/monitoring/main.tf b/stacks/monitoring/modules/monitoring/main.tf index 80e09cfe..41e313a3 100644 --- a/stacks/monitoring/modules/monitoring/main.tf +++ b/stacks/monitoring/modules/monitoring/main.tf @@ -95,8 +95,8 @@ resource "kubernetes_cron_job_v1" "monitor_prom" { } # ----------------------------------------------------------------------------- -# Cloud Sync Monitor — check TrueNAS Cloud Sync job status, push to Pushgateway -# Runs every 6h. Alert fires if no successful sync in 8 days. +# Cloud Sync Monitor — DEPRECATED: TrueNAS decommissioned 2026-04-13 +# TODO: Remove this resource entirely once TrueNAS VM is shut down # ----------------------------------------------------------------------------- resource "kubernetes_cron_job_v1" "cloudsync_monitor" { metadata { @@ -123,11 +123,11 @@ resource "kubernetes_cron_job_v1" "cloudsync_monitor" { set -euo pipefail apk add --no-cache curl jq - # Query TrueNAS Cloud Sync tasks + # Query TrueNAS Cloud Sync tasks (TrueNAS deprecated — this monitor should be removed) RESPONSE=$(curl -sf -H "Authorization: Bearer $TRUENAS_API_KEY" \ "http://10.0.10.15/api/v2.0/cloudsync" 2>&1) || { - echo "ERROR: Failed to query TrueNAS API" - exit 1 + echo "WARN: TrueNAS API unreachable (VM deprecated)" + exit 0 } # Parse each task's last successful run diff --git a/stacks/monitoring/modules/monitoring/prometheus_chart_values.tpl b/stacks/monitoring/modules/monitoring/prometheus_chart_values.tpl index fab25bf7..7572d4d2 100755 --- a/stacks/monitoring/modules/monitoring/prometheus_chart_values.tpl +++ b/stacks/monitoring/modules/monitoring/prometheus_chart_values.tpl @@ -1013,7 +1013,7 @@ serverFiles: labels: severity: critical annotations: - summary: "Only {{ $value | printf \"%.0f\" }} node(s) have NFS activity — TrueNAS (10.0.10.15) may be down (need ≥2)" + summary: "Only {{ $value | printf \"%.0f\" }} node(s) have NFS activity — Proxmox NFS (192.168.1.127) may be down (need ≥2)" - name: K8s Health rules: - alert: PodCrashLooping diff --git a/stacks/nfs-csi/modules/nfs-csi/main.tf b/stacks/nfs-csi/modules/nfs-csi/main.tf index 5fa186c9..c68c4875 100644 --- a/stacks/nfs-csi/modules/nfs-csi/main.tf +++ b/stacks/nfs-csi/modules/nfs-csi/main.tf @@ -87,7 +87,7 @@ resource "kubernetes_storage_class" "nfs_truenas" { ] parameters = { - server = "192.168.1.127" + server = var.nfs_server share = "/srv/nfs" } }