diff --git a/scripts/t3-autoupdate.sh b/scripts/t3-autoupdate.sh index bdd26b87..a3928211 100644 --- a/scripts/t3-autoupdate.sh +++ b/scripts/t3-autoupdate.sh @@ -21,7 +21,7 @@ # - canary rollout: restart idle instances ONE AT A TIME, verifying pairing # through the real dispatch after each, and roll back (binary + that user's DB) # + self-freeze on the first failure — active-agent instances are deferred, -# never killed; +# never killed (deferred instances are recorded for t3-migrate-idle to drain); # - rollback target is the recorded LAST-GOOD build, not "whatever was installed". # Detection backstop (real-user pairing failure/fallback) lives in the dispatch # logs + Loki alerts (T3PairingBroken / T3PairFallbackHigh / T3AutoUpdate*). @@ -29,24 +29,17 @@ # Full procedure + manual rollback: docs/runbooks/t3-version-bump.md. set -uo pipefail +# ---- autoupdate-specific config (shared config + helpers come from the lib) ----- T3_TRACK="${T3_TRACK:-nightly}" # npm dist-tag to follow (nightly | latest) T3_PIN="${T3_PIN:-}" # optional HARD pin to an exact version (disables tracking) -FREEZE_FILE="${T3_FREEZE_FILE:-/etc/t3-autoupdate.freeze}" -STATE_DIR="${T3_STATE_DIR:-/var/lib/t3-autoupdate}" -LAST_GOOD_FILE="$STATE_DIR/last-good" -BACKUP_DIR="${T3_BACKUP_DEST:-/var/backups/t3-state}" SMOKE_PORT="${T3_SMOKE_PORT:-3799}" -DISPATCH="${T3_DISPATCH:-127.0.0.1:3780}" -USER_MAP="${T3_USER_MAP:-/etc/ttyd-user-map}" DRY_RUN="${T3_DRY_RUN:-0}" TMPROOT="${T3_TMPDIR:-/var/tmp}" # health-check scratch on DISK — /tmp is a 2G tmpfs and a populated state.sqlite (~hundreds of MB) overflows it -LOG() { logger -t t3-autoupdate "$*"; echo "t3-autoupdate: $*"; } -ver() { t3 --version 2>/dev/null | awk '{print $NF}' | sed 's/^v//'; } -# OS users owning a ~/.t3 (RHS of each non-comment "authentik=os_user" map line). -osusers() { awk -F= '!/^[[:space:]]*#/&&NF==2{gsub(/[[:space:]]/,"",$2);print $2}' "$USER_MAP" 2>/dev/null | sort -u; } -# authentik username for an OS user (reverse map; first match) — for dispatch verify. -ak_for() { awk -F= -v u="$1" '!/^[[:space:]]*#/&&NF==2{gsub(/[[:space:]]/,"",$1);gsub(/[[:space:]]/,"",$2);if($2==u){print $1;exit}}' "$USER_MAP" 2>/dev/null; } +LOG_TAG=t3-autoupdate +# shellcheck source=scripts/t3-safe-restart.sh +. "${T3_SAFE_RESTART_LIB:-/usr/local/lib/t3-safe-restart.sh}" + # is $1 a strictly-newer version than $2 (version-sort)? newer() { [ "$1" != "$2" ] && [ "$(printf '%s\n%s\n' "$1" "$2" | sort -V | tail -1)" = "$1" ]; } @@ -86,27 +79,21 @@ LOG "candidate: $current -> $target (track=$T3_TRACK, last_good=$last_good, dry_ # ---- helpers: backup, health-check, rollback, restart-verify -------------------- # Online consistent per-user snapshot (run AS the owner so WAL stays owned; never # stops the serve). Sets $ADMIN_SEED to wizard's backup for the migration health -# check. Mirrors t3-backup-state.sh. +# check. Mirrors t3-backup-state.sh. (backup_user lives in the shared lib.) ADMIN_SEED="" backup_all() { - local u src out dst ts; ts="$(date +%Y%m%d-%H%M%S)" + local u dst for u in $(osusers); do - src="/home/$u/.t3/userdata/state.sqlite"; [ -f "$src" ] || continue - out="$BACKUP_DIR/$u"; dst="$out/state-prebump-$target-$ts.sqlite" - install -d -o "$u" -g "$u" -m700 "$out" 2>/dev/null || mkdir -p "$out" - if runuser -u "$u" -- timeout "${T3_BACKUP_TIMEOUT:-900}" sqlite3 "$src" "VACUUM INTO '$dst'" 2>/dev/null && [ -s "$dst" ]; then + if dst="$(backup_user "$u")"; then LOG "pre-bump backup: $u -> $dst ($(stat -c%s "$dst" 2>/dev/null) bytes)" [ "$u" = "wizard" ] && ADMIN_SEED="$dst" else - LOG "WARN: pre-bump backup FAILED for $u ($src)"; rm -f "$dst" + LOG "WARN: pre-bump backup FAILED for $u (/home/$u/.t3/userdata/state.sqlite)" fi done [ -n "$ADMIN_SEED" ] || ADMIN_SEED="$(ls -1t "$BACKUP_DIR"/*/"state-prebump-$target-"*.sqlite 2>/dev/null | head -1)" } -# newest pre-bump backup taken THIS run for a user (for restore-on-rollback). -prebump_of() { ls -1t "$BACKUP_DIR/$1/state-prebump-$target-"*.sqlite 2>/dev/null | head -1; } - # health_check [seed_db]: start a throwaway serve (seeded with a copy of a # real populated DB if given, so the forward migration runs on real data), then do # the real mint -> credential-exchange -> t3_session pairing handshake with the @@ -143,27 +130,12 @@ health_check() { rm -rf "$dir"; return 1 } -# roll the GLOBAL binary back to last-good. Pre-restart failures need only this -# (no real DB migrated yet); post-restart failures also restore the user's DB. -rollback_binary() { - LOG "rolling back binary $target -> $last_good" - if npm i -g "t3@$last_good" >/dev/null 2>&1; then LOG "rolled back to $last_good"; return 0; fi - LOG "ROLLBACK FAILED — could not reinstall t3@$last_good (t3 may be broken; manual fix per runbook)"; return 1 -} - # is this t3-serve@ running an active agent (claude/codex/opencode)? never restart those. unit_busy() { local unit="$1" pid; pid="$(systemctl show -p MainPID --value "$unit" 2>/dev/null)" [ -n "$pid" ] && [ "$pid" != "0" ] && pgrep -aP "$pid" 2>/dev/null | grep -qiE 'claude|codex|opencode' } -# verify a user's pairing through the REAL dispatch (mint -> exchange -> cookie). -verify_pairing() { - local u="$1" ak out; ak="$(ak_for "$u")"; [ -n "$ak" ] || { LOG "no authentik mapping for $u — skipping dispatch verify"; return 0; } - out="$(curl -s -i --max-time 10 -H "X-authentik-username: $ak" -H 'Sec-Fetch-Dest: document' "http://$DISPATCH/" 2>/dev/null)" - printf '%s' "$out" | grep -qi '^set-cookie:[[:space:]]*t3_session=' -} - # ---- 3. DRY RUN: preview only (install candidate to temp prefix, gate it) ------- if [ "$DRY_RUN" = "1" ]; then LOG "DRY_RUN: would back up [$(osusers | tr '\n' ' ')]; testing candidate $target in a temp prefix (no global change, no restarts)" @@ -196,31 +168,15 @@ restarted=0; deferred=0 for unit in $(systemctl list-units --type=service --state=running --no-legend 't3-serve@*' 2>/dev/null | awk '{print $1}'); do u="$(printf '%s' "$unit" | sed -n 's/^t3-serve@\(.*\)\.service$/\1/p')"; [ -n "$u" ] || continue if unit_busy "$unit"; then - LOG "deferring $unit (active agent) — migrates on its next idle restart"; deferred=$((deferred+1)); continue + LOG "deferring $unit (active agent) — migrates on its next idle restart" + mkdir -p "$DEFER_DIR" 2>/dev/null && printf '%s\n' "$target" >"$DEFER_DIR/$u" # record for t3-migrate-idle + deferred=$((deferred+1)); continue fi - systemctl restart "$unit" || LOG "WARN: systemctl restart $unit returned non-zero" - ok=0 - for _ in $(seq 1 15); do - if verify_pairing "$u"; then ok=1; break; fi - sleep 2 - done - if [ "$ok" = "1" ]; then - LOG "restarted $unit -> $target (pairing verified via dispatch)"; restarted=$((restarted+1)) + if safe_restart_unit "$unit" "$u"; then + restarted=$((restarted+1)) + rm -f "$DEFER_DIR/$u" 2>/dev/null # now current — clear any stale marker else - LOG "HEALTH-CHECK FAILED: $u pairing broken AFTER restart onto $target — rolling back + restoring its DB" - rollback_binary - bak="$(prebump_of "$u")" - if [ -n "$bak" ]; then - systemctl stop "$unit" 2>/dev/null - if install -o "$u" -g "$u" -m600 "$bak" "/home/$u/.t3/userdata/state.sqlite" 2>/dev/null; then - rm -f "/home/$u/.t3/userdata/state.sqlite-wal" "/home/$u/.t3/userdata/state.sqlite-shm" - LOG "restored $u state.sqlite from $bak" - fi - systemctl start "$unit" 2>/dev/null - fi - touch "$FREEZE_FILE" 2>/dev/null - LOG "FROZEN ($FREEZE_FILE) after canary $u failed on $target; last_good stays $last_good — investigate, then remove the freeze file to resume" - exit 1 + exit 1 # frozen by safe_restart_unit — preserve today's behavior fi done