t3-dispatch: re-pair on present-but-invalid t3_session cookie

The dispatcher only re-paired on an ABSENT cookie. After the 2026-06-09
auth-schema rollback wiped all server-side sessions, browsers kept dead
30-day t3_session cookies; the dispatcher proxied them straight through
and t3 rendered its pair page ("all users must pair again").

Now a present cookie on a top-level document navigation is validated via
the instance's /api/auth/session and re-paired on authenticated:false.
Gated to document navs (Sec-Fetch-Dest: document, else Accept: text/html)
so XHR/asset/WebSocket sub-requests are never answered with a 302; fails
open (proxy through) on any validation error. Unit + handler tests added.

[ci skip]

Co-Authored-By: Claude <noreply@anthropic.com>

docs/plans/2026-06-01-t3-auto-provision-design.md

@@ -85,12 +85,14 @@ Replaces the in-cluster nginx `t3-dispatch` (the session-mint needs `sudo` + loc
 
 Per request (Authentik forward-auth has injected a trustworthy `X-authentik-username`):
 1. Resolve `X-authentik-username` → OS user via `/etc/ttyd-user-map`. No mapping → **403**.
-2. **Has a valid t3 session cookie?** → reverse-proxy (incl. WebSocket upgrade) to `127.0.0.1:<T3_PORT>`. (Steady state — the common path.)
-3. **No cookie** (first visit / expired) → auto-pair:
+2. **Has a valid t3 session cookie?** → reverse-proxy (incl. WebSocket upgrade) to `127.0.0.1:<T3_PORT>`. (Steady state — the common path.) Sub-requests (XHR/asset/WebSocket) take the cookie at face value; on a **top-level document navigation** the cookie is verified against the instance's `GET /api/auth/session` so a present-but-dead cookie doesn't slip through.
+3. **No cookie, or an invalid cookie on a document navigation** (first visit / expired / server-side session wiped) → auto-pair:
    - `sudo -u <os_user> t3 auth pairing create --base-dir /home/<os_user>/.t3 --ttl 5m --json` → one-time token.
    - exchange it at the instance's `POST /api/auth/bootstrap` → capture the returned `Set-Cookie`.
    - relay that `Set-Cookie` to the browser + `302 /`. Browser now holds the t3 session cookie → next request is the steady-state path. **Login → straight in.**
 
+> **As-built note (2026-06-09):** the first implementation re-paired only on an *absent* cookie. After an auth-schema rollback wiped every server-side session, browsers still held live-looking-but-dead 30-day `t3_session` cookies, which the dispatcher proxied straight through → t3 rendered its pair page (the "all users must pair again" incident). Fixed by validating a present cookie via `/api/auth/session` and re-pairing on `authenticated:false` — **gated to document navigations** (`isDocumentNav`: trust `Sec-Fetch-Dest: document`, else fall back to `Accept: text/html`) so XHR/asset/WebSocket sub-requests are never answered with a `302`, and **fail-open** (proxy through) on any validation error so no new failure mode is introduced. See `scripts/t3-dispatch/main.go` (`sessionValid`, `isDocumentNav`) + `main_test.go`.
+
 Implementation: a small reverse proxy that supports WebSocket upgrade (Go `httputil.ReverseProxy`, or Python aiohttp) — chosen at plan time.
 
 ### 4. Terraform — `stacks/t3code` shrinks

scripts/t3-dispatch/main.go

@@ -59,6 +59,60 @@ func lookup(ak string) (entry, bool) {
 	return e, ok
 }
 
+// mintToken mints a one-time pairing token for osUser via the scoped sudoers
+// entry (the dispatch service can invoke nothing else). Indirected through a var
+// so tests can stub the privileged exec.
+var mintToken = func(osUser string) ([]byte, error) {
+	return exec.Command("sudo", "-n", "/usr/local/bin/t3-mint", osUser).Output()
+}
+
+var sessionClient = &http.Client{Timeout: 5 * time.Second}
+
+// sessionValid asks the user's instance whether the presented t3_session cookie
+// is still valid. Server-side sessions can be wiped/expired independently of the
+// 30-day cookie (e.g. an auth-schema rollback drops every session row), leaving
+// the browser with a live-looking but dead cookie. Fails OPEN: any error/non-200/
+// parse failure returns true so the request still proxies — a re-pair is forced
+// only on a definitive authenticated:false.
+func sessionValid(e entry, c *http.Cookie) bool {
+	req, err := http.NewRequest(http.MethodGet,
+		fmt.Sprintf("http://127.0.0.1:%d/api/auth/session", e.Port), nil)
+	if err != nil {
+		return true
+	}
+	req.AddCookie(c)
+	resp, err := sessionClient.Do(req)
+	if err != nil {
+		return true
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return true
+	}
+	var s struct {
+		Authenticated bool `json:"authenticated"`
+	}
+	if json.NewDecoder(resp.Body).Decode(&s) != nil {
+		return true
+	}
+	return s.Authenticated
+}
+
+// isDocumentNav reports whether r is a top-level browser document navigation, as
+// opposed to an XHR/fetch/asset/WebSocket sub-request. Only such requests are
+// safe to answer with a re-pair 302 — redirecting a sub-resource would corrupt
+// the SPA's fetch/WebSocket contract. Trust Sec-Fetch-Dest when present (all
+// modern browsers send it); fall back to the Accept header otherwise.
+func isDocumentNav(r *http.Request) bool {
+	if r.Method != http.MethodGet {
+		return false
+	}
+	if dest := r.Header.Get("Sec-Fetch-Dest"); dest != "" {
+		return dest == "document"
+	}
+	return strings.Contains(r.Header.Get("Accept"), "text/html")
+}
+
 // autoPair mints a one-time pairing token for the user's instance (as that OS
 // user, via the scoped sudoers entry) and exchanges it at the instance's
 // /api/auth/bootstrap, relaying the returned t3_session Set-Cookie to the browser.
@@ -66,7 +120,7 @@ func autoPair(e entry, w http.ResponseWriter, r *http.Request) {
 	// t3-mint (root, via scoped sudoers) validates the OS user is in
 	// /etc/ttyd-user-map, then mints as that user. The dispatch service itself
 	// runs unprivileged and can invoke nothing else.
-	out, err := exec.Command("sudo", "-n", "/usr/local/bin/t3-mint", e.OsUser).Output()
+	out, err := mintToken(e.OsUser)
 	if err != nil {
 		log.Printf("mint for %s failed: %v", e.OsUser, err)
 		http.Error(w, "pairing mint failed", http.StatusInternalServerError)
@@ -111,7 +165,16 @@ func handler(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "no t3 instance provisioned for this user", http.StatusForbidden)
 		return
 	}
-	if _, err := r.Cookie(cookieName); err != nil {
+	c, err := r.Cookie(cookieName)
+	if err != nil {
+		autoPair(e, w, r)
+		return
+	}
+	// A present cookie can still be server-side-invalid (sessions wiped/expired
+	// while the 30-day cookie lingers). On a top-level navigation, verify it and
+	// re-pair if dead — otherwise the instance just renders its pair page. Gated
+	// to document navs so we never 302 an XHR/asset/WebSocket sub-request.
+	if isDocumentNav(r) && !sessionValid(e, c) {
 		autoPair(e, w, r)
 		return
 	}

scripts/t3-dispatch/main_test.go

@@ -0,0 +1,200 @@
+package main
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strconv"
+	"testing"
+)
+
+func portOf(t *testing.T, ts *httptest.Server) int {
+	t.Helper()
+	u, err := url.Parse(ts.URL)
+	if err != nil {
+		t.Fatalf("parse %s: %v", ts.URL, err)
+	}
+	p, err := strconv.Atoi(u.Port())
+	if err != nil {
+		t.Fatalf("port %s: %v", u.Port(), err)
+	}
+	return p
+}
+
+func TestIsDocumentNav(t *testing.T) {
+	cases := []struct {
+		name    string
+		method  string
+		headers map[string]string
+		want    bool
+	}{
+		{"GET sec-fetch-dest document", "GET", map[string]string{"Sec-Fetch-Dest": "document"}, true},
+		{"GET accept html (no sec-fetch)", "GET", map[string]string{"Accept": "text/html,application/xhtml+xml"}, true},
+		{"GET xhr empty dest beats accept", "GET", map[string]string{"Sec-Fetch-Dest": "empty", "Accept": "text/html"}, false},
+		{"GET json", "GET", map[string]string{"Accept": "application/json"}, false},
+		{"POST html", "POST", map[string]string{"Accept": "text/html"}, false},
+		{"GET no headers", "GET", map[string]string{}, false},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			r, _ := http.NewRequest(c.method, "/", nil)
+			for k, v := range c.headers {
+				r.Header.Set(k, v)
+			}
+			if got := isDocumentNav(r); got != c.want {
+				t.Errorf("isDocumentNav = %v, want %v", got, c.want)
+			}
+		})
+	}
+}
+
+func sessionServer(status int, body string) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/api/auth/session" {
+			http.NotFound(w, r)
+			return
+		}
+		w.WriteHeader(status)
+		_, _ = w.Write([]byte(body))
+	}))
+}
+
+func TestSessionValid(t *testing.T) {
+	ck := &http.Cookie{Name: cookieName, Value: "x"}
+
+	t.Run("authenticated true -> valid", func(t *testing.T) {
+		ts := sessionServer(200, `{"authenticated":true}`)
+		defer ts.Close()
+		if !sessionValid(entry{Port: portOf(t, ts)}, ck) {
+			t.Fatal("want valid (true) for authenticated:true")
+		}
+	})
+	t.Run("authenticated false -> invalid", func(t *testing.T) {
+		ts := sessionServer(200, `{"authenticated":false}`)
+		defer ts.Close()
+		if sessionValid(entry{Port: portOf(t, ts)}, ck) {
+			t.Fatal("want invalid (false) for authenticated:false")
+		}
+	})
+	t.Run("500 -> fail-open valid", func(t *testing.T) {
+		ts := sessionServer(500, `boom`)
+		defer ts.Close()
+		if !sessionValid(entry{Port: portOf(t, ts)}, ck) {
+			t.Fatal("want fail-open true on 500")
+		}
+	})
+	t.Run("malformed json -> fail-open valid", func(t *testing.T) {
+		ts := sessionServer(200, `not json`)
+		defer ts.Close()
+		if !sessionValid(entry{Port: portOf(t, ts)}, ck) {
+			t.Fatal("want fail-open true on unparseable body")
+		}
+	})
+	t.Run("unreachable -> fail-open valid", func(t *testing.T) {
+		ts := sessionServer(200, `{"authenticated":false}`)
+		p := portOf(t, ts)
+		ts.Close() // nothing listening now
+		if !sessionValid(entry{Port: p}, ck) {
+			t.Fatal("want fail-open true on connection refused")
+		}
+	})
+}
+
+// fakeInstance serves the three endpoints the dispatcher touches: the session
+// check, the bootstrap exchange, and a catch-all standing in for the proxied app.
+func fakeInstance(authenticated bool, bootstrapCalled *bool) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/api/auth/session":
+			if authenticated {
+				_, _ = w.Write([]byte(`{"authenticated":true}`))
+			} else {
+				_, _ = w.Write([]byte(`{"authenticated":false}`))
+			}
+		case "/api/auth/bootstrap":
+			if bootstrapCalled != nil {
+				*bootstrapCalled = true
+			}
+			http.SetCookie(w, &http.Cookie{Name: cookieName, Value: "fresh", Path: "/"})
+			_, _ = w.Write([]byte(`{"authenticated":true}`))
+		default:
+			_, _ = w.Write([]byte("APP"))
+		}
+	}))
+}
+
+func setTable(port int) {
+	mu.Lock()
+	table = map[string]entry{"vbarzin": {OsUser: "wizard", Port: port}}
+	mu.Unlock()
+}
+
+func TestHandlerRepairsOnInvalidCookieDocNav(t *testing.T) {
+	called := false
+	ts := fakeInstance(false, &called)
+	defer ts.Close()
+	setTable(portOf(t, ts))
+
+	orig := mintToken
+	mintToken = func(string) ([]byte, error) { return []byte(`{"credential":"tok"}`), nil }
+	defer func() { mintToken = orig }()
+
+	r := httptest.NewRequest("GET", "/", nil)
+	r.Header.Set("X-authentik-username", "vbarzin@gmail.com")
+	r.Header.Set("Sec-Fetch-Dest", "document")
+	r.AddCookie(&http.Cookie{Name: cookieName, Value: "stale"})
+	w := httptest.NewRecorder()
+
+	handler(w, r)
+
+	if w.Code != http.StatusFound {
+		t.Fatalf("stale cookie on doc-nav should re-pair (302), got %d body=%q", w.Code, w.Body.String())
+	}
+	if !called {
+		t.Fatal("expected bootstrap to be called during re-pair")
+	}
+	cookies := w.Result().Cookies()
+	if len(cookies) == 0 || cookies[0].Value != "fresh" {
+		t.Fatalf("expected fresh t3_session relayed, got %+v", cookies)
+	}
+}
+
+func TestHandlerProxiesOnValidCookie(t *testing.T) {
+	ts := fakeInstance(true, nil)
+	defer ts.Close()
+	setTable(portOf(t, ts))
+
+	r := httptest.NewRequest("GET", "/", nil)
+	r.Header.Set("X-authentik-username", "vbarzin@gmail.com")
+	r.Header.Set("Sec-Fetch-Dest", "document")
+	r.AddCookie(&http.Cookie{Name: cookieName, Value: "good"})
+	w := httptest.NewRecorder()
+
+	handler(w, r)
+
+	if w.Code != http.StatusOK || w.Body.String() != "APP" {
+		t.Fatalf("valid cookie should proxy (200 APP), got %d %q", w.Code, w.Body.String())
+	}
+}
+
+func TestHandlerProxiesXHREvenIfCookieInvalid(t *testing.T) {
+	called := false
+	ts := fakeInstance(false, &called) // session would say invalid, but XHR must NOT be re-paired
+	defer ts.Close()
+	setTable(portOf(t, ts))
+
+	r := httptest.NewRequest("GET", "/api/threads", nil)
+	r.Header.Set("X-authentik-username", "vbarzin@gmail.com")
+	r.Header.Set("Sec-Fetch-Dest", "empty") // XHR/fetch, not a document nav
+	r.AddCookie(&http.Cookie{Name: cookieName, Value: "stale"})
+	w := httptest.NewRecorder()
+
+	handler(w, r)
+
+	if called {
+		t.Fatal("must NOT re-pair (302) a non-document sub-request — would corrupt the SPA fetch contract")
+	}
+	if w.Code != http.StatusOK || w.Body.String() != "APP" {
+		t.Fatalf("XHR should proxy through, got %d %q", w.Code, w.Body.String())
+	}
+}