breakglass: in-cluster emergency-recovery UI for the devvm

Viktor wanted a web UI on the claude service to act as his breakglass when
the devvm is down: open it, have Claude SSH in to diagnose/repair, and
power-cycle the VM via the Proxmox host if needed. This is the app half
(the infra stack + host bootstrap live in the infra repo).

New, ISOLATED ASGI app under app/breakglass/ (never imports app.main, so the
untrusted-input agents — recruiter-triage, nextcloud-todos — can't share a
process with the root-on-devvm / PVE-reset SSH key):
- pve.py: the LLM-independent power-verb path (status|forensics|reset|stop|
  start|cycle on VM 102), whitelist-validated client-side, executed over the
  forced-command SSH key (list argv, no shell).
- agent_session.py: multi-turn streamed chat — claude -p --session-id /
  --resume with --output-format stream-json, translated to a small SSE
  vocabulary (session/text/tool/result/error/done).
- auth.py: edge Authentik header OR bearer; fail-closed.
- server.py: FastAPI (session/chat-SSE/pve-verb routes) + serves the Svelte UI.
- Svelte SPA (frontend/, built into app/breakglass/static/ and committed — no
  in-cluster build, per ADR-0002): streamed chat + danger-styled manual VM
  controls with confirm-on-mutate.
- agents/breakglass.md: narrow tools (Bash/Read/Grep/Glob, no web), taught the
  ssh devvm / ssh pve aliases and cycle-vs-reset.
- docker-entrypoint-breakglass.sh: ssh-agent bootstrap from the mounted key +
  ssh aliases, then uvicorn app.breakglass.server. The breakglass Deployment
  overrides the image CMD with this; the existing service is untouched.

26 new tests (verb whitelist incl. injection attempts, stream-json→SSE
translation, auth gating, route behaviour); full suite 58 green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

frontend/src/lib/api.js

@@ -0,0 +1,92 @@
+// Same-origin API client. Auth is handled entirely by the edge proxy
+// (Authentik / basic-auth / bearer) — this UI never sends or stores a token.
+import { readEventStream } from './sse.js';
+
+/** Open a fresh chat session. @returns {Promise<string>} session_id */
+export async function openSession() {
+  const res = await fetch('/api/session', {
+    method: 'POST',
+    headers: { 'content-type': 'application/json' },
+  });
+  if (!res.ok) {
+    throw new Error(`could not open a session (HTTP ${res.status})`);
+  }
+  const body = await res.json();
+  if (!body || typeof body.session_id !== 'string') {
+    throw new Error('session response missing session_id');
+  }
+  return body.session_id;
+}
+
+/**
+ * Run one chat turn. Streams events to onEvent until the backend sends
+ * {kind:"done"} and the connection closes. Pass an AbortSignal to cancel.
+ *
+ * @param {{session_id: string, prompt: string, model?: string, signal?: AbortSignal}} opts
+ * @param {(event: object) => void} onEvent
+ */
+export async function streamChat({ session_id, prompt, model, signal }, onEvent) {
+  const payload = { session_id, prompt };
+  if (model) payload.model = model;
+
+  const res = await fetch('/api/chat', {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json',
+      accept: 'text/event-stream',
+    },
+    body: JSON.stringify(payload),
+    signal,
+  });
+  await readEventStream(res, onEvent);
+}
+
+/**
+ * List the PVE power verbs and which of them mutate VM state.
+ * @returns {Promise<{verbs: string[], mutating: string[]}>}
+ */
+export async function fetchVerbs() {
+  const res = await fetch('/api/pve/verbs');
+  if (!res.ok) {
+    throw new Error(`could not load VM controls (HTTP ${res.status})`);
+  }
+  const body = await res.json();
+  return {
+    verbs: Array.isArray(body.verbs) ? body.verbs : [],
+    mutating: Array.isArray(body.mutating) ? body.mutating : [],
+  };
+}
+
+/**
+ * Run a PVE power verb directly (no AI in the path). The backend returns 200
+ * on success and 502 when the verb's exit code is non-zero, but the JSON body
+ * carries {verb, exit_code, stdout, stderr, rejected} in BOTH cases — so we
+ * read the body regardless of HTTP status and let the caller style on
+ * exit_code / rejected.
+ *
+ * @param {string} verb
+ * @returns {Promise<{verb: string, exit_code: number|null, stdout: string, stderr: string, rejected: boolean}>}
+ */
+export async function runVerb(verb) {
+  const res = await fetch(`/api/pve/${encodeURIComponent(verb)}`, {
+    method: 'POST',
+    headers: { 'content-type': 'application/json' },
+  });
+  // 400 = unknown verb (FastAPI HTTPException) — has {detail}, not the verb shape.
+  let body;
+  try {
+    body = await res.json();
+  } catch {
+    throw new Error(`VM control '${verb}' failed (HTTP ${res.status}, no body)`);
+  }
+  if (res.status === 400) {
+    throw new Error(body?.detail || `'${verb}' was rejected by the server`);
+  }
+  return {
+    verb: body.verb ?? verb,
+    exit_code: body.exit_code ?? null,
+    stdout: body.stdout ?? '',
+    stderr: body.stderr ?? '',
+    rejected: Boolean(body.rejected),
+  };
+}

frontend/src/lib/sse.js

@@ -0,0 +1,150 @@
+// SSE frame parsing — the load-bearing core of the breakglass UI.
+//
+// The /api/chat endpoint returns a text/event-stream that we read with
+// fetch() + response.body.getReader() (NOT EventSource, which cannot POST).
+// The backend emits one frame per event as:
+//
+//     data: {json}\n\n
+//
+// getReader() hands us bytes at arbitrary boundaries: a single frame can be
+// split across reads, and one read can contain several frames. So we keep a
+// rolling text buffer, split it on the blank-line frame delimiter, and only
+// hand back the JSON payload of *complete* frames. Per the SSE spec a frame may
+// carry multiple `data:` lines (joined with "\n"); the backend emits single
+// line JSON today, but we handle the general case so a future multi-line
+// payload can't silently corrupt the stream.
+
+/**
+ * Parse a single SSE event block (the text between blank lines) into its data
+ * payload string, or null if the block carries no `data:` field (e.g. a bare
+ * comment or a `:` heartbeat).
+ * @param {string} block
+ * @returns {string|null}
+ */
+export function dataFromEventBlock(block) {
+  const dataLines = [];
+  for (const rawLine of block.split('\n')) {
+    const line = rawLine.replace(/\r$/, '');
+    if (line.startsWith(':')) continue; // SSE comment / heartbeat
+    if (line === 'data:' || line === 'data') {
+      dataLines.push('');
+    } else if (line.startsWith('data:')) {
+      // Spec: a single leading space after the colon is stripped.
+      let v = line.slice('data:'.length);
+      if (v.startsWith(' ')) v = v.slice(1);
+      dataLines.push(v);
+    }
+    // field lines we don't care about (event:, id:, retry:) are ignored
+  }
+  if (dataLines.length === 0) return null;
+  return dataLines.join('\n');
+}
+
+/**
+ * A stateful splitter that turns an arbitrary sequence of decoded text chunks
+ * into a sequence of complete SSE event-block strings. Frames are delimited by
+ * a blank line; we tolerate both "\n\n" and "\r\n\r\n".
+ */
+export class SSEFrameSplitter {
+  constructor() {
+    this.buffer = '';
+  }
+
+  /**
+   * Feed a decoded text chunk; returns the event blocks that are now complete.
+   * Any trailing partial frame stays buffered for the next chunk.
+   * @param {string} chunk
+   * @returns {string[]} complete event blocks (text between delimiters)
+   */
+  push(chunk) {
+    this.buffer += chunk;
+    const blocks = [];
+    // Normalise CRLF delimiters to LF so a single split rule covers both.
+    let idx;
+    // Process every complete frame currently in the buffer.
+    while ((idx = this._nextDelimiter()) !== -1) {
+      const block = this.buffer.slice(0, idx.start);
+      this.buffer = this.buffer.slice(idx.end);
+      if (block.length > 0) blocks.push(block);
+    }
+    return blocks;
+  }
+
+  /**
+   * On stream end, return whatever complete-looking content remains. A
+   * well-behaved backend always terminates the last frame with a blank line,
+   * so this is usually empty — but if the connection closed mid-trailing-frame
+   * with a parseable block, surface it rather than dropping data.
+   * @returns {string[]}
+   */
+  flush() {
+    const rest = this.buffer.trim();
+    this.buffer = '';
+    return rest ? [rest] : [];
+  }
+
+  _nextDelimiter() {
+    // Find the earliest of "\n\n", "\r\n\r\n", "\r\r".
+    const candidates = [
+      { token: '\r\n\r\n', i: this.buffer.indexOf('\r\n\r\n') },
+      { token: '\n\n', i: this.buffer.indexOf('\n\n') },
+      { token: '\r\r', i: this.buffer.indexOf('\r\r') },
+    ].filter((c) => c.i !== -1);
+    if (candidates.length === 0) return -1;
+    candidates.sort((a, b) => a.i - b.i);
+    const { token, i } = candidates[0];
+    return { start: i, end: i + token.length };
+  }
+}
+
+/**
+ * Read an SSE Response body to completion, invoking onEvent for every parsed
+ * JSON event object. Resolves when the stream ends. Throws if the response is
+ * not ok or has no readable body (caller shows the error inline).
+ *
+ * @param {Response} response  a fetch() Response with a streaming body
+ * @param {(event: object) => void} onEvent  called per parsed JSON event
+ */
+export async function readEventStream(response, onEvent) {
+  if (!response.ok) {
+    throw new Error(`server returned ${response.status} ${response.statusText}`);
+  }
+  if (!response.body) {
+    throw new Error('response has no readable body (streaming unsupported)');
+  }
+
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  const splitter = new SSEFrameSplitter();
+
+  const handleBlock = (block) => {
+    const payload = dataFromEventBlock(block);
+    if (payload == null || payload.trim() === '') return;
+    let obj;
+    try {
+      obj = JSON.parse(payload);
+    } catch {
+      // A malformed frame must not abort an in-progress recovery stream;
+      // skip it and keep reading.
+      return;
+    }
+    onEvent(obj);
+  };
+
+  try {
+    for (;;) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      const text = decoder.decode(value, { stream: true });
+      for (const block of splitter.push(text)) handleBlock(block);
+    }
+  } finally {
+    reader.releaseLock?.();
+  }
+  // Drain any trailing bytes the decoder held, then any final frame.
+  const tail = decoder.decode();
+  if (tail) {
+    for (const block of splitter.push(tail)) handleBlock(block);
+  }
+  for (const block of splitter.flush()) handleBlock(block);
+}

frontend/src/lib/sse.test.mjs

@@ -0,0 +1,152 @@
+// Standalone test of the SSE frame parser — no test framework, just node.
+// Run: node src/lib/sse.test.mjs   (exits non-zero on any failure)
+//
+// These pin the protocol described in the API contract: frames are
+// `data: {json}\n\n`, the event `kind` is one of session/text/tool/result/
+// error/done, and bytes arrive at arbitrary boundaries via getReader().
+import { SSEFrameSplitter, dataFromEventBlock, readEventStream } from './sse.js';
+
+let failures = 0;
+function ok(name, cond) {
+  if (cond) {
+    console.log(`  ok  ${name}`);
+  } else {
+    failures++;
+    console.error(`FAIL  ${name}`);
+  }
+}
+function eq(name, got, want) {
+  const g = JSON.stringify(got);
+  const w = JSON.stringify(want);
+  ok(`${name}  (got ${g})`, g === w);
+}
+
+// --- dataFromEventBlock ---------------------------------------------------
+eq(
+  'extracts JSON payload from a data: line',
+  dataFromEventBlock('data: {"kind":"text","text":"hi"}'),
+  '{"kind":"text","text":"hi"}'
+);
+eq(
+  'strips exactly one space after the colon',
+  dataFromEventBlock('data:  leading-space-kept'),
+  ' leading-space-kept'
+);
+eq('ignores comment/heartbeat lines', dataFromEventBlock(': keep-alive'), null);
+eq(
+  'joins multi-line data fields with newline',
+  dataFromEventBlock('data: line1\ndata: line2'),
+  'line1\nline2'
+);
+
+// --- SSEFrameSplitter: whole frames --------------------------------------
+{
+  const s = new SSEFrameSplitter();
+  const blocks = s.push('data: {"kind":"session","session_id":"abc"}\n\n');
+  eq('one complete frame yields one block', blocks, [
+    'data: {"kind":"session","session_id":"abc"}',
+  ]);
+}
+
+// --- SSEFrameSplitter: multiple frames in one chunk ----------------------
+{
+  const s = new SSEFrameSplitter();
+  const blocks = s.push(
+    'data: {"kind":"text","text":"a"}\n\ndata: {"kind":"text","text":"b"}\n\n'
+  );
+  eq('two frames in one chunk yield two blocks', blocks.length, 2);
+  eq('first block', dataFromEventBlock(blocks[0]), '{"kind":"text","text":"a"}');
+  eq('second block', dataFromEventBlock(blocks[1]), '{"kind":"text","text":"b"}');
+}
+
+// --- SSEFrameSplitter: frame split across chunks -------------------------
+{
+  const s = new SSEFrameSplitter();
+  let blocks = s.push('data: {"kind":"te');
+  eq('partial frame yields nothing yet', blocks, []);
+  blocks = s.push('xt","text":"split"}\n\n');
+  eq('completing the frame yields it whole', dataFromEventBlock(blocks[0]), '{"kind":"text","text":"split"}');
+}
+
+// --- SSEFrameSplitter: delimiter split across chunks ---------------------
+{
+  const s = new SSEFrameSplitter();
+  let blocks = s.push('data: {"kind":"done"}\n');
+  eq('frame held while delimiter incomplete', blocks, []);
+  blocks = s.push('\n');
+  eq('frame released once blank line completes', dataFromEventBlock(blocks[0]), '{"kind":"done"}');
+}
+
+// --- SSEFrameSplitter: CRLF delimiters -----------------------------------
+{
+  const s = new SSEFrameSplitter();
+  const blocks = s.push('data: {"kind":"text","text":"crlf"}\r\n\r\n');
+  eq('CRLF-delimited frame parses', dataFromEventBlock(blocks[0]), '{"kind":"text","text":"crlf"}');
+}
+
+// --- end-to-end via readEventStream over a mock streaming Response --------
+function mockResponse(chunks) {
+  const enc = new TextEncoder();
+  let i = 0;
+  return {
+    ok: true,
+    status: 200,
+    body: {
+      getReader() {
+        return {
+          read() {
+            if (i < chunks.length) {
+              return Promise.resolve({ value: enc.encode(chunks[i++]), done: false });
+            }
+            return Promise.resolve({ value: undefined, done: true });
+          },
+          releaseLock() {},
+        };
+      },
+    },
+  };
+}
+
+await (async () => {
+  // A realistic turn, deliberately chopped at ugly boundaries:
+  //  - the session frame split mid-JSON
+  //  - two text frames glued together
+  //  - a tool frame
+  //  - a result frame and the terminal done frame in one chunk
+  const chunks = [
+    'data: {"kind":"sess',
+    'ion","session_id":"S1"}\n\n',
+    'data: {"kind":"text","text":"checking "}\n\ndata: {"kind":"text","text":"disk"}\n\n',
+    'data: {"kind":"tool","name":"Bash","input":{"command":"df -h"}}\n\n',
+    'data: {"kind":"result","is_error":false,"result":"ok","duration_ms":12}\n\ndata: {"kind":"done"}\n\n',
+  ];
+  const events = [];
+  await readEventStream(mockResponse(chunks), (e) => events.push(e));
+
+  eq('event count', events.length, 6);
+  eq('1: session id', events[0], { kind: 'session', session_id: 'S1' });
+  eq('2: first text', events[1], { kind: 'text', text: 'checking ' });
+  eq('3: second text', events[2], { kind: 'text', text: 'disk' });
+  eq('4: tool kind+name', { kind: events[3].kind, name: events[3].name }, { kind: 'tool', name: 'Bash' });
+  eq('4: tool command', events[3].input.command, 'df -h');
+  eq('5: result', events[4], { kind: 'result', is_error: false, result: 'ok', duration_ms: 12 });
+  eq('6: done terminal', events[5], { kind: 'done' });
+})();
+
+// malformed frame in the middle must be skipped, not abort the stream
+await (async () => {
+  const chunks = [
+    'data: {"kind":"text","text":"before"}\n\n',
+    'data: {this is not json}\n\n',
+    'data: {"kind":"done"}\n\n',
+  ];
+  const events = [];
+  await readEventStream(mockResponse(chunks), (e) => events.push(e));
+  eq('malformed frame skipped, stream continues', events.map((e) => e.kind), ['text', 'done']);
+})();
+
+if (failures) {
+  console.error(`\n${failures} assertion(s) FAILED`);
+  process.exit(1);
+}
+console.log('\nall SSE parser assertions passed');