Merge pull request 'workstation: homelab-memory for all users (retire claude-memory MCP)' (#13) from wizard/memory-allusers into master

scripts/t3-provision-users.sh

@@ -381,6 +381,50 @@ install_playwright() {
   run systemctl enable --now "playwright-snapshot-refresh@$user.timer" >/dev/null 2>&1 || true
 }
 
+# Per-user homelab-memory setup — migrate off the claude-memory MCP/plugin to the
+# homelab CLI hooks (auto-recall + auto-learn + compaction backup/recovery).
+# Idempotent, if-absent, ADDITIVE: never clobbers `env` (the per-user
+# MEMORY_API_KEY) or other MCP servers; removes ONLY the `claude_memory` MCP.
+# Reuses the user's existing key — does NOT mint one (per-user isolation stays
+# deferred, design 2026-06-08). The homelab CLI (/usr/local/bin/homelab) hits the
+# same remote HTTP API the MCP used. Hook scripts: $WORKSTATION_DIR/claude-hooks.
+install_memory() {
+  local user="$1" home
+  home="$(getent passwd "$user" | cut -d: -f6)"
+  [[ -n "$home" && -d "$home" ]] || return 0
+  local src="$WORKSTATION_DIR/claude-hooks" hooks_dst="$home/.claude/hooks" settings="$home/.claude/settings.json"
+  [[ -d "$src" ]] || { log "WARN: $src missing -> skip memory setup for $user"; return 0; }
+
+  if [[ "$DRY_RUN" == 1 ]]; then echo "[dry-run] memory: hooks + settings wire + claude_memory MCP removal -> $user"; return 0; fi
+
+  # (1) (re)install the 4 hook scripts, owned by the user (refreshed each reconcile so fixes land)
+  install -d -o "$user" -g "$user" -m 0755 "$hooks_dst"
+  local h
+  for h in homelab-memory-recall.py auto-learn.py pre-compact-backup.sh post-compact-recovery.sh; do
+    install -o "$user" -g "$user" -m 0755 "$src/$h" "$hooks_dst/$h"
+  done
+
+  # (2) wire the hooks in settings.json (AS the user -> correct ownership), if-absent + additive;
+  #     enforce 0600 (it holds the per-user MEMORY_API_KEY).
+  if runuser -u "$user" -- python3 "$src/wire-memory-hooks.py" "$home" >/dev/null 2>&1; then
+    log "memory hooks wired -> $user"
+  else
+    log "WARN: memory hook wiring failed for $user (retries next reconcile)"
+  fi
+  [[ -f "$settings" ]] && chmod 600 "$settings"
+
+  # (2b) reuse the user's existing key; warn (do NOT mint — needs an admin vault write) if absent.
+  if [[ -f "$settings" ]] && ! grep -q 'MEMORY_API_KEY' "$settings"; then
+    log "WARN: $user has no MEMORY_API_KEY in settings.json — homelab memory no-ops until an admin mints one"
+  fi
+
+  # (3) remove the now-superseded claude_memory MCP (AS the user, if-present) + the plugin dir.
+  if runuser -u "$user" -- bash -lc 'command -v claude >/dev/null 2>&1 && claude mcp get claude_memory >/dev/null 2>&1'; then
+    runuser -u "$user" -- bash -lc 'claude mcp remove claude_memory >/dev/null 2>&1' && log "removed claude_memory MCP -> $user" || true
+  fi
+  [[ -d "$home/.claude/plugins/claude-memory" ]] && rm -rf "$home/.claude/plugins/claude-memory" && log "removed claude-memory plugin dir -> $user"
+}
+
 [[ $EUID -eq 0 ]] || { echo "t3-provision-users: must run as root" >&2; exit 1; }
 for bin in python3 jq; do command -v "$bin" >/dev/null || { echo "missing $bin" >&2; exit 1; }; done
 [[ -f "$ROSTER" && -f "$ENGINE" ]] || { echo "roster/engine not under $WORKSTATION_DIR" >&2; exit 1; }
@@ -494,6 +538,14 @@ while IFS=$'\t' read -r os_user pw_port; do
   install_playwright "$os_user"
 done < <(jq -r '.playwright_ports | to_entries[] | [.key, .value] | @tsv' "$desired_file")
 
+# 5d) per-user homelab-memory (ALL users): replace the claude-memory MCP/plugin with the
+#     homelab CLI memory hooks. Idempotent + additive + if-absent; never touches the
+#     per-user MEMORY_API_KEY or other MCP servers (removes ONLY claude_memory).
+while IFS=$'\t' read -r os_user; do
+  id "$os_user" >/dev/null 2>&1 || continue
+  install_memory "$os_user"
+done < <(jq -r '.accounts[].os_user' "$desired_file")
+
 # 5b) machine-wide (once, not per-user): keep the t3 gated nightly TRACKER timer enabled (it
 #     follows t3@nightly daily, gated; see t3-autoupdate.sh / docs/runbooks/t3-version-bump.md).
 #     NEVER --now: the tracker installs a NEW build + migrates DBs + restarts serves, so firing

scripts/workstation/claude-hooks/auto-learn.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""
+Stop hook (async): automatic learning extraction via haiku-as-judge.
+
+After each Claude response, sends the user message + assistant response to
+haiku to detect corrections, preferences, decisions, or facts worth storing.
+If learning events are detected, stores them via the memory API (or SQLite fallback).
+
+Runs with async: true — does NOT block the user.
+"""
+
+import io
+import json
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import urllib.error
+import urllib.request
+
+logger = logging.getLogger(__name__)
+
+API_BASE_URL = os.environ.get("MEMORY_API_URL") or os.environ.get("CLAUDE_MEMORY_API_URL", "")
+API_KEY = os.environ.get("MEMORY_API_KEY") or os.environ.get("CLAUDE_MEMORY_API_KEY", "")
+
+JUDGE_PROMPT = """You are a memory extraction judge. Analyze this exchange between a user and an AI assistant.
+
+USER MESSAGE:
+{user_message}
+
+ASSISTANT RESPONSE:
+{assistant_response}
+
+Your job: determine if any of these learning events occurred:
+1. USER CORRECTION — user corrected the assistant's mistake or misunderstanding
+2. PREFERENCE — user stated a preference, habit, or "I like/prefer/want" statement
+3. DECISION — a decision was reached about how to do something
+4. FACT — user shared a durable fact about themselves, their team, tools, or environment
+
+If ANY learning event occurred, return JSON:
+{{"events": [{{"type": "correction|preference|decision|fact", "content": "concise fact to remember (one sentence)", "importance": 0.7, "expanded_keywords": "space-separated semantically related search terms for recall (minimum 5 words)", "supersedes": null}}]}}
+
+If NO learning event occurred, return:
+{{"events": []}}
+
+Rules:
+- Only extract DURABLE facts, not transient task details
+- Corrections are highest value (0.8-0.9)
+- Be conservative — false negatives are better than false positives
+- "expanded_keywords" should include synonyms, related concepts, and adjacent topics that would help find this memory later
+- "supersedes" should be a search query to find the old outdated memory, or null
+- Return ONLY valid JSON, no other text"""
+
+
+def _api_request(method: str, path: str, body: dict | None = None) -> dict:
+    url = f"{API_BASE_URL}{path}"
+    data = json.dumps(body).encode() if body else None
+    req = urllib.request.Request(
+        url, data=data, method=method,
+        headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"},
+    )
+    with urllib.request.urlopen(req, timeout=15) as resp:
+        return json.loads(resp.read().decode())
+
+
+def _store_via_api(content, category, tags, importance, expanded_keywords):
+    _api_request("POST", "/api/memories", {
+        "content": content, "category": category, "tags": tags,
+        "expanded_keywords": expanded_keywords, "importance": importance,
+    })
+
+
+def _store_via_sqlite(content, category, tags, importance, expanded_keywords):
+    import sqlite3
+    from datetime import datetime, timezone
+
+    memory_home = os.environ.get("MEMORY_HOME", os.path.expanduser("~/.claude/claude-memory"))
+    db_path = os.path.join(memory_home, "memory", "memory.db")
+
+    # Also check legacy path
+    if not os.path.exists(db_path):
+        legacy_db = os.path.join(os.path.expanduser("~/.claude/metaclaw"), "memory", "memory.db")
+        if os.path.exists(legacy_db):
+            db_path = legacy_db
+
+    conn = sqlite3.connect(db_path, timeout=10.0)
+    conn.execute("PRAGMA journal_mode=WAL")
+    now = datetime.now(timezone.utc).isoformat()
+    conn.execute(
+        "INSERT INTO memories (content, category, tags, importance, expanded_keywords, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
+        (content, category, tags, importance, expanded_keywords, now, now),
+    )
+    conn.commit()
+    conn.close()
+
+
+def main() -> None:
+    # Graceful exit if claude CLI is not available
+    if not shutil.which("claude"):
+        return
+
+    try:
+        hook_input = json.load(sys.stdin)
+    except (json.JSONDecodeError, EOFError):
+        return
+
+    if isinstance(hook_input, dict) and hook_input.get("stop_hook_active", False):
+        return
+
+    transcript_path = ""
+    if isinstance(hook_input, dict):
+        transcript_path = hook_input.get("transcript_path", "")
+
+    if not transcript_path or not os.path.exists(transcript_path):
+        return
+
+    user_message = ""
+    assistant_response = ""
+    try:
+        MAX_TAIL_BYTES = 50_000
+        with open(transcript_path, "rb") as f:
+            f.seek(0, io.SEEK_END)
+            size = f.tell()
+            f.seek(max(0, size - MAX_TAIL_BYTES))
+            tail = f.read().decode("utf-8", errors="replace")
+        lines = tail.split("\n")
+
+        for line in reversed(lines):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            role = entry.get("role", "")
+            content = entry.get("content", "")
+            if isinstance(content, list):
+                content = " ".join(
+                    b.get("text", "") for b in content
+                    if isinstance(b, dict) and b.get("type") == "text"
+                )
+            content = str(content)[:2000]
+            if role == "assistant" and not assistant_response:
+                assistant_response = content
+            elif role == "user" and not user_message:
+                user_message = content
+            if user_message and assistant_response:
+                break
+    except Exception:
+        return
+
+    if not user_message or len(user_message.strip()) < 10:
+        return
+
+    prompt = JUDGE_PROMPT.format(
+        user_message=user_message,
+        assistant_response=assistant_response[:1000],
+    )
+
+    try:
+        result = subprocess.run(
+            ["claude", "-p", prompt, "--model", "haiku"],
+            capture_output=True, text=True, timeout=30,
+            env={**os.environ, "CLAUDECODE": ""},
+        )
+        if result.returncode != 0:
+            return
+        response_text = result.stdout.strip()
+        if response_text.startswith("```"):
+            lines = response_text.split("\n")
+            lines = [l for l in lines if not l.strip().startswith("```")]
+            response_text = "\n".join(lines).strip()
+        judge_result = json.loads(response_text)
+        events = judge_result.get("events", [])
+        if not events:
+            return
+    except (subprocess.TimeoutExpired, json.JSONDecodeError, OSError):
+        return
+
+    category_map = {
+        "correction": "preferences",
+        "preference": "preferences",
+        "decision": "decisions",
+        "fact": "facts",
+    }
+
+    for event in events:
+        content = event.get("content", "")
+        if not content:
+            continue
+        event_type = event.get("type", "fact")
+        importance = max(0.0, min(1.0, float(event.get("importance", 0.7))))
+        category = category_map.get(event_type, "facts")
+        tags = f"auto-learned,{event_type}"
+        expanded_keywords = event.get("expanded_keywords", "")
+
+        try:
+            if API_KEY and API_BASE_URL:
+                _store_via_api(content, category, tags, importance, expanded_keywords)
+            else:
+                _store_via_sqlite(content, category, tags, importance, expanded_keywords)
+        except Exception:
+            pass  # Never crash the async hook
+
+
+if __name__ == "__main__":
+    main()

scripts/workstation/claude-hooks/homelab-memory-recall.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""UserPromptSubmit hook: inject relevant memories via `homelab memory recall`.
+
+Replaces the claude-memory MCP recall path. Instead of instructing the model to
+call the memory_recall MCP tool, this hook runs the homelab CLI (a direct client
+to the same claude-memory HTTP API) and injects the ACTUAL results as context —
+so recall is automatic, needs no model tool-call, and works with the MCP
+uninstalled. Best-effort: any failure exits 0 silently (recall just doesn't
+happen that turn, exactly like the MCP being unavailable).
+
+Wizard-only trial of the MCP deprecation (2026-06-20). Reversible: restore the
+plugin command in ~/.claude/settings.json (backup: settings.json.bak-pre-homelab-memory).
+"""
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+
+
+def main() -> None:
+    try:
+        hook_input = json.load(sys.stdin)
+    except (json.JSONDecodeError, EOFError):
+        return
+
+    prompt = ""
+    if isinstance(hook_input, dict):
+        prompt = hook_input.get("prompt") or hook_input.get("user_prompt") or ""
+        if not prompt and isinstance(hook_input.get("content"), str):
+            prompt = hook_input["content"]
+    prompt = (prompt or "").strip()
+
+    # Same gates as the original recall hook: skip short prompts, code/JSON/XML blobs.
+    if len(prompt) < 10 or prompt[0] in "`{<":
+        return
+
+    homelab = shutil.which("homelab") or "/usr/local/bin/homelab"
+    if not os.path.exists(homelab):
+        return
+    if not (os.environ.get("CLAUDE_MEMORY_API_KEY") or os.environ.get("MEMORY_API_KEY")):
+        return
+
+    try:
+        res = subprocess.run(
+            [homelab, "memory", "recall", prompt, "--limit", "5"],
+            capture_output=True, text=True, timeout=4, env=os.environ,
+        )
+    except (subprocess.TimeoutExpired, OSError):
+        return
+
+    out = (res.stdout or "").strip()
+    if res.returncode != 0 or not out:
+        return
+
+    context = (
+        "Relevant stored memories (via `homelab memory recall`) — incorporate "
+        "naturally if useful; do NOT mention this lookup to the user:\n\n" + out
+    )
+    print(json.dumps({
+        "hookSpecificOutput": {
+            "hookEventName": "UserPromptSubmit",
+            "additionalContext": context,
+        }
+    }))
+
+
+if __name__ == "__main__":
+    main()

scripts/workstation/claude-hooks/post-compact-recovery.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+# UserPromptSubmit hook: Inject recovery context after compaction
+# This hook runs on each user prompt, but only injects context once after compaction.
+
+# Read hook input from stdin
+INPUT=$(cat)
+
+# Extract session ID
+SESSION_ID=$(echo "$INPUT" | jq -r '.session_id // .sessionId // "unknown"')
+
+# Define marker path
+MEMORY_HOME="${MEMORY_HOME:-$HOME/.claude/claude-memory}"
+MARKER_DIR="${MEMORY_HOME}/state/compaction-markers"
+MARKER_FILE="${MARKER_DIR}/${SESSION_ID}.json"
+
+# Fast path: no marker means no recent compaction, exit immediately
+if [ ! -f "$MARKER_FILE" ]; then
+    exit 0
+fi
+
+# Read marker contents
+MARKER=$(cat "$MARKER_FILE")
+
+# Validate JSON before processing
+if ! echo "$MARKER" | jq -e . >/dev/null 2>&1; then
+    rm -f "$MARKER_FILE"
+    exit 0
+fi
+
+# Extract data from marker
+COMPACTED_AT=$(echo "$MARKER" | jq -r '.compactedAt // "unknown"')
+PERSONALITY=$(echo "$MARKER" | jq -r '.personalityReminder // ""')
+
+# Build remembered facts summary (limit to ~500 chars)
+FACTS_SUMMARY=$(echo "$MARKER" | jq -r '
+    .rememberedFacts[:10] |
+    map("- [\(.category // "fact")] \(.content)") |
+    join("\n")
+' 2>/dev/null || echo "")
+
+# Build recovery context (kept under 1000 tokens)
+RECOVERY_CONTEXT="[Claude Memory Recovery - Context compacted at ${COMPACTED_AT}]
+
+${PERSONALITY}
+
+Key memories from before compaction:
+${FACTS_SUMMARY}
+
+Use the memory_recall MCP tool if you need more context about past conversations."
+
+# Output JSON with additional context for injection
+cat << EOF
+{
+  "hookSpecificOutput": {
+    "hookEventName": "UserPromptSubmit",
+    "additionalContext": $(echo "$RECOVERY_CONTEXT" | jq -Rs .)
+  }
+}
+EOF
+
+# Delete marker file (one-time injection)
+rm -f "$MARKER_FILE"
+
+exit 0

scripts/workstation/claude-hooks/pre-compact-backup.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# PreCompact hook: Save key memories before compaction
+set -e
+
+INPUT=$(cat)
+SESSION_ID=$(echo "$INPUT" | jq -r '.session_id // .sessionId // "unknown"')
+
+MEMORY_HOME="${MEMORY_HOME:-$HOME/.claude/claude-memory}"
+MARKER_DIR="${MEMORY_HOME}/state/compaction-markers"
+MEMORY_DB="${MEMORY_HOME}/memory/memory.db"
+MARKER_FILE="${MARKER_DIR}/${SESSION_ID}.json"
+
+mkdir -p "$MARKER_DIR"
+
+TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+
+# Try API first, fall back to SQLite
+REMEMBERED_FACTS="[]"
+if [ -n "${MEMORY_API_KEY:-${CLAUDE_MEMORY_API_KEY:-}}" ]; then
+    API_KEY="${MEMORY_API_KEY:-${CLAUDE_MEMORY_API_KEY:-}}"
+    API_URL="${MEMORY_API_URL:-${CLAUDE_MEMORY_API_URL:-}}"
+    if [ -n "$API_URL" ]; then
+        REMEMBERED_FACTS=$(curl -sf -H "Authorization: Bearer ${API_KEY}" \
+            "${API_URL}/api/memories?limit=20" 2>/dev/null | \
+            jq '[.memories[] | {content, category, importance}]' 2>/dev/null || echo "[]")
+    fi
+elif [ -f "$MEMORY_DB" ]; then
+    REMEMBERED_FACTS=$(sqlite3 -json "$MEMORY_DB" \
+        "SELECT content, category, importance FROM memories ORDER BY importance DESC, created_at DESC LIMIT 20" 2>/dev/null || echo "[]")
+fi
+
+if ! echo "$REMEMBERED_FACTS" | jq empty 2>/dev/null; then
+    REMEMBERED_FACTS="[]"
+fi
+
+jq -n \
+  --arg sid "$SESSION_ID" \
+  --arg ts "$TIMESTAMP" \
+  --argjson facts "$REMEMBERED_FACTS" \
+  '{sessionId: $sid, compactedAt: $ts, rememberedFacts: $facts}' \
+  > "$MARKER_FILE"
+
+exit 0

scripts/workstation/claude-hooks/wire-memory-hooks.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""Wire the homelab-memory hooks into a user's ~/.claude/settings.json, if-absent.
+
+Part of the claude-memory MCP -> homelab CLI migration (all-users rollout).
+Idempotent + ADDITIVE: only ADDS a hook group when no existing command references
+that hook script. Never removes/reorders existing hooks, and never touches `env`
+(the per-user MEMORY_API_KEY) or any other setting. Safe to run on every reconcile
+and on a user's already-populated config.
+
+Usage: wire-memory-hooks.py <home_dir>
+Exit 0 on success (changed or already-present); 1 only on an unreadable settings file.
+"""
+import json
+import os
+import sys
+
+home = sys.argv[1]
+settings = os.path.join(home, ".claude", "settings.json")
+hooks_dir = os.path.join(home, ".claude", "hooks")
+
+# (event, script-basename used for the if-absent check, full command, extra fields)
+WANT = [
+    ("PreCompact", "pre-compact-backup.sh", f"{hooks_dir}/pre-compact-backup.sh", {"timeout": 30}),
+    ("UserPromptSubmit", "post-compact-recovery.sh", f"{hooks_dir}/post-compact-recovery.sh", {"timeout": 10}),
+    ("UserPromptSubmit", "homelab-memory-recall.py", f"python3 {hooks_dir}/homelab-memory-recall.py", {"timeout": 8}),
+    ("Stop", "auto-learn.py", f"python3 {hooks_dir}/auto-learn.py", {"async": True}),
+]
+
+try:
+    if os.path.exists(settings) and os.path.getsize(settings) > 0:
+        with open(settings) as fh:
+            data = json.load(fh)
+    else:
+        data = {}
+except (json.JSONDecodeError, OSError) as e:
+    print(f"ERROR: cannot read {settings}: {e}", file=sys.stderr)
+    sys.exit(1)
+
+hooks = data.setdefault("hooks", {})
+changed = False
+for event, basename, command, extra in WANT:
+    groups = hooks.setdefault(event, [])
+    already = any(
+        basename in (h.get("command", "") or "")
+        for g in groups
+        for h in (g.get("hooks", []) or [])
+    )
+    if already:
+        continue
+    entry = {"type": "command", "command": command}
+    entry.update(extra)
+    groups.append({"hooks": [entry]})
+    changed = True
+
+if changed:
+    tmp = settings + ".tmp"
+    with open(tmp, "w") as fh:
+        json.dump(data, fh, indent=2)
+    os.replace(tmp, settings)
+    print(f"wired memory hooks -> {settings}")
+else:
+    print(f"memory hooks already present -> {settings} (no change)")