research: benchmark hybrid (lexical+dense+graph) recall vs current FTS

Viktor asked to enhance the memory system with 'semantics' — remember concepts
(not just tokens) linked in a graph — and to prove, by benchmarking against the
current system, that it actually improves recall. A multi-phase research workflow
(18 agents) did landscape research, an adversarially-reviewed integration design,
a stratified eval set over the real 5,452-memory corpus, and a head-to-head
prototype-vs-current benchmark.

Result: hybrid (lexical FTS + dense embeddings, RRF-fused) beats FTS on every
overall metric, driven by a robust paraphrase win (recall@10 +0.350). Recommend
adopting lexical+dense; the concept graph is DEFERRED.

Post-run adversarial review correction (applied to all docs before commit): the
prototype's fusion config structurally barred the graph leg from the ranked top-k,
so the 'graph contributes nothing' ablation was a math artifact, NOT an empirical
result — the graph is UNEVALUATED, not disproven (deferred on cost+uncertainty).
Multi-hop deltas are not statistically significant. Glossary in CONTEXT.md; framing
in ADR-0001-0003; findings in ADR-0004-0006 + docs/research/.

Privacy: the corpus/queries/qrels/results are the user's real memories and stay
gitignored (data/, cache/, results/, build_eval_set.py); only harness code,
aggregate numbers, and synthetic examples are committed.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

benchmarks/scripts/dataset_stats.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+"""Validate the eval set and print AGGREGATE stats (safe to share / commit-able
+numbers only — prints NO raw memory content)."""
+from __future__ import annotations
+
+import json
+import statistics
+import sys
+from collections import Counter
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from harness import load_dataset  # noqa: E402
+
+
+def main() -> None:
+    ds = load_dataset(validate=True)  # raises on any referential-integrity issue
+
+    strata = Counter(q.stratum for q in ds.queries)
+    rel_per_q = {s: [] for s in strata}
+    for q in ds.queries:
+        rel_per_q[q.stratum].append(len(ds.qrels[q.query_id]))
+
+    # how many DISTINCT corpus memories are exercised as relevant
+    relevant_union = set()
+    for rels in ds.qrels.values():
+        relevant_union |= rels
+
+    out = {
+        "corpus_count": len(ds.corpus),
+        "query_count": len(ds.queries),
+        "strata": dict(strata),
+        "relevant_ids_per_query": {
+            s: {
+                "min": min(v),
+                "median": statistics.median(v),
+                "max": max(v),
+                "mean": round(statistics.fmean(v), 2),
+            }
+            for s, v in rel_per_q.items()
+        },
+        "distinct_relevant_memories": len(relevant_union),
+        "validation": "PASS (all qrels ids exist in corpus; every query has qrels)",
+    }
+    print(json.dumps(out, indent=2))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/scripts/export_corpus.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""Export the local SQLite memory cache to a LOCAL-ONLY corpus.jsonl.
+
+Privacy: emits ONLY rows where is_sensitive=0. The output file lives under
+benchmarks/data/ which is gitignored. NEVER commit corpus.jsonl.
+
+Fields emitted per line: {id, content, category, tags, expanded_keywords, importance}
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sqlite3
+import sys
+from pathlib import Path
+
+DEFAULT_DB = Path.home() / ".claude" / "claude-memory" / "memory" / "memory.db"
+DEFAULT_OUT = Path(__file__).resolve().parents[1] / "data" / "corpus.jsonl"
+
+
+def export(db_path: Path, out_path: Path) -> dict:
+    if not db_path.exists():
+        raise SystemExit(f"DB not found: {db_path}")
+
+    con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
+    con.row_factory = sqlite3.Row
+    cur = con.cursor()
+
+    total = cur.execute("SELECT COUNT(*) FROM memories").fetchone()[0]
+    sensitive = cur.execute(
+        "SELECT COUNT(*) FROM memories WHERE is_sensitive=1"
+    ).fetchone()[0]
+
+    rows = cur.execute(
+        """
+        SELECT id, content, category, tags, expanded_keywords, importance
+        FROM memories
+        WHERE is_sensitive=0
+        ORDER BY id
+        """
+    ).fetchall()
+
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    written = 0
+    with out_path.open("w", encoding="utf-8") as f:
+        for r in rows:
+            rec = {
+                "id": r["id"],
+                "content": r["content"],
+                "category": r["category"],
+                "tags": r["tags"],
+                "expanded_keywords": r["expanded_keywords"],
+                "importance": r["importance"],
+            }
+            f.write(json.dumps(rec, ensure_ascii=False) + "\n")
+            written += 1
+    con.close()
+
+    return {
+        "total_rows": total,
+        "sensitive_excluded": sensitive,
+        "non_sensitive_written": written,
+        "out_path": str(out_path),
+    }
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--db", type=Path, default=DEFAULT_DB)
+    ap.add_argument("--out", type=Path, default=DEFAULT_OUT)
+    args = ap.parse_args()
+    stats = export(args.db, args.out)
+    json.dump(stats, sys.stdout, indent=2)
+    print()
+
+
+if __name__ == "__main__":
+    main()

benchmarks/scripts/run_eval.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Run the benchmark for a named retriever and print overall + per-stratum metrics.
+
+Usage:
+    .venv/bin/python scripts/run_eval.py --retriever fts5      # lexical baseline
+    .venv/bin/python scripts/run_eval.py --retriever substring # demo
+    .venv/bin/python scripts/run_eval.py --retriever mypkg.mymod:MyRetriever
+    .venv/bin/python scripts/run_eval.py --retriever fts5 --json results/fts5.json
+
+The --retriever value is either a built-in alias or a "module:Class" path. The
+class is instantiated with no args; the runner calls build_index() if present.
+
+Outputs are LOCAL-ONLY when written under results/ (gitignored): a results file
+may echo retrieved ids (not content), but keep it local to be safe.
+"""
+from __future__ import annotations
+
+import argparse
+import importlib
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from harness import load_dataset, run_benchmark  # noqa: E402
+from harness.baselines import SqliteFtsRetriever  # noqa: E402
+from harness.example_retriever import SubstringRetriever  # noqa: E402
+
+ALIASES = {
+    "fts5": lambda: SqliteFtsRetriever(sort_by="relevance"),
+    "fts5_importance": lambda: SqliteFtsRetriever(sort_by="importance"),
+    "substring": SubstringRetriever,
+}
+
+
+def resolve(spec: str):
+    if spec in ALIASES:
+        return ALIASES[spec]()
+    if ":" not in spec:
+        raise SystemExit(f"unknown retriever alias '{spec}' (use module:Class or one of {list(ALIASES)})")
+    mod_name, cls_name = spec.split(":", 1)
+    mod = importlib.import_module(mod_name)
+    return getattr(mod, cls_name)()
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--retriever", default="fts5")
+    ap.add_argument("--k", type=int, default=20, help="depth requested from retriever")
+    ap.add_argument("--json", type=Path, default=None, help="write full result JSON here")
+    args = ap.parse_args()
+
+    ds = load_dataset(validate=True)
+    retr = resolve(args.retriever)
+    res = run_benchmark(retr, ds, retrieve_k=args.k)
+    print(res.summary())
+
+    if args.json:
+        args.json.parent.mkdir(parents=True, exist_ok=True)
+        args.json.write_text(json.dumps(res.to_dict(), indent=2))
+        print(f"\nwrote {args.json}")
+
+
+if __name__ == "__main__":
+    main()