research: benchmark hybrid (lexical+dense+graph) recall vs current FTS

Viktor asked to enhance the memory system with 'semantics' — remember concepts
(not just tokens) linked in a graph — and to prove, by benchmarking against the
current system, that it actually improves recall. A multi-phase research workflow
(18 agents) did landscape research, an adversarially-reviewed integration design,
a stratified eval set over the real 5,452-memory corpus, and a head-to-head
prototype-vs-current benchmark.

Result: hybrid (lexical FTS + dense embeddings, RRF-fused) beats FTS on every
overall metric, driven by a robust paraphrase win (recall@10 +0.350). Recommend
adopting lexical+dense; the concept graph is DEFERRED.

Post-run adversarial review correction (applied to all docs before commit): the
prototype's fusion config structurally barred the graph leg from the ranked top-k,
so the 'graph contributes nothing' ablation was a math artifact, NOT an empirical
result — the graph is UNEVALUATED, not disproven (deferred on cost+uncertainty).
Multi-hop deltas are not statistically significant. Glossary in CONTEXT.md; framing
in ADR-0001-0003; findings in ADR-0004-0006 + docs/research/.

Privacy: the corpus/queries/qrels/results are the user's real memories and stay
gitignored (data/, cache/, results/, build_eval_set.py); only harness code,
aggregate numbers, and synthetic examples are committed.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

benchmarks/harness/test_harness.py

@@ -0,0 +1,145 @@
+"""Unit tests for metrics + runner. No real corpus needed (synthetic data).
+
+Run:  .venv/bin/python -m pytest harness/test_harness.py -q
+"""
+from __future__ import annotations
+
+import math
+
+from harness import metrics
+from harness.dataset import Dataset
+from harness.runner import run_benchmark, _percentile
+from harness.types import Memory, Query
+
+
+# ---------------- metrics ----------------
+
+def test_recall_at_k_basic():
+    ranked = [9, 8, 3, 7, 1]
+    rel = {3, 1, 99}  # 99 never retrieved
+    assert metrics.recall_at_k(ranked, rel, 5) == 2 / 3
+    assert metrics.recall_at_k(ranked, rel, 2) == 0.0  # neither in top2
+    assert metrics.recall_at_k(ranked, rel, 3) == 1 / 3  # only id 3 in top3
+
+
+def test_recall_perfect_and_zero():
+    assert metrics.recall_at_k([1, 2, 3], {1, 2, 3}, 5) == 1.0
+    assert metrics.recall_at_k([4, 5, 6], {1, 2, 3}, 5) == 0.0
+
+
+def test_reciprocal_rank():
+    assert metrics.reciprocal_rank([5, 4, 3], {3}) == 1 / 3
+    assert metrics.reciprocal_rank([3, 4, 5], {3}) == 1.0
+    assert metrics.reciprocal_rank([7, 8], {3}) == 0.0
+    # first relevant wins
+    assert metrics.reciprocal_rank([9, 3, 1], {1, 3}) == 1 / 2
+
+
+def test_ndcg_perfect():
+    # all relevant at the top -> nDCG == 1
+    assert math.isclose(metrics.ndcg_at_k([1, 2, 3, 4], {1, 2, 3}, 10), 1.0)
+
+
+def test_ndcg_known_value():
+    # single relevant doc at rank 2: DCG = 1/log2(3); IDCG = 1/log2(2)=1
+    ranked = [9, 1, 8]
+    val = metrics.ndcg_at_k(ranked, {1}, 10)
+    assert math.isclose(val, (1 / math.log2(3)) / 1.0)
+
+
+def test_ndcg_two_relevant_suboptimal_order():
+    # relevant {1,2}; retrieved at ranks 1 and 3
+    ranked = [1, 9, 2]
+    dcg = 1 / math.log2(2) + 1 / math.log2(4)  # ranks 1 and 3
+    idcg = 1 / math.log2(2) + 1 / math.log2(3)  # ideal ranks 1 and 2
+    assert math.isclose(metrics.ndcg_at_k(ranked, {1, 2}, 10), dcg / idcg)
+
+
+def test_dedup_does_not_inflate():
+    # repeating a relevant id must not increase recall beyond 1 hit's worth
+    ranked = [3, 3, 3, 3]
+    assert metrics.recall_at_k(ranked, {3, 7}, 5) == 0.5
+    assert metrics.reciprocal_rank(ranked, {3}) == 1.0
+
+
+def test_empty_relevant_is_zero():
+    assert metrics.recall_at_k([1, 2], set(), 5) == 0.0
+    assert metrics.ndcg_at_k([1, 2], set(), 5) == 0.0
+
+
+# ---------------- percentile ----------------
+
+def test_percentile():
+    vals = [10, 20, 30, 40]
+    assert _percentile(vals, 50) == 25.0  # interpolated median
+    assert _percentile(vals, 0) == 10
+    assert _percentile(vals, 100) == 40
+    assert _percentile([5.0], 95) == 5.0
+    assert _percentile([], 50) == 0.0
+
+
+# ---------------- runner ----------------
+
+def _toy_dataset() -> Dataset:
+    corpus = [Memory(id=i, content=f"memory {i}") for i in range(1, 11)]
+    queries = [
+        Query("q_exact_1", "find 1", "exact", (1,)),
+        Query("q_para_1", "restate 5", "paraphrase", (5,)),
+        Query("q_multi_1", "join 3 and 4", "multihop", (3, 4)),
+    ]
+    qrels = {"q_exact_1": {1}, "q_para_1": {5}, "q_multi_1": {3, 4}}
+    return Dataset(corpus=corpus, queries=queries, qrels=qrels)
+
+
+class _PerfectRetriever:
+    """Returns exactly the relevant ids first (oracle) — for runner plumbing."""
+
+    def __init__(self, qrels):
+        self._qrels = qrels
+        self._by_text = None
+
+    def build_index(self, corpus):
+        self._n = len(corpus)
+
+    def index_size_bytes(self):
+        return 1234
+
+    def retrieve(self, query, k):
+        # map query text back via the toy queries' known answers
+        mapping = {"find 1": [1], "restate 5": [5], "join 3 and 4": [3, 4]}
+        ids = mapping.get(query, [])
+        # pad with distractors
+        pad = [x for x in range(100, 100 + k)]
+        return (ids + pad)[:k]
+
+
+def test_runner_perfect_retriever():
+    ds = _toy_dataset()
+    r = _PerfectRetriever(ds.qrels)
+    res = run_benchmark(r, ds, retriever_name="perfect")
+    assert res.n_queries == 3
+    assert math.isclose(res.overall["recall@10"], 1.0)
+    assert math.isclose(res.overall["mrr"], 1.0)
+    assert math.isclose(res.overall["ndcg@10"], 1.0)
+    # per-stratum present
+    assert set(res.per_stratum) == {"exact", "paraphrase", "multihop"}
+    assert res.per_stratum["multihop"].n_queries == 1
+    # lifecycle hooks captured
+    assert res.index_build_seconds is not None
+    assert res.index_size_bytes == 1234
+    # latency recorded
+    assert res.latency_ms["p95"] >= 0.0
+
+
+def test_runner_callable_retriever_and_misses():
+    ds = _toy_dataset()
+
+    def retrieve(query, k):  # always wrong
+        return [999][:k]
+
+    res = run_benchmark(retrieve, ds, retriever_name="bad", warmup=False)
+    assert res.overall["recall@10"] == 0.0
+    assert res.overall["mrr"] == 0.0
+    assert res.index_build_seconds is None  # no hook on a bare callable
+    assert "perfect" not in res.summary()
+    assert "bad" in res.summary()