feat(demo): HyDE Theater — hypothetical document embeddings, before/after

demo/agent.html

@@ -173,6 +173,7 @@
     <a href="/pipeline">Pipeline</a>
     <a href="/agent" class="active" aria-current="page">Agent</a>
     <a href="/security">Security</a>
+    <a href="/hyde">HyDE</a>
   </nav>
   <div class="spacer"></div>
   <div class="status-pill"><span class="dot" id="dot"></span><span id="status">connecting…</span></div>

demo/hyde.py

@@ -0,0 +1,138 @@
+"""Kyro HyDE Theater — Hypothetical Document Embeddings, side by side.
+
+HyDE (Gao et al. 2022) closes the query↔document distribution gap: instead of
+embedding a short query, you first write a *hypothetical answer paragraph* and
+embed that, landing the vector in document-space. ``konjoai.retrieve.hyde``
+implements exactly this — :func:`generate_hypothesis` (LLM) +
+:func:`hyde_encode` (encode the hypothesis).
+
+This engine shows the effect live and honestly:
+
+* **Hypothesis (no LLM).** The demo synthesizes the hypothetical answer with a
+  deterministic, generic template — labelled ``rule-based`` in the payload —
+  rather than calling a generator. It follows kyro's real ``_HYDE_PROMPT``
+  intent (a short, declarative, document-shaped paragraph).
+* **Everything else is real.** Both the raw query and the hypothesis are
+  embedded with the demo encoder (float32 / L2-unit, the K4 contract) and run
+  through the *same* real dense cosine retrieval over the corpus
+  (:meth:`pipeline.PipelineEngine.dense`). The before/after rankings, scores,
+  and the closed gap are measured, not staged.
+"""
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from typing import Any
+
+import numpy as np
+from pipeline import PipelineEngine
+
+from konjoai.retrieve.hyde import _HYDE_PROMPT  # the real prompt intent (template only)
+
+__all__ = ["HyDEEngine"]
+
+# Mirror of the question words / stop tokens we strip when distilling a query
+# into the content terms the hypothesis is built around.
+_STOP = {
+    "the", "a", "an", "is", "are", "was", "were", "be", "of", "to", "for", "in",
+    "on", "at", "what", "which", "how", "where", "when", "why", "who", "do",
+    "does", "did", "you", "your", "my", "i", "it", "that", "this", "with", "and",
+    "or", "but", "as", "from", "by", "so", "if", "can", "could", "would",
+    "should", "have", "has", "had", "will", "about", "tell", "me", "us", "any",
+}
+
+
+def _content_terms(question: str) -> list[str]:
+    toks = re.findall(r"[a-zA-Z][a-zA-Z'-]+", question.lower())
+    return [t for t in toks if t not in _STOP and len(t) > 2]
+
+
+def _synthesize_hypothesis(question: str) -> str:
+    """Build a deterministic, document-shaped answer paragraph (no LLM).
+
+    Generic transform — never hand-tuned per query: it restates the question
+    declaratively and repeats its content terms so the paragraph reads like a
+    corpus passage rather than a short query. That density is the whole point
+    of HyDE; the embedding + retrieval that follow are fully real.
+    """
+    q = question.strip().rstrip("?.! ")
+    terms = _content_terms(q)
+    topic = " ".join(terms[:6]) if terms else q
+    lead = q[0].upper() + q[1:] if q else "This"
+    return (
+        f"{lead}. In practice, {topic} is handled according to the documented "
+        f"policy and procedure. The relevant details cover {topic}, including the "
+        f"specific steps, requirements, and timeframes that apply. This section "
+        f"explains {topic} so the answer is precise and complete."
+    )
+
+
+class HyDEEngine:
+    """Compares raw-query retrieval against HyDE-hypothesis retrieval.
+
+    Parameters
+    ----------
+    pipeline:
+        Loaded :class:`~pipeline.PipelineEngine` — provides the real dense
+        cosine retrieval used for both the baseline and HyDE runs.
+    embed_fn:
+        Encoder returning an L2-unit ``float32`` vector — used only to report
+        the query vs hypothesis embedding statistics the UI renders.
+    """
+
+    def __init__(self, pipeline: PipelineEngine, embed_fn: Callable[[str], np.ndarray]) -> None:
+        self.pipeline = pipeline
+        self._embed = embed_fn
+
+    def _vec_stats(self, text: str) -> dict[str, Any]:
+        v = self._embed(text)
+        if v.ndim == 2:
+            v = v.reshape(-1)
+        return {
+            "dim": int(v.shape[0]),
+            "dtype": str(v.dtype),
+            "nonzero": int(np.count_nonzero(v)),
+            "tokens": len(re.findall(r"[a-zA-Z][a-zA-Z'-]+", text)),
+        }
+
+    def analyze(self, question: str, top_k: int = 4) -> dict[str, Any]:
+        """Run baseline vs HyDE retrieval and return a structured comparison.
+
+        Wire contract consumed by ``hyde.html``; pinned by
+        :mod:`tests.unit.test_demo_hyde`.
+        """
+        question = (question or "").strip()
+        if not question:
+            return {"error": "question must be non-empty"}
+
+        hypothesis = _synthesize_hypothesis(question)
+        baseline = self.pipeline.dense(question, top_k=top_k)
+        hyde = self.pipeline.dense(hypothesis, top_k=top_k)
+
+        base_top = baseline[0]["score"] if baseline else 0.0
+        hyde_top = hyde[0]["score"] if hyde else 0.0
+        base_rank = {r["source"]: r["rank"] for r in baseline}
+        hyde_winner = hyde[0]["source"] if hyde else None
+        # How far the HyDE winner sat in the raw-query ranking (None = unranked).
+        winner_prev_rank = base_rank.get(hyde_winner)
+
+        return {
+            "question": question,
+            "hypothesis": hypothesis,
+            "hypothesis_source": "rule-based template (no LLM in demo)",
+            "prompt_intent": _HYDE_PROMPT.split("\n", 1)[0],
+            "top_k": top_k,
+            "baseline": baseline,
+            "hyde": hyde,
+            "query_vec": self._vec_stats(question),
+            "hyde_vec": self._vec_stats(hypothesis),
+            "comparison": {
+                "baseline_top_score": round(float(base_top), 4),
+                "hyde_top_score": round(float(hyde_top), 4),
+                "delta": round(float(hyde_top - base_top), 4),
+                "winner": hyde_winner,
+                "winner_changed": bool(baseline and hyde and baseline[0]["source"] != hyde_winner),
+                "winner_prev_rank": winner_prev_rank,
+            },
+            "source": "konjoai.retrieve.hyde intent + real dense cosine (pipeline.dense)",
+        }

demo/index.html

@@ -348,6 +348,7 @@
     <a class="tab-btn" href="/pipeline" style="text-decoration:none">⌥ Pipeline</a>
     <a class="tab-btn" href="/agent" style="text-decoration:none">⊹ Agent</a>
     <a class="tab-btn" href="/security" style="text-decoration:none">🛡 Security</a>
+    <a class="tab-btn" href="/hyde" style="text-decoration:none">🔭 HyDE</a>
     <a class="tab-btn" href="/observatory" style="text-decoration:none">🛰 Observatory</a>
   </nav>
   <div id="health-pill">

demo/observatory.html

@@ -295,6 +295,7 @@ <h1>kyro · observatory</h1>
       <a href="/pipeline" class="nav-link">⌥ pipeline</a>
       <a href="/agent" class="nav-link">⊹ agent</a>
       <a href="/security" class="nav-link">🛡 security</a>
+      <a href="/hyde" class="nav-link">🔭 hyde</a>
       <a href="/" class="nav-link">↩ search</a>
     </div>
   </header>

demo/pipeline.html

@@ -257,6 +257,7 @@
     <a href="/pipeline" class="active" aria-current="page">Pipeline</a>
     <a href="/agent">Agent</a>
     <a href="/security">Security</a>
+    <a href="/hyde">HyDE</a>
   </nav>
   <div class="spacer"></div>
   <div class="status-pill"><span class="dot" id="dot"></span><span id="status">connecting…</span></div>

demo/pipeline.py

@@ -164,6 +164,18 @@ def _dense(self, query: str, top_k: int) -> list[SearchResult]:
     def _sparse(self, query: str, top_k: int) -> list[BM25Result]:
         return self._bm25.search(query, top_k=top_k)
 
+    def dense(self, query: str, top_k: int = 5) -> list[dict[str, Any]]:
+        """Real dense cosine retrieval as formatted rows (rank 0 = best).
+
+        Embeds ``query`` with the engine's encoder — so passing a HyDE
+        hypothesis paragraph here yields document-space retrieval, which is
+        exactly what the HyDE theater compares against the raw query.
+        """
+        if not self._loaded:
+            self.load()
+        top_k = max(1, min(int(top_k), len(self._docs)))
+        return [self._fmt_dense(r, rank) for rank, r in enumerate(self._dense(query, top_k))]
+
     def hybrid(self, query: str, top_k: int = 5, alpha: float = 0.7) -> list[HybridResult]:
         """Real dense ∥ BM25 retrieval fused with ``reciprocal_rank_fusion``.
 

demo/security.html

@@ -161,6 +161,7 @@
     <a href="/pipeline">Pipeline</a>
     <a href="/agent">Agent</a>
     <a href="/security" class="active" aria-current="page">Security</a>
+    <a href="/hyde">HyDE</a>
   </nav>
   <div class="spacer"></div>
   <div class="status-pill"><span class="dot" id="dot"></span><span id="status">connecting…</span></div>

demo/server.py

@@ -48,6 +48,7 @@
     sys.path.insert(0, str(_REPO_ROOT))
 
 from agent import AgentEngine  # noqa: E402  (sibling module, demo/ on sys.path)
+from hyde import HyDEEngine  # noqa: E402  (sibling module, demo/ on sys.path)
 from pipeline import PipelineEngine  # noqa: E402  (sibling module, demo/ on sys.path)
 from security import SecurityEngine  # noqa: E402  (sibling module, demo/ on sys.path)
 
@@ -683,11 +684,13 @@ def search(self, query: str, top_k: int = 3) -> list[dict[str, Any]]:
 _pipeline = PipelineEngine(Path(__file__).parent / "corpus", encode)
 _agent = AgentEngine(_pipeline)
 _security = SecurityEngine(encode)
+_hyde = HyDEEngine(_pipeline, encode)
 _HTML_PATH = Path(__file__).parent / "index.html"
 _OBSERVATORY_PATH = Path(__file__).parent / "observatory.html"
 _PIPELINE_PATH = Path(__file__).parent / "pipeline.html"
 _AGENT_PATH = Path(__file__).parent / "agent.html"
 _SECURITY_PATH = Path(__file__).parent / "security.html"
+_HYDE_PATH = Path(__file__).parent / "hyde.html"
 
 
 class Handler(BaseHTTPRequestHandler):
@@ -772,6 +775,8 @@ def do_GET(self) -> None:  # noqa: N802
             return self._serve_html(_AGENT_PATH)
         if path in ("/security", "/security.html"):
             return self._serve_html(_SECURITY_PATH)
+        if path in ("/hyde", "/hyde.html"):
+            return self._serve_html(_HYDE_PATH)
         if path == "/api/health":
             return self._send_json(
                 {
@@ -849,6 +854,16 @@ def do_GET(self) -> None:  # noqa: N802
             if path == "/api/agent/stream":
                 return self._send_sse(_agent.stream(question, max_steps=max_steps, top_k=top_k))
             return self._send_json(_agent.analyze(question, max_steps=max_steps, top_k=top_k))
+        if path == "/api/hyde/analyze":
+            raw = (params.get("query", [""])[0] or "").strip()[:256]
+            if not raw:
+                idx = int(time.time()) % len(CorpusIndex.DEFAULT_DEMO_QUERIES)
+                raw = CorpusIndex.DEFAULT_DEMO_QUERIES[idx]
+            try:
+                top_k = max(1, min(int(params.get("top_k", ["4"])[0]), 6))
+            except ValueError:
+                top_k = 4
+            return self._send_json(_hyde.analyze(raw, top_k=top_k))
         if path == "/api/security/scenario":
             return self._send_json(_security.scenario())
         if path == "/api/security/stats":
@@ -926,6 +941,7 @@ def main() -> None:
     log.info("  GET  /pipeline          → demo/pipeline.html (hybrid retrieval theater)")
     log.info("  GET  /agent             → demo/agent.html (ReAct agent theater)")
     log.info("  GET  /security          → demo/security.html (cache-poisoning guard theater)")
+    log.info("  GET  /hyde              → demo/hyde.html (HyDE retrieval theater)")
     log.info("  GET  /api/health        → liveness")
     log.info("  GET  /api/cache/stats   → real SemanticCache.stats()")
     log.info("  POST /api/cache/ask     → real cosine + lookup, JSON {question}")

tests/unit/test_demo_hyde.py

@@ -0,0 +1,121 @@
+"""HyDE Theater contract — ``demo/hyde.py`` real before/after retrieval.
+
+``demo/hyde.html`` renders every field these tests pin down. The headline
+guarantee is *honesty*: only the hypothesis text is rule-based (labelled); both
+the raw query and the hypothesis are embedded and run through the *same* real
+dense cosine retrieval, so the before/after scores and rankings are measured.
+
+Konjo gates exercised:
+  K3 — retrieval is real konjoai dense cosine; the hypothesis is disclosed.
+  K4 — vector stats report the float32 embedding contract.
+"""
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+_DEMO_DIR = Path(__file__).resolve().parents[2] / "demo"
+if str(_DEMO_DIR) not in sys.path:
+    sys.path.insert(0, str(_DEMO_DIR))
+
+from hyde import HyDEEngine, _synthesize_hypothesis  # noqa: E402
+from pipeline import PipelineEngine  # noqa: E402
+
+_DIM = 64
+
+
+def _embed(text: str) -> np.ndarray:
+    """Deterministic bag-of-words embedder — denser text → more non-zero dims,
+    which is exactly the query→document shift HyDE exploits."""
+    v = np.zeros(_DIM, dtype=np.float32)
+    for tok in text.lower().split():
+        v[hash(tok) % _DIM] += 1.0
+    n = float(np.linalg.norm(v))
+    if n < 1e-12:
+        v[0] = 1.0
+        return v
+    return (v / n).astype(np.float32)
+
+
+@pytest.fixture
+def engine(tmp_path: Path) -> HyDEEngine:
+    corpus = tmp_path / "corpus"
+    corpus.mkdir()
+    docs = {
+        "technical_01_api_authentication.txt": "API authentication uses bearer tokens and OAuth scopes for requests.",
+        "technical_02_rate_limiting.txt": "Rate limiting throttles requests per minute using a token bucket policy.",
+        "legal_01_privacy_policy.txt": "The privacy policy describes GDPR data subject rights, consent, and retention.",
+        "medical_01_acetaminophen.txt": "Acetaminophen pediatric dosing depends on the child weight in kilograms.",
+    }
+    for name, body in docs.items():
+        (corpus / name).write_text(body, encoding="utf-8")
+    pipe = PipelineEngine(corpus, _embed)
+    pipe.load()
+    return HyDEEngine(pipe, _embed)
+
+
+# ── 1. Shape + honesty labels ───────────────────────────────────────────────
+
+
+def test_analyze_has_full_shape(engine: HyDEEngine) -> None:
+    out = engine.analyze("What are my GDPR rights?", top_k=3)
+    assert out.keys() >= {
+        "question", "hypothesis", "hypothesis_source", "baseline", "hyde",
+        "query_vec", "hyde_vec", "comparison", "source",
+    }
+    assert "no llm" in out["hypothesis_source"].lower()
+    assert "hyde" in out["source"].lower()
+    assert out["comparison"].keys() >= {
+        "baseline_top_score", "hyde_top_score", "delta", "winner", "winner_changed",
+    }
+
+
+def test_empty_question_is_rejected(engine: HyDEEngine) -> None:
+    assert engine.analyze("   ")["error"]
+
+
+# ── 2. Hypothesis is document-shaped (denser than the query) ────────────────
+
+
+def test_hypothesis_is_denser_than_query(engine: HyDEEngine) -> None:
+    out = engine.analyze("What are my GDPR rights?", top_k=3)
+    assert out["hyde_vec"]["tokens"] > out["query_vec"]["tokens"]
+    assert out["query_vec"]["dtype"] == "float32"
+    assert out["hyde_vec"]["dtype"] == "float32"
+
+
+def test_hypothesis_contains_query_terms(engine: HyDEEngine) -> None:
+    hyp = _synthesize_hypothesis("How do I deploy kyro to Kubernetes?")
+    assert "deploy" in hyp.lower()
+    assert "kubernetes" in hyp.lower()
+    assert len(hyp) > 80  # a paragraph, not a query
+
+
+# ── 3. Retrieval is real and bounded ────────────────────────────────────────
+
+
+def test_both_runs_are_ranked_and_bounded(engine: HyDEEngine) -> None:
+    out = engine.analyze("GDPR data subject rights", top_k=2)
+    for rows in (out["baseline"], out["hyde"]):
+        assert 1 <= len(rows) <= 2
+        assert [r["rank"] for r in rows] == sorted(r["rank"] for r in rows)
+        scores = [r["score"] for r in rows]
+        assert scores == sorted(scores, reverse=True)
+
+
+def test_hyde_closes_the_gap(engine: HyDEEngine) -> None:
+    """The document-shaped hypothesis should raise the top-document cosine —
+    the measured HyDE effect, not a staged one."""
+    out = engine.analyze("What are my GDPR rights?", top_k=4)
+    c = out["comparison"]
+    assert c["hyde_top_score"] >= c["baseline_top_score"]
+    assert c["delta"] == pytest.approx(c["hyde_top_score"] - c["baseline_top_score"], abs=1e-6)
+
+
+def test_comparison_winner_is_a_real_source(engine: HyDEEngine) -> None:
+    out = engine.analyze("rate limiting policy", top_k=3)
+    assert out["comparison"]["winner"] == out["hyde"][0]["source"]
+    assert out["comparison"]["winner"].endswith(".txt")