From 9fe500ce4b97b5a04c2de196c88d879f1f3e3ccd Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 13 Jun 2026 20:51:54 +0000 Subject: [PATCH] =?UTF-8?q?feat(demo):=20HyDE=20Theater=20=E2=80=94=20hypo?= =?UTF-8?q?thetical=20document=20embeddings,=20before/after?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a sixth flagship surface (/hyde) showcasing HyDE — embedding a hypothetical *answer* (document-space) instead of the raw query (query-space) to close the distribution gap. - demo/hyde.py — HyDEEngine runs the real comparison: it synthesizes a document-shaped hypothesis with a deterministic, generic template (labelled rule-based / no-LLM, following konjoai.retrieve.hyde's real _HYDE_PROMPT intent), then runs the *same* real dense cosine retrieval on both the raw query and the hypothesis. Before/after rankings, per-doc score deltas, and the closed top-document gap are measured, not staged. - demo/pipeline.py — public dense() (real cosine retrieval as formatted rows), reused for both baseline and HyDE runs. - demo/hyde.html — cinematic "close the gap" view: query vs hypothesis with embedding stats, a typewriter hypothesis reveal, an animated top-similarity gap meter (query → HyDE delta), and side-by-side retrieval columns with per-document rank/score deltas. Ships reduced-motion, focus-visible, view-transition and responsive layout. - demo/server.py — GET /hyde + GET /api/hyde/analyze (bounded inputs). - HyDE linked in the nav across all surfaces (now: Playground · Observatory · Pipeline · Agent · Security · HyDE). - tests/unit/test_demo_hyde.py — 7 contract tests (shape + no-LLM label, hypothesis denser than query, real bounded ranked retrieval, delta arithmetic, winner is a real source). 43 demo tests pass (stable across PYTHONHASHSEED); ruff clean repo-wide. https://claude.ai/code/session_01WWWgV1NwxaaNdPfy4PHN1K --- demo/agent.html | 1 + demo/hyde.html | 340 +++++++++++++++++++++++++++++++++++ demo/hyde.py | 138 ++++++++++++++ demo/index.html | 1 + demo/observatory.html | 1 + demo/pipeline.html | 1 + demo/pipeline.py | 12 ++ demo/security.html | 1 + demo/server.py | 16 ++ tests/unit/test_demo_hyde.py | 121 +++++++++++++ 10 files changed, 632 insertions(+) create mode 100644 demo/hyde.html create mode 100644 demo/hyde.py create mode 100644 tests/unit/test_demo_hyde.py diff --git a/demo/agent.html b/demo/agent.html index 56b7ed6..efcc3c7 100644 --- a/demo/agent.html +++ b/demo/agent.html @@ -173,6 +173,7 @@ Pipeline Agent Security + HyDE
connecting…
diff --git a/demo/hyde.html b/demo/hyde.html new file mode 100644 index 0000000..2619468 --- /dev/null +++ b/demo/hyde.html @@ -0,0 +1,340 @@ + + + + + +kyro · HyDE Theater + + + +
+
kyro hyde theater
+ +
+
connecting…
+
+ +
+
+
距離 · hypothetical document embeddings · real retrieval
+

Close the gap.

+

A short query lives in query-space, far from the documents. HyDE writes a hypothetical answer first, embeds that, and lands in document-space — closing the distribution gap. Watch the same real cosine retrieval run on the raw query vs the hypothesis, side by side.

+
+ +
+
+ + +
+
+ +
+ +
+
query-space Raw query
+
+
+
+ +
+
document-space Hypothetical answer rule-based · no LLM
+
+
+
+ +
+
+ Top-document similarity · query → HyDE + +
+
+
+
+ + +
+
+
+ +
+
+
Baseline · embed(query)
+
+
+
+
HyDE · embed(hypothesis)
+
+
+
+ +
+ source · konjoai.retrieve.hyde intent + real dense cosine over the corpus (pipeline.dense) · + the hypothesis is a labelled rule-based template (no LLM); the embeddings, scores & rankings are measured +
+
+ + + + diff --git a/demo/hyde.py b/demo/hyde.py new file mode 100644 index 0000000..4b40b79 --- /dev/null +++ b/demo/hyde.py @@ -0,0 +1,138 @@ +"""Kyro HyDE Theater — Hypothetical Document Embeddings, side by side. + +HyDE (Gao et al. 2022) closes the query↔document distribution gap: instead of +embedding a short query, you first write a *hypothetical answer paragraph* and +embed that, landing the vector in document-space. ``konjoai.retrieve.hyde`` +implements exactly this — :func:`generate_hypothesis` (LLM) + +:func:`hyde_encode` (encode the hypothesis). + +This engine shows the effect live and honestly: + +* **Hypothesis (no LLM).** The demo synthesizes the hypothetical answer with a + deterministic, generic template — labelled ``rule-based`` in the payload — + rather than calling a generator. It follows kyro's real ``_HYDE_PROMPT`` + intent (a short, declarative, document-shaped paragraph). +* **Everything else is real.** Both the raw query and the hypothesis are + embedded with the demo encoder (float32 / L2-unit, the K4 contract) and run + through the *same* real dense cosine retrieval over the corpus + (:meth:`pipeline.PipelineEngine.dense`). The before/after rankings, scores, + and the closed gap are measured, not staged. +""" +from __future__ import annotations + +import re +from collections.abc import Callable +from typing import Any + +import numpy as np +from pipeline import PipelineEngine + +from konjoai.retrieve.hyde import _HYDE_PROMPT # the real prompt intent (template only) + +__all__ = ["HyDEEngine"] + +# Mirror of the question words / stop tokens we strip when distilling a query +# into the content terms the hypothesis is built around. +_STOP = { + "the", "a", "an", "is", "are", "was", "were", "be", "of", "to", "for", "in", + "on", "at", "what", "which", "how", "where", "when", "why", "who", "do", + "does", "did", "you", "your", "my", "i", "it", "that", "this", "with", "and", + "or", "but", "as", "from", "by", "so", "if", "can", "could", "would", + "should", "have", "has", "had", "will", "about", "tell", "me", "us", "any", +} + + +def _content_terms(question: str) -> list[str]: + toks = re.findall(r"[a-zA-Z][a-zA-Z'-]+", question.lower()) + return [t for t in toks if t not in _STOP and len(t) > 2] + + +def _synthesize_hypothesis(question: str) -> str: + """Build a deterministic, document-shaped answer paragraph (no LLM). + + Generic transform — never hand-tuned per query: it restates the question + declaratively and repeats its content terms so the paragraph reads like a + corpus passage rather than a short query. That density is the whole point + of HyDE; the embedding + retrieval that follow are fully real. + """ + q = question.strip().rstrip("?.! ") + terms = _content_terms(q) + topic = " ".join(terms[:6]) if terms else q + lead = q[0].upper() + q[1:] if q else "This" + return ( + f"{lead}. In practice, {topic} is handled according to the documented " + f"policy and procedure. The relevant details cover {topic}, including the " + f"specific steps, requirements, and timeframes that apply. This section " + f"explains {topic} so the answer is precise and complete." + ) + + +class HyDEEngine: + """Compares raw-query retrieval against HyDE-hypothesis retrieval. + + Parameters + ---------- + pipeline: + Loaded :class:`~pipeline.PipelineEngine` — provides the real dense + cosine retrieval used for both the baseline and HyDE runs. + embed_fn: + Encoder returning an L2-unit ``float32`` vector — used only to report + the query vs hypothesis embedding statistics the UI renders. + """ + + def __init__(self, pipeline: PipelineEngine, embed_fn: Callable[[str], np.ndarray]) -> None: + self.pipeline = pipeline + self._embed = embed_fn + + def _vec_stats(self, text: str) -> dict[str, Any]: + v = self._embed(text) + if v.ndim == 2: + v = v.reshape(-1) + return { + "dim": int(v.shape[0]), + "dtype": str(v.dtype), + "nonzero": int(np.count_nonzero(v)), + "tokens": len(re.findall(r"[a-zA-Z][a-zA-Z'-]+", text)), + } + + def analyze(self, question: str, top_k: int = 4) -> dict[str, Any]: + """Run baseline vs HyDE retrieval and return a structured comparison. + + Wire contract consumed by ``hyde.html``; pinned by + :mod:`tests.unit.test_demo_hyde`. + """ + question = (question or "").strip() + if not question: + return {"error": "question must be non-empty"} + + hypothesis = _synthesize_hypothesis(question) + baseline = self.pipeline.dense(question, top_k=top_k) + hyde = self.pipeline.dense(hypothesis, top_k=top_k) + + base_top = baseline[0]["score"] if baseline else 0.0 + hyde_top = hyde[0]["score"] if hyde else 0.0 + base_rank = {r["source"]: r["rank"] for r in baseline} + hyde_winner = hyde[0]["source"] if hyde else None + # How far the HyDE winner sat in the raw-query ranking (None = unranked). + winner_prev_rank = base_rank.get(hyde_winner) + + return { + "question": question, + "hypothesis": hypothesis, + "hypothesis_source": "rule-based template (no LLM in demo)", + "prompt_intent": _HYDE_PROMPT.split("\n", 1)[0], + "top_k": top_k, + "baseline": baseline, + "hyde": hyde, + "query_vec": self._vec_stats(question), + "hyde_vec": self._vec_stats(hypothesis), + "comparison": { + "baseline_top_score": round(float(base_top), 4), + "hyde_top_score": round(float(hyde_top), 4), + "delta": round(float(hyde_top - base_top), 4), + "winner": hyde_winner, + "winner_changed": bool(baseline and hyde and baseline[0]["source"] != hyde_winner), + "winner_prev_rank": winner_prev_rank, + }, + "source": "konjoai.retrieve.hyde intent + real dense cosine (pipeline.dense)", + } diff --git a/demo/index.html b/demo/index.html index 4896098..63d3dac 100644 --- a/demo/index.html +++ b/demo/index.html @@ -348,6 +348,7 @@ ⌥ Pipeline ⊹ Agent 🛡 Security + 🔭 HyDE 🛰 Observatory
diff --git a/demo/observatory.html b/demo/observatory.html index dad1485..540626d 100644 --- a/demo/observatory.html +++ b/demo/observatory.html @@ -295,6 +295,7 @@

kyro · observatory

⌥ pipeline ⊹ agent 🛡 security + 🔭 hyde ↩ search
diff --git a/demo/pipeline.html b/demo/pipeline.html index 98c2f44..49a26ea 100644 --- a/demo/pipeline.html +++ b/demo/pipeline.html @@ -257,6 +257,7 @@ Pipeline Agent Security + HyDE
connecting…
diff --git a/demo/pipeline.py b/demo/pipeline.py index fa88edb..87d09d4 100644 --- a/demo/pipeline.py +++ b/demo/pipeline.py @@ -164,6 +164,18 @@ def _dense(self, query: str, top_k: int) -> list[SearchResult]: def _sparse(self, query: str, top_k: int) -> list[BM25Result]: return self._bm25.search(query, top_k=top_k) + def dense(self, query: str, top_k: int = 5) -> list[dict[str, Any]]: + """Real dense cosine retrieval as formatted rows (rank 0 = best). + + Embeds ``query`` with the engine's encoder — so passing a HyDE + hypothesis paragraph here yields document-space retrieval, which is + exactly what the HyDE theater compares against the raw query. + """ + if not self._loaded: + self.load() + top_k = max(1, min(int(top_k), len(self._docs))) + return [self._fmt_dense(r, rank) for rank, r in enumerate(self._dense(query, top_k))] + def hybrid(self, query: str, top_k: int = 5, alpha: float = 0.7) -> list[HybridResult]: """Real dense ∥ BM25 retrieval fused with ``reciprocal_rank_fusion``. diff --git a/demo/security.html b/demo/security.html index 4d126ba..62325a6 100644 --- a/demo/security.html +++ b/demo/security.html @@ -161,6 +161,7 @@ Pipeline Agent Security + HyDE
connecting…
diff --git a/demo/server.py b/demo/server.py index 034acb1..9bc736f 100644 --- a/demo/server.py +++ b/demo/server.py @@ -48,6 +48,7 @@ sys.path.insert(0, str(_REPO_ROOT)) from agent import AgentEngine # noqa: E402 (sibling module, demo/ on sys.path) +from hyde import HyDEEngine # noqa: E402 (sibling module, demo/ on sys.path) from pipeline import PipelineEngine # noqa: E402 (sibling module, demo/ on sys.path) from security import SecurityEngine # noqa: E402 (sibling module, demo/ on sys.path) @@ -683,11 +684,13 @@ def search(self, query: str, top_k: int = 3) -> list[dict[str, Any]]: _pipeline = PipelineEngine(Path(__file__).parent / "corpus", encode) _agent = AgentEngine(_pipeline) _security = SecurityEngine(encode) +_hyde = HyDEEngine(_pipeline, encode) _HTML_PATH = Path(__file__).parent / "index.html" _OBSERVATORY_PATH = Path(__file__).parent / "observatory.html" _PIPELINE_PATH = Path(__file__).parent / "pipeline.html" _AGENT_PATH = Path(__file__).parent / "agent.html" _SECURITY_PATH = Path(__file__).parent / "security.html" +_HYDE_PATH = Path(__file__).parent / "hyde.html" class Handler(BaseHTTPRequestHandler): @@ -772,6 +775,8 @@ def do_GET(self) -> None: # noqa: N802 return self._serve_html(_AGENT_PATH) if path in ("/security", "/security.html"): return self._serve_html(_SECURITY_PATH) + if path in ("/hyde", "/hyde.html"): + return self._serve_html(_HYDE_PATH) if path == "/api/health": return self._send_json( { @@ -849,6 +854,16 @@ def do_GET(self) -> None: # noqa: N802 if path == "/api/agent/stream": return self._send_sse(_agent.stream(question, max_steps=max_steps, top_k=top_k)) return self._send_json(_agent.analyze(question, max_steps=max_steps, top_k=top_k)) + if path == "/api/hyde/analyze": + raw = (params.get("query", [""])[0] or "").strip()[:256] + if not raw: + idx = int(time.time()) % len(CorpusIndex.DEFAULT_DEMO_QUERIES) + raw = CorpusIndex.DEFAULT_DEMO_QUERIES[idx] + try: + top_k = max(1, min(int(params.get("top_k", ["4"])[0]), 6)) + except ValueError: + top_k = 4 + return self._send_json(_hyde.analyze(raw, top_k=top_k)) if path == "/api/security/scenario": return self._send_json(_security.scenario()) if path == "/api/security/stats": @@ -926,6 +941,7 @@ def main() -> None: log.info(" GET /pipeline → demo/pipeline.html (hybrid retrieval theater)") log.info(" GET /agent → demo/agent.html (ReAct agent theater)") log.info(" GET /security → demo/security.html (cache-poisoning guard theater)") + log.info(" GET /hyde → demo/hyde.html (HyDE retrieval theater)") log.info(" GET /api/health → liveness") log.info(" GET /api/cache/stats → real SemanticCache.stats()") log.info(" POST /api/cache/ask → real cosine + lookup, JSON {question}") diff --git a/tests/unit/test_demo_hyde.py b/tests/unit/test_demo_hyde.py new file mode 100644 index 0000000..e9d55c4 --- /dev/null +++ b/tests/unit/test_demo_hyde.py @@ -0,0 +1,121 @@ +"""HyDE Theater contract — ``demo/hyde.py`` real before/after retrieval. + +``demo/hyde.html`` renders every field these tests pin down. The headline +guarantee is *honesty*: only the hypothesis text is rule-based (labelled); both +the raw query and the hypothesis are embedded and run through the *same* real +dense cosine retrieval, so the before/after scores and rankings are measured. + +Konjo gates exercised: + K3 — retrieval is real konjoai dense cosine; the hypothesis is disclosed. + K4 — vector stats report the float32 embedding contract. +""" +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import pytest + +_DEMO_DIR = Path(__file__).resolve().parents[2] / "demo" +if str(_DEMO_DIR) not in sys.path: + sys.path.insert(0, str(_DEMO_DIR)) + +from hyde import HyDEEngine, _synthesize_hypothesis # noqa: E402 +from pipeline import PipelineEngine # noqa: E402 + +_DIM = 64 + + +def _embed(text: str) -> np.ndarray: + """Deterministic bag-of-words embedder — denser text → more non-zero dims, + which is exactly the query→document shift HyDE exploits.""" + v = np.zeros(_DIM, dtype=np.float32) + for tok in text.lower().split(): + v[hash(tok) % _DIM] += 1.0 + n = float(np.linalg.norm(v)) + if n < 1e-12: + v[0] = 1.0 + return v + return (v / n).astype(np.float32) + + +@pytest.fixture +def engine(tmp_path: Path) -> HyDEEngine: + corpus = tmp_path / "corpus" + corpus.mkdir() + docs = { + "technical_01_api_authentication.txt": "API authentication uses bearer tokens and OAuth scopes for requests.", + "technical_02_rate_limiting.txt": "Rate limiting throttles requests per minute using a token bucket policy.", + "legal_01_privacy_policy.txt": "The privacy policy describes GDPR data subject rights, consent, and retention.", + "medical_01_acetaminophen.txt": "Acetaminophen pediatric dosing depends on the child weight in kilograms.", + } + for name, body in docs.items(): + (corpus / name).write_text(body, encoding="utf-8") + pipe = PipelineEngine(corpus, _embed) + pipe.load() + return HyDEEngine(pipe, _embed) + + +# ── 1. Shape + honesty labels ─────────────────────────────────────────────── + + +def test_analyze_has_full_shape(engine: HyDEEngine) -> None: + out = engine.analyze("What are my GDPR rights?", top_k=3) + assert out.keys() >= { + "question", "hypothesis", "hypothesis_source", "baseline", "hyde", + "query_vec", "hyde_vec", "comparison", "source", + } + assert "no llm" in out["hypothesis_source"].lower() + assert "hyde" in out["source"].lower() + assert out["comparison"].keys() >= { + "baseline_top_score", "hyde_top_score", "delta", "winner", "winner_changed", + } + + +def test_empty_question_is_rejected(engine: HyDEEngine) -> None: + assert engine.analyze(" ")["error"] + + +# ── 2. Hypothesis is document-shaped (denser than the query) ──────────────── + + +def test_hypothesis_is_denser_than_query(engine: HyDEEngine) -> None: + out = engine.analyze("What are my GDPR rights?", top_k=3) + assert out["hyde_vec"]["tokens"] > out["query_vec"]["tokens"] + assert out["query_vec"]["dtype"] == "float32" + assert out["hyde_vec"]["dtype"] == "float32" + + +def test_hypothesis_contains_query_terms(engine: HyDEEngine) -> None: + hyp = _synthesize_hypothesis("How do I deploy kyro to Kubernetes?") + assert "deploy" in hyp.lower() + assert "kubernetes" in hyp.lower() + assert len(hyp) > 80 # a paragraph, not a query + + +# ── 3. Retrieval is real and bounded ──────────────────────────────────────── + + +def test_both_runs_are_ranked_and_bounded(engine: HyDEEngine) -> None: + out = engine.analyze("GDPR data subject rights", top_k=2) + for rows in (out["baseline"], out["hyde"]): + assert 1 <= len(rows) <= 2 + assert [r["rank"] for r in rows] == sorted(r["rank"] for r in rows) + scores = [r["score"] for r in rows] + assert scores == sorted(scores, reverse=True) + + +def test_hyde_closes_the_gap(engine: HyDEEngine) -> None: + """The document-shaped hypothesis should raise the top-document cosine — + the measured HyDE effect, not a staged one.""" + out = engine.analyze("What are my GDPR rights?", top_k=4) + c = out["comparison"] + assert c["hyde_top_score"] >= c["baseline_top_score"] + assert c["delta"] == pytest.approx(c["hyde_top_score"] - c["baseline_top_score"], abs=1e-6) + + +def test_comparison_winner_is_a_real_source(engine: HyDEEngine) -> None: + out = engine.analyze("rate limiting policy", top_k=3) + assert out["comparison"]["winner"] == out["hyde"][0]["source"] + assert out["comparison"]["winner"].endswith(".txt")