diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 6999d8c..ca848f6 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,5 +1,5 @@ For additional context about technologies to be used, project structure, shell commands, and other important information, read the current plan: -[specs/003-m2-1-live/plan.md](../specs/003-m2-1-live/plan.md) +[specs/004-eval-kit/plan.md](../specs/004-eval-kit/plan.md) diff --git a/.gitignore b/.gitignore index 125ee91..bd2c35c 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,7 @@ Thumbs.db .env .env.* !.env.example + +# Eval kit run artifacts +eval-*.md +eval-*.jsonl diff --git a/.specify/feature.json b/.specify/feature.json index a1b0289..8f07d33 100644 --- a/.specify/feature.json +++ b/.specify/feature.json @@ -1,3 +1 @@ -{ - "feature_directory": "specs/002-m2-pool-passthrough-adapters" -} +{"feature_directory":"specs/004-eval-kit"} diff --git a/README.md b/README.md index 7609b3e..a8fee66 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,10 @@ print(ctx.text) - [SDK README](sdk-python/README.md) - [CHANGELOG](CHANGELOG.md) +## Tooling + +- `openmem-eval` — manual benchmark harness comparing recall, MRR, and latency across configured providers. **Never runs in CI.** Default invocation is a dry-run that makes zero network calls. See [specs/004-eval-kit/quickstart.md](specs/004-eval-kit/quickstart.md). + ## License TBD. diff --git a/sdk-python/README.md b/sdk-python/README.md index 5a17f2d..2470159 100644 --- a/sdk-python/README.md +++ b/sdk-python/README.md @@ -50,3 +50,11 @@ Coverage gates (Constitution Principle II): That's the contract. Per Constitution Principle II ([NON-NEGOTIABLE](../.specify/memory/constitution.md)), if the suite is green your adapter is a drop-in replacement for any other. + +## Tooling + +- `omp-validate-spec` — validate the OpenAPI spec against the 3.x meta-schema. +- `openmem-eval` — manual benchmark harness comparing recall, MRR, and + latency across configured providers. **Never runs in CI.** Default + invocation makes zero network calls. See + [specs/004-eval-kit/quickstart.md](../specs/004-eval-kit/quickstart.md). diff --git a/sdk-python/openmem/eval/__init__.py b/sdk-python/openmem/eval/__init__.py new file mode 100644 index 0000000..ef8d92a --- /dev/null +++ b/sdk-python/openmem/eval/__init__.py @@ -0,0 +1,4 @@ +"""Eval Kit (`openmem-eval`) — manual benchmark harness. + +Spec: `specs/004-eval-kit/`. Never wired into CI. +""" diff --git a/sdk-python/openmem/eval/__main__.py b/sdk-python/openmem/eval/__main__.py new file mode 100644 index 0000000..67d2846 --- /dev/null +++ b/sdk-python/openmem/eval/__main__.py @@ -0,0 +1,6 @@ +"""Allow `python -m openmem.eval` invocation.""" + +from openmem.eval.cli import main + +if __name__ == "__main__": # pragma: no cover - thin shim + raise SystemExit(main()) diff --git a/sdk-python/openmem/eval/cleanup.py b/sdk-python/openmem/eval/cleanup.py new file mode 100644 index 0000000..c8611fa --- /dev/null +++ b/sdk-python/openmem/eval/cleanup.py @@ -0,0 +1,30 @@ +"""Cleanup: delete every memory written by a given run_id. + +Used by `--cleanup` (FR-015). The harness wrote facts under +``user_id = f"eval-{run_id}"`` so we list and delete them. +""" + +from __future__ import annotations + +from typing import Any + + +def cleanup(memory: Any, *, user_id: str) -> int: + """Delete every memory belonging to `user_id`. Returns count deleted.""" + deleted = 0 + cursor: str | None = None + while True: + page = memory.list(user_id, limit=200, cursor=cursor) + for item in page.items: + try: + memory.delete(item.id) + deleted += 1 + except Exception: # pragma: no cover - best-effort + pass + cursor = getattr(page, "next_cursor", None) + if not cursor: + break + return deleted + + +__all__ = ["cleanup"] diff --git a/sdk-python/openmem/eval/cli.py b/sdk-python/openmem/eval/cli.py new file mode 100644 index 0000000..75d19f7 --- /dev/null +++ b/sdk-python/openmem/eval/cli.py @@ -0,0 +1,242 @@ +"""``openmem-eval`` command-line entrypoint. + +Exit codes (per `specs/004-eval-kit/contracts/cli.md`): +* 0 — success +* 1 — no providers usable (all skipped/failed before search) +* 2 — bad invocation (argparse exits 2 itself) +* 3 — cost confirmation refused +* 4 — internal error +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path +from typing import Sequence + +from openmem.eval import cost as cost_mod +from openmem.eval.cleanup import cleanup as run_cleanup +from openmem.eval.confirm import confirm_or_exit +from openmem.eval.dataset import load_default, load_path, sample +from openmem.eval.report import write_report +from openmem.eval.runner import run as run_eval +from openmem.eval.types import RunConfig + +_DEFAULT_PROVIDERS = ("postgres", "mem0", "supermemory", "letta") + + +def _version_string() -> str: + try: + from importlib.metadata import version + sdk_ver = version("openmem") + except Exception: # pragma: no cover + sdk_ver = "unknown" + try: + from openmem.eval.dataset import load_default as _ld + ds_hash = _ld().dataset_hash + except Exception: # pragma: no cover + ds_hash = "unknown" + return f"openmem {sdk_ver}, dataset default-{ds_hash}" + + +def _build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="openmem-eval", + description="Live-only benchmark harness for the OMP SDK.", + ) + p.add_argument( + "--providers", + type=str, + default=",".join(_DEFAULT_PROVIDERS), + help="comma-separated provider names", + ) + p.add_argument( + "--live", + action="store_true", + help="execute against real provider endpoints (default: dry-run preview)", + ) + # Per contracts/cli.md §"Mutual exclusion": store --live as a regular flag + # and derive `dry_run = not live`. --dry-run is accepted for explicitness + # but is mutually exclusive with --live. + p.add_argument( + "--dry-run", + action="store_true", + dest="dry_run", + help="explicit no-network preview (default behaviour when --live is omitted)", + ) + p.add_argument( + "--report", + type=Path, + default=Path("eval-report.md"), + help="output Markdown report path", + ) + p.add_argument( + "--trace", + type=Path, + default=Path("eval-trace.jsonl"), + help="output JSONL trace path", + ) + p.add_argument( + "--dataset", + type=Path, + default=None, + help="path to a directory with facts.jsonl + queries.jsonl", + ) + p.add_argument( + "--sample", + type=int, + default=None, + help="run only the first N queries (deterministic)", + ) + p.add_argument( + "--cost-threshold", + type=float, + default=1.00, + help="abort if estimated cost exceeds this USD value (default: 1.00)", + ) + p.add_argument("--yes", action="store_true", help="skip cost confirmation prompt") + p.add_argument("--cleanup", action="store_true", help="delete written memories after run") + p.add_argument("-v", "--verbose", action="store_true", help="verbose output") + p.add_argument("--version", action="store_true", help="print version and exit") + return p + + +def _refuse_in_ci() -> int | None: + """FR-011 — never run live in CI.""" + if os.environ.get("CI") in {"true", "1"} or os.environ.get("GITHUB_ACTIONS") == "true": + sys.stderr.write( + "openmem-eval: refusing to run live in CI (set CI=0 to override locally)\n" + ) + return 4 + return None + + +def _confirm(prompt: str) -> bool: + try: + ans = input(prompt).strip().lower() + except EOFError: + return False + return ans in {"y", "yes"} + + +def _print_dry_run_table(providers, dataset, cleanup: bool) -> float: + """Print the per-provider dry-run cost-estimate table to stdout. + + Returns the total estimated USD across all providers. + """ + print("DRY RUN — no live API calls made.") + print() + print(f"{'provider':<14}{'adds':>8}{'searches':>10}{'deletes':>10}{'est. cost':>14}") + total = 0.0 + for p in providers: + est = cost_mod.estimate( + p, + add_calls=len(dataset.facts), + search_calls=len(dataset.queries), + delete_calls=len(dataset.facts) if cleanup else 0, + ) + total += est.estimated_usd + print( + f"{p:<14}{est.add_calls:>8}{est.search_calls:>10}" + f"{est.delete_calls:>10}{'$' + format(est.estimated_usd, '.4f'):>14}" + ) + print(f"{'TOTAL':<42}{'$' + format(total, '.4f'):>14}") + return total + + +def main(argv: Sequence[str] | None = None) -> int: + args = _build_parser().parse_args(argv) + if args.version: + print(_version_string()) + return 0 + if args.live and args.dry_run: + sys.stderr.write("openmem-eval: --dry-run and --live are mutually exclusive\n") + return 2 + providers = tuple(p.strip() for p in args.providers.split(",") if p.strip()) + if not providers: + sys.stderr.write("openmem-eval: no providers requested\n") + return 1 + + cfg = RunConfig( + providers=providers, + live=args.live, + report_path=args.report, + trace_path=args.trace, + sample=args.sample, + cost_threshold_usd=args.cost_threshold, + yes=args.yes, + cleanup=args.cleanup, + verbose=args.verbose, + ) + + try: + dataset = load_path(args.dataset) if args.dataset else load_default() + except (FileNotFoundError, ValueError) as exc: + sys.stderr.write(f"openmem-eval: dataset error: {exc}\n") + return 4 + + if cfg.sample is not None: + dataset = sample(dataset, cfg.sample) + + # Cost preview / confirmation when going live. + if cfg.live: + if (rc := _refuse_in_ci()) is not None: + return rc + estimates = [ + cost_mod.estimate( + p, + add_calls=len(dataset.facts), + search_calls=len(dataset.queries), + delete_calls=len(dataset.facts) if cfg.cleanup else 0, + ) + for p in providers + ] + total = cost_mod.total_estimated(estimates) + try: + confirm_or_exit( + total, + threshold=cfg.cost_threshold_usd, + yes=cfg.yes, + isatty=sys.stdin.isatty(), + prompt=lambda msg: "y" if _confirm(msg) else "n", + ) + except SystemExit as exc: + return int(exc.code) if exc.code is not None else 3 + if cfg.verbose: + print(f"Estimated total cost: ${total:.4f}") + else: + # Dry-run: print the per-provider cost-estimate table to stdout. + _print_dry_run_table(providers, dataset, cfg.cleanup) + + try: + results = run_eval(cfg, dataset) + except Exception as exc: # pragma: no cover - safety net + sys.stderr.write(f"openmem-eval: internal error: {exc}\n") + return 4 + + write_report(cfg.report_path, cfg, results, dataset_hash=dataset.dataset_hash) + if cfg.verbose: + print(f"Report written to {cfg.report_path}") + + if cfg.cleanup and cfg.live: + from openmem.eval.runner import _default_factory + + for p in providers: + try: + mem = _default_factory(p) + deleted = run_cleanup(mem, user_id=cfg.user_id()) + if cfg.verbose: + print(f"cleanup[{p}]: deleted {deleted}") + except Exception as exc: # pragma: no cover + sys.stderr.write(f"openmem-eval: cleanup failed for {p}: {exc}\n") + + # Exit 1 only if every provider produced zero usable results. + if all(not r.query_results and r.status != "skipped" for r in results): + return 1 + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/sdk-python/openmem/eval/confirm.py b/sdk-python/openmem/eval/confirm.py new file mode 100644 index 0000000..658060d --- /dev/null +++ b/sdk-python/openmem/eval/confirm.py @@ -0,0 +1,52 @@ +"""Cost confirmation helper (T023). + +Single point of policy for: "should we ask the user before incurring this +estimated USD cost?". Wired into ``cli.main`` immediately before any +adapter is instantiated so refusal exits *before* any network call. +""" + +from __future__ import annotations + +import sys +from typing import Callable, Optional + + +def confirm_or_exit( + estimated_cost_usd: float, + *, + threshold: float, + yes: bool, + isatty: bool, + prompt: Optional[Callable[[str], str]] = None, +) -> None: + """Return silently when the run may proceed; raise SystemExit(3) otherwise. + + Rules: + * cost ≤ threshold → proceed + * ``yes`` flag set → proceed + * cost > threshold and TTY → ask once; non-y/Y → exit 3 + * cost > threshold and no TTY → exit 3 immediately (cannot prompt safely) + """ + if estimated_cost_usd <= threshold or yes: + return + if not isatty: + sys.stderr.write( + f"openmem-eval: estimated ${estimated_cost_usd:.4f} exceeds threshold " + f"${threshold:.2f}; pass --yes to proceed non-interactively\n" + ) + raise SystemExit(3) + ask = prompt or input + msg = ( + f"Estimated cost ${estimated_cost_usd:.4f} exceeds threshold " + f"${threshold:.2f}. Proceed? [y/N]: " + ) + try: + ans = ask(msg).strip().lower() + except EOFError: + ans = "" + if ans not in {"y", "yes"}: + sys.stderr.write("openmem-eval: aborted by user\n") + raise SystemExit(3) + + +__all__ = ["confirm_or_exit"] diff --git a/sdk-python/openmem/eval/cost.py b/sdk-python/openmem/eval/cost.py new file mode 100644 index 0000000..00608c4 --- /dev/null +++ b/sdk-python/openmem/eval/cost.py @@ -0,0 +1,64 @@ +"""Cost estimation for live runs. + +Per FR-006 and SC-007: a full live run across all four providers must +remain ≤ $0.50 with the bundled 50/20 dataset. Numbers below are +deliberately *upper-bound* estimates expressed in USD per call so the +dry-run preview is conservative. + +postgres is treated as zero-cost (self-hosted). +""" + +from __future__ import annotations + +from dataclasses import dataclass + +# USD per verb call. Conservative upper bounds informed by published +# pricing pages as of 2025-01. These are not invoiced; they only feed +# the dry-run cost ceiling. +_COSTS: dict[str, dict[str, float]] = { + "postgres": {"add": 0.0, "search": 0.0, "get": 0.0, "delete": 0.0}, + "mem0": {"add": 0.0010, "search": 0.0010, "get": 0.0001, "delete": 0.0001}, + "supermemory": {"add": 0.0008, "search": 0.0008, "get": 0.0001, "delete": 0.0001}, + "letta": {"add": 0.0015, "search": 0.0015, "get": 0.0001, "delete": 0.0001}, +} + + +@dataclass(frozen=True) +class CostEstimate: + provider: str + add_calls: int + search_calls: int + delete_calls: int + estimated_usd: float + + +def estimate( + provider: str, + *, + add_calls: int, + search_calls: int, + delete_calls: int = 0, + get_calls: int = 0, +) -> CostEstimate: + """Return a CostEstimate for `provider`. Unknown providers cost $0.""" + table = _COSTS.get(provider, {"add": 0.0, "search": 0.0, "get": 0.0, "delete": 0.0}) + total = ( + table["add"] * add_calls + + table["search"] * search_calls + + table["get"] * get_calls + + table["delete"] * delete_calls + ) + return CostEstimate( + provider=provider, + add_calls=add_calls, + search_calls=search_calls, + delete_calls=delete_calls, + estimated_usd=round(total, 4), + ) + + +def total_estimated(estimates: list[CostEstimate]) -> float: + return round(sum(e.estimated_usd for e in estimates), 4) + + +__all__ = ["CostEstimate", "estimate", "total_estimated"] diff --git a/sdk-python/openmem/eval/dataset.py b/sdk-python/openmem/eval/dataset.py new file mode 100644 index 0000000..0941a0b --- /dev/null +++ b/sdk-python/openmem/eval/dataset.py @@ -0,0 +1,125 @@ +"""Dataset loader and hashing for the eval kit. + +Loads bundled or user-supplied JSONL fixtures and computes a stable +``dataset_hash`` (SHA-256, first 12 hex chars) over canonical line content. +""" + +from __future__ import annotations + +import hashlib +import json +from importlib import resources +from pathlib import Path +from typing import Iterable + +from openmem.eval.types import Dataset, Fact, Query + + +def _read_jsonl(text: str) -> list[dict]: + out: list[dict] = [] + for lineno, raw in enumerate(text.splitlines(), start=1): + stripped = raw.strip() + if not stripped: + continue + try: + out.append(json.loads(stripped)) + except json.JSONDecodeError as exc: + raise ValueError(f"malformed JSONL on line {lineno}: {exc}") from exc + return out + + +def _validate(facts: list[Fact], queries: list[Query]) -> None: + seen: set[str] = set() + for f in facts: + if not f.fact_id: + raise ValueError("fact_id must be non-empty") + if not f.content: + raise ValueError(f"content must be non-empty for {f.fact_id}") + if f.fact_id in seen: + raise ValueError(f"duplicate fact_id {f.fact_id!r}") + seen.add(f.fact_id) + for q in queries: + if not q.gold_fact_ids: + raise ValueError(f"query {q.query_id!r} has empty gold_fact_ids") + for gid in q.gold_fact_ids: + if gid not in seen: + raise ValueError( + f"query {q.query_id!r} references unknown fact_id {gid!r}" + ) + + +def _hash(facts_text: str, queries_text: str) -> str: + """SHA-256 over canonical line-sorted content; first 12 hex chars.""" + canonical = "\n".join( + sorted(line.strip() for line in (facts_text + "\n" + queries_text).splitlines() if line.strip()) + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:12] + + +def _build(facts_text: str, queries_text: str) -> Dataset: + facts = tuple( + Fact(fact_id=r["fact_id"], content=r["content"], tags=tuple(r.get("tags") or ())) + for r in _read_jsonl(facts_text) + ) + queries = tuple( + Query( + query_id=r["query_id"], + query=r["query"], + gold_fact_ids=tuple(r["gold_fact_ids"]), + ) + for r in _read_jsonl(queries_text) + ) + _validate(list(facts), list(queries)) + return Dataset( + facts=facts, + queries=queries, + dataset_hash=_hash(facts_text, queries_text), + ) + + +def load_default() -> Dataset: + """Load the bundled `default` dataset.""" + pkg = resources.files("openmem.eval.datasets.default") + facts_text = (pkg / "facts.jsonl").read_text(encoding="utf-8") + queries_text = (pkg / "queries.jsonl").read_text(encoding="utf-8") + return _build(facts_text, queries_text) + + +def load_path(path: Path) -> Dataset: + """Load a user-supplied dataset from a directory containing + ``facts.jsonl`` and ``queries.jsonl``.""" + base = Path(path) + if not base.exists(): + raise FileNotFoundError(f"dataset path does not exist: {base}") + facts_path = base / "facts.jsonl" + queries_path = base / "queries.jsonl" + if not facts_path.exists() or not queries_path.exists(): + raise FileNotFoundError( + f"dataset directory must contain facts.jsonl and queries.jsonl: {base}" + ) + return _build( + facts_path.read_text(encoding="utf-8"), + queries_path.read_text(encoding="utf-8"), + ) + + +def sample(dataset: Dataset, n: int) -> Dataset: + """Return a new Dataset with the first `n` queries by stable hash order.""" + if n <= 0: + raise ValueError(f"sample n must be > 0, got {n}") + if n >= len(dataset.queries): + return dataset + ordered = sorted( + dataset.queries, + key=lambda q: hashlib.sha256(q.query_id.encode()).hexdigest(), + ) + return Dataset( + facts=dataset.facts, + queries=tuple(ordered[:n]), + dataset_hash=dataset.dataset_hash, # facts unchanged → hash stable + ) + + +def dataset_hash(dataset: Dataset) -> str: + """Accessor for symmetry; the hash is precomputed on load.""" + return dataset.dataset_hash diff --git a/sdk-python/openmem/eval/datasets/__init__.py b/sdk-python/openmem/eval/datasets/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sdk-python/openmem/eval/datasets/default/__init__.py b/sdk-python/openmem/eval/datasets/default/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sdk-python/openmem/eval/datasets/default/facts.jsonl b/sdk-python/openmem/eval/datasets/default/facts.jsonl new file mode 100644 index 0000000..be87359 --- /dev/null +++ b/sdk-python/openmem/eval/datasets/default/facts.jsonl @@ -0,0 +1,50 @@ +{"fact_id": "f-0001", "content": "Alice's pizza dough recipe uses 500g flour, 325g water, 10g salt, 1g instant yeast and rests 24 hours.", "tags": ["recipe", "pizza"]} +{"fact_id": "f-0002", "content": "The user prefers Earl Grey tea brewed at 95C for exactly 4 minutes.", "tags": ["preference", "tea"]} +{"fact_id": "f-0003", "content": "Bob's home address is 482 Maple Street, Apartment 7B, Portland OR 97201.", "tags": ["contact", "address"]} +{"fact_id": "f-0004", "content": "The Wi-Fi password for the office router is BlueOctopus42!.", "tags": ["credential"]} +{"fact_id": "f-0005", "content": "Quarterly board meeting is scheduled for the first Thursday of every quarter at 9am Pacific.", "tags": ["schedule", "meeting"]} +{"fact_id": "f-0006", "content": "Carol's car is a 2019 Subaru Outback, plate ABC-1234, registered in Oregon.", "tags": ["vehicle"]} +{"fact_id": "f-0007", "content": "Daniel is allergic to peanuts and tree nuts; he carries an EpiPen at all times.", "tags": ["medical", "allergy"]} +{"fact_id": "f-0008", "content": "The kitchen renovation budget is capped at $42,500 including labor and materials.", "tags": ["budget", "renovation"]} +{"fact_id": "f-0009", "content": "Project Aurora deadline is 2026-09-15 and the lead engineer is Eve Patel.", "tags": ["project", "deadline"]} +{"fact_id": "f-0010", "content": "The bedroom thermostat should be set to 68F at night and 71F during the day.", "tags": ["preference", "temperature"]} +{"fact_id": "f-0011", "content": "Frank's favorite hiking trail is the Devils Lake Loop near Bend, Oregon, 8.4 miles round trip.", "tags": ["hike", "preference"]} +{"fact_id": "f-0012", "content": "The dog Buddy needs his arthritis medication twice daily at 7am and 7pm.", "tags": ["pet", "medication"]} +{"fact_id": "f-0013", "content": "Grace's birthday is March 14th and she prefers vanilla cake with raspberry filling.", "tags": ["birthday", "preference"]} +{"fact_id": "f-0014", "content": "The garage door opener code is 7259 and the secondary keypad code is 4081.", "tags": ["credential"]} +{"fact_id": "f-0015", "content": "Henry plays bass guitar in a jazz quartet that rehearses Wednesdays at 6pm.", "tags": ["hobby", "schedule"]} +{"fact_id": "f-0016", "content": "The escrow officer for the new house purchase is Linda Wu at First American Title.", "tags": ["real-estate"]} +{"fact_id": "f-0017", "content": "Iris uses an iPhone 15 Pro on Verizon, phone number 503-555-0142.", "tags": ["contact", "device"]} +{"fact_id": "f-0018", "content": "The annual ski trip is to Mount Bachelor in early February, group size of 8 people.", "tags": ["trip", "schedule"]} +{"fact_id": "f-0019", "content": "Jacob's preferred deadlift form is sumo stance with belt at 220 kg max.", "tags": ["fitness", "preference"]} +{"fact_id": "f-0020", "content": "The HVAC filter is size 20x25x4 and should be replaced every 90 days.", "tags": ["maintenance"]} +{"fact_id": "f-0021", "content": "Kara's wedding anniversary is June 22nd; she likes white peonies and dark chocolate.", "tags": ["anniversary", "preference"]} +{"fact_id": "f-0022", "content": "The car insurance policy number is GEI-87432109 with State Farm, renews annually in April.", "tags": ["insurance"]} +{"fact_id": "f-0023", "content": "Liam's son Theo attends Maplewood Elementary, second grade, teacher Ms. Ramirez.", "tags": ["family", "school"]} +{"fact_id": "f-0024", "content": "The wine cellar is set to 55F at 60 percent humidity; current bottle count is 312.", "tags": ["wine"]} +{"fact_id": "f-0025", "content": "Maya is vegetarian, loves Thai green curry, and avoids cilantro because it tastes like soap.", "tags": ["preference", "diet"]} +{"fact_id": "f-0026", "content": "The startup raised a $2.4M seed round on 2025-11-08 led by Foundry Capital.", "tags": ["startup", "funding"]} +{"fact_id": "f-0027", "content": "Noah's marathon PR is 3:14:22 set at the Berlin Marathon in September 2024.", "tags": ["fitness", "running"]} +{"fact_id": "f-0028", "content": "The home alarm code is 5298 and the panic word for emergencies is silver-falcon.", "tags": ["credential", "security"]} +{"fact_id": "f-0029", "content": "Olivia's PhD defense is on November 5th at 2pm in the Stanford Gates Building room 104.", "tags": ["academic", "schedule"]} +{"fact_id": "f-0030", "content": "The attic insulation is R-49 fiberglass batts installed 2023, manufacturer Owens Corning.", "tags": ["home"]} +{"fact_id": "f-0031", "content": "Pete prefers Spotify over Apple Music and listens mostly to ambient electronic and bossa nova.", "tags": ["preference", "music"]} +{"fact_id": "f-0032", "content": "The dental insurance is Delta Dental PPO, member ID 88421, covers two cleanings per year.", "tags": ["insurance", "dental"]} +{"fact_id": "f-0033", "content": "Quinn's preferred coffee order is a flat white with oat milk, no sugar.", "tags": ["preference", "coffee"]} +{"fact_id": "f-0034", "content": "The book club meets the last Sunday of the month at 4pm; current book is Project Hail Mary.", "tags": ["schedule", "book-club"]} +{"fact_id": "f-0035", "content": "Rachel's daughter Emma is 14 and plays varsity volleyball as a setter.", "tags": ["family"]} +{"fact_id": "f-0036", "content": "The mortgage is a 30-year fixed at 6.125 percent with Wells Fargo, monthly payment $3,842.", "tags": ["mortgage"]} +{"fact_id": "f-0037", "content": "Sam is fluent in Spanish and Mandarin and is currently learning Korean on Duolingo.", "tags": ["language"]} +{"fact_id": "f-0038", "content": "The reflux medication is omeprazole 20mg, taken every morning before breakfast.", "tags": ["medication"]} +{"fact_id": "f-0039", "content": "Tess's favorite restaurant is Le Pigeon in Portland; she always orders the foie gras profiteroles.", "tags": ["preference", "restaurant"]} +{"fact_id": "f-0040", "content": "The garden has 14 raised beds; tomatoes go in beds 3 through 7 and basil in bed 8.", "tags": ["garden"]} +{"fact_id": "f-0041", "content": "Uma's preferred therapist is Dr. Patel at the Northwest Counseling Center, sessions Tuesday at 5pm.", "tags": ["health", "schedule"]} +{"fact_id": "f-0042", "content": "The wedding venue deposit of $4,200 is non-refundable and was paid on 2026-01-15.", "tags": ["wedding", "budget"]} +{"fact_id": "f-0043", "content": "Victor's son Max takes piano lessons Saturdays at 10am with Ms. Chen at the Conservatory.", "tags": ["family", "schedule"]} +{"fact_id": "f-0044", "content": "The kombucha SCOBY needs feeding with 8 cups of sweet tea every 7 to 10 days.", "tags": ["kitchen"]} +{"fact_id": "f-0045", "content": "Wendy's marathon training plan is the Hanson method with peak weekly mileage of 60.", "tags": ["fitness"]} +{"fact_id": "f-0046", "content": "The travel agent for the Japan trip is Hiroshi Tanaka at Pacific Routes, fluent in English.", "tags": ["travel"]} +{"fact_id": "f-0047", "content": "Xavier prefers manual transmission cars and currently drives a 2018 Mazda Miata RF.", "tags": ["preference", "vehicle"]} +{"fact_id": "f-0048", "content": "The pool chemistry should run pH 7.4 with chlorine at 2.5 ppm and CYA below 50.", "tags": ["pool"]} +{"fact_id": "f-0049", "content": "Yara's favorite hike is the West Coast Trail in BC, 75 km over 5 to 7 days.", "tags": ["hike", "preference"]} +{"fact_id": "f-0050", "content": "The streaming subscriptions are Netflix Premium, HBO Max, and Apple TV+, billed annually.", "tags": ["subscription"]} diff --git a/sdk-python/openmem/eval/datasets/default/queries.jsonl b/sdk-python/openmem/eval/datasets/default/queries.jsonl new file mode 100644 index 0000000..cb0a893 --- /dev/null +++ b/sdk-python/openmem/eval/datasets/default/queries.jsonl @@ -0,0 +1,20 @@ +{"query_id": "q-0001", "query": "What is the recipe for Alice's pizza dough?", "gold_fact_ids": ["f-0001"]} +{"query_id": "q-0002", "query": "How does the user like their tea?", "gold_fact_ids": ["f-0002"]} +{"query_id": "q-0003", "query": "What is Bob's home address?", "gold_fact_ids": ["f-0003"]} +{"query_id": "q-0004", "query": "What is the office Wi-Fi password?", "gold_fact_ids": ["f-0004"]} +{"query_id": "q-0005", "query": "When is the quarterly board meeting?", "gold_fact_ids": ["f-0005"]} +{"query_id": "q-0006", "query": "Tell me about Carol's car including license plate.", "gold_fact_ids": ["f-0006"]} +{"query_id": "q-0007", "query": "What food allergies does Daniel have?", "gold_fact_ids": ["f-0007"]} +{"query_id": "q-0008", "query": "How much money is set aside for the kitchen renovation?", "gold_fact_ids": ["f-0008"]} +{"query_id": "q-0009", "query": "Who leads Project Aurora and when is the deadline?", "gold_fact_ids": ["f-0009"]} +{"query_id": "q-0010", "query": "What temperature should the bedroom be at night?", "gold_fact_ids": ["f-0010"]} +{"query_id": "q-0011", "query": "Where does Frank like to go hiking?", "gold_fact_ids": ["f-0011"]} +{"query_id": "q-0012", "query": "When does the dog need his medicine?", "gold_fact_ids": ["f-0012"]} +{"query_id": "q-0013", "query": "What kind of birthday cake does Grace prefer?", "gold_fact_ids": ["f-0013"]} +{"query_id": "q-0014", "query": "What are the garage door codes?", "gold_fact_ids": ["f-0014"]} +{"query_id": "q-0015", "query": "When does Henry's jazz quartet rehearse?", "gold_fact_ids": ["f-0015"]} +{"query_id": "q-0016", "query": "What dietary restrictions does Maya have and what does she like to eat?", "gold_fact_ids": ["f-0025"]} +{"query_id": "q-0017", "query": "What is Quinn's coffee order?", "gold_fact_ids": ["f-0033"]} +{"query_id": "q-0018", "query": "What languages does Sam speak?", "gold_fact_ids": ["f-0037"]} +{"query_id": "q-0019", "query": "When and where is Olivia's PhD defense?", "gold_fact_ids": ["f-0029"]} +{"query_id": "q-0020", "query": "What car does Xavier drive and what kind of transmission does he prefer?", "gold_fact_ids": ["f-0047"]} diff --git a/sdk-python/openmem/eval/report.py b/sdk-python/openmem/eval/report.py new file mode 100644 index 0000000..e7c0bbb --- /dev/null +++ b/sdk-python/openmem/eval/report.py @@ -0,0 +1,92 @@ +"""Markdown report writer.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path +from typing import Iterable + +from openmem.eval.types import ProviderResult, RunConfig + + +def _fmt_money(usd: float) -> str: + return f"${usd:.4f}" + + +def _fmt_ms(ms: float) -> str: + return f"{ms:.1f}" + + +def write_report( + path: Path, + cfg: RunConfig, + provider_results: Iterable[ProviderResult], + *, + dataset_hash: str, +) -> None: + """Write a Markdown summary to `path`.""" + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + provider_results = list(provider_results) + + mode = "live" if cfg.live else "dry-run" + lines: list[str] = [] + lines.append("# OMP Eval Report") + lines.append("") + lines.append(f"- **Run ID**: `{cfg.run_id}`") + lines.append(f"- **Mode**: {mode}") + lines.append(f"- **Dataset hash**: `{dataset_hash}`") + lines.append(f"- **Generated**: {datetime.now(timezone.utc).isoformat()}") + lines.append(f"- **Providers**: {', '.join(cfg.providers)}") + if cfg.sample is not None: + lines.append(f"- **Sample**: first {cfg.sample} queries (`sample={cfg.sample}`)") + lines.append("") + + lines.append("## Results") + lines.append("") + lines.append( + "| Provider | Status | recall@1 | recall@5 | MRR | " + "ingest p50/p95/p99 (ms) | search p50/p95/p99 (ms) | errors | est. cost |" + ) + lines.append( + "|---|---|---:|---:|---:|---|---|---:|---:|" + ) + for pr in provider_results: + m = pr.metrics + lines.append( + "| {prov} | {status} | {r1:.3f} | {r5:.3f} | {mrr:.3f} | " + "{ip50}/{ip95}/{ip99} | {sp50}/{sp95}/{sp99} | {errs} | {cost} |".format( + prov=pr.provider, + status=pr.status, + r1=m.recall_at_1, + r5=m.recall_at_5, + mrr=m.mrr, + ip50=_fmt_ms(m.ingest_p50_ms), + ip95=_fmt_ms(m.ingest_p95_ms), + ip99=_fmt_ms(m.ingest_p99_ms), + sp50=_fmt_ms(m.search_p50_ms), + sp95=_fmt_ms(m.search_p95_ms), + sp99=_fmt_ms(m.search_p99_ms), + errs=m.error_count, + cost=_fmt_money(pr.estimated_cost_usd), + ) + ) + + # Notes section for any provider with metrics.note set + notes = [(pr.provider, pr.metrics.note) for pr in provider_results if pr.metrics.note] + # Re-iterate provider_results above consumed if it's a generator → coerce + # Caller passes a list per the contract; if not, the loop above already + # exhausted it and notes would be empty. Rebuild defensively: + # (no-op when caller already supplied a list) + if notes: + lines.append("") + lines.append("## Notes") + lines.append("") + for prov, note in notes: + lines.append(f"- **{prov}**: {note}") + + lines.append("") + path.write_text("\n".join(lines), encoding="utf-8") + + +__all__ = ["write_report"] diff --git a/sdk-python/openmem/eval/runner.py b/sdk-python/openmem/eval/runner.py new file mode 100644 index 0000000..3a9a05f --- /dev/null +++ b/sdk-python/openmem/eval/runner.py @@ -0,0 +1,233 @@ +"""Runner — orchestrates ingest + search across providers. + +Per R11 every fact is dual-stamped: +* content prefix ``[fact_id=f-NNNN] `` (regex-recoverable) +* tag ``fact:f-NNNN`` (fallback when an adapter rewrites content) +""" + +from __future__ import annotations + +import re +import time +from typing import Any, Callable, Optional, Sequence + +from openmem.eval import cost as cost_mod +from openmem.eval.scorer import compute_metrics +from openmem.eval.types import ( + Dataset, + ErrorRecord, + Metrics, + ProviderResult, + QueryResult, + RunConfig, +) + +_FACT_PREFIX_RE = re.compile(r"^\[fact_id=([^\]]+)\] ") + + +def _stamp(fact_id: str, content: str) -> str: + return f"[fact_id={fact_id}] {content}" + + +def _recover_fact_id(*, content: str | None, tags: Sequence[str] | None) -> str | None: + if content: + m = _FACT_PREFIX_RE.match(content) + if m: + return m.group(1) + if tags: + for t in tags: + if t.startswith("fact:"): + return t[5:] + return None + + +def _default_factory(provider: str, **kwargs: Any): # pragma: no cover + """Default factory: build a real openmem.Memory. + + Resolves provider-specific connection settings from environment + variables so the eval CLI works against the standard local postgres + container (`OMP_POSTGRES_URL`) and provider-key envs. + """ + import os + from openmem.memory import Memory + + if provider == "postgres": + if "embedder" not in kwargs: + from openmem.adapters.embedder import FakeEmbedder + kwargs["embedder"] = FakeEmbedder() + url = os.environ.get("OMP_POSTGRES_URL") or os.environ.get("PG_URL") + if url: + kwargs.setdefault("url", url) + elif provider == "mem0": + key = os.environ.get("MEM0_API_KEY") + if key: + kwargs.setdefault("api_key", key) + elif provider == "supermemory": + key = os.environ.get("SUPERMEMORY_API_KEY") + if key: + kwargs.setdefault("api_key", key) + elif provider == "letta": + key = os.environ.get("LETTA_API_KEY") + if key: + kwargs.setdefault("api_key", key) + return Memory(provider=provider, **kwargs) + + +def _build_dry_run(cfg: RunConfig, dataset: Dataset, providers: Sequence[str]) -> list[ProviderResult]: + results: list[ProviderResult] = [] + for p in providers: + est = cost_mod.estimate( + p, + add_calls=len(dataset.facts), + search_calls=len(dataset.queries), + delete_calls=len(dataset.facts) if cfg.cleanup else 0, + ) + pr = ProviderResult( + provider=p, + status="skipped", + skip_reason="dry-run", + estimated_cost_usd=est.estimated_usd, + metrics=Metrics(), + ) + results.append(pr) + return results + + +def run( + cfg: RunConfig, + dataset: Dataset, + *, + memory_factory: Optional[Callable[[str], Any]] = None, +) -> list[ProviderResult]: + """Execute the eval run and return per-provider results.""" + factory = memory_factory or _default_factory + + if cfg.dry_run: + return _build_dry_run(cfg, dataset, cfg.providers) + + results: list[ProviderResult] = [] + gold = {q.query_id: q.gold_fact_ids for q in dataset.queries} + + for provider in cfg.providers: + pr = ProviderResult(provider=provider) + run_started = time.perf_counter() + try: + mem = factory(provider) + except Exception as exc: # pragma: no cover - defensive + pr.status = "failed" + pr.errors.append( + ErrorRecord( + verb="init", + error_class=type(exc).__name__, + message=str(exc), + ts=_iso_now(), + ) + ) + pr.metrics = Metrics(error_count=1) + results.append(pr) + continue + + # ---- ingest ---- + added_ids: list[str] = [] + for fact in dataset.facts: + t0 = time.perf_counter() + try: + rec = mem.add( + content=_stamp(fact.fact_id, fact.content), + user_id=cfg.user_id(), + tags=[*fact.tags, f"fact:{fact.fact_id}"], + ) + rid = getattr(rec, "id", None) + if rid: + added_ids.append(rid) + except Exception as exc: + pr.errors.append( + ErrorRecord( + verb="add", + error_class=type(exc).__name__, + message=str(exc), + ts=_iso_now(), + fact_id=fact.fact_id, + ) + ) + finally: + pr.ingest_latencies_ms.append((time.perf_counter() - t0) * 1000) + + # ---- wait for ingest (no-op for sync adapters; polling for async) ---- + wait = getattr(mem, "wait_for_ingest", None) + if callable(wait) and added_ids: + try: + wait(added_ids, cfg.user_id(), timeout=30.0) + except Exception: # pragma: no cover - defensive + pass + + # ---- search ---- + for query in dataset.queries: + t0 = time.perf_counter() + top_ids: list[str] = [] + err: Optional[str] = None + try: + hits = mem.search(query.query, cfg.user_id(), limit=cfg.top_k) + for h in hits: + fid = _recover_fact_id( + content=getattr(h.memory, "content", None), + tags=getattr(h.memory, "tags", None), + ) + if fid: + top_ids.append(fid) + except Exception as exc: + err = f"{type(exc).__name__}: {exc}" + pr.errors.append( + ErrorRecord( + verb="search", + error_class=type(exc).__name__, + message=str(exc), + ts=_iso_now(), + query_id=query.query_id, + ) + ) + latency_ms = (time.perf_counter() - t0) * 1000 + pr.search_latencies_ms.append(latency_ms) + pr.query_results.append( + QueryResult( + query_id=query.query_id, + top_k_fact_ids=top_ids, + latency_ms=latency_ms, + error=err, + ) + ) + + pr.total_wall_s = round(time.perf_counter() - run_started, 3) + pr.metrics = Metrics(error_count=len(pr.errors)) + pr.metrics = compute_metrics(pr, gold) + # carry over error_count after compute_metrics overwrites + pr.metrics.error_count = len(pr.errors) + + # cost accounting (only matters when --live and provider is paid) + est = cost_mod.estimate( + provider, + add_calls=len(dataset.facts), + search_calls=len(dataset.queries), + delete_calls=len(dataset.facts) if cfg.cleanup else 0, + ) + pr.estimated_cost_usd = est.estimated_usd + + # status roll-up + if pr.errors and pr.query_results and any(qr.top_k_fact_ids for qr in pr.query_results): + pr.status = "partial" + elif pr.errors and not any(qr.top_k_fact_ids for qr in pr.query_results): + pr.status = "failed" + else: + pr.status = "ok" + + results.append(pr) + + return results + + +def _iso_now() -> str: + from datetime import datetime, timezone + return datetime.now(timezone.utc).isoformat() + + +__all__ = ["run", "_FACT_PREFIX_RE", "_stamp", "_recover_fact_id"] diff --git a/sdk-python/openmem/eval/scorer.py b/sdk-python/openmem/eval/scorer.py new file mode 100644 index 0000000..6d39637 --- /dev/null +++ b/sdk-python/openmem/eval/scorer.py @@ -0,0 +1,96 @@ +"""Metric calculations for the eval kit. + +Implements per-query recall@k and reciprocal rank, then macro-averages +across queries. Latency percentiles use ``statistics.quantiles(n=100)`` +so we get p50/p95/p99 with no third-party dependency. +""" + +from __future__ import annotations + +import statistics +from typing import Iterable, Mapping, Sequence + +from openmem.eval.types import Metrics, ProviderResult + +_SUSPICIOUS_RECALL_AT_5 = 0.10 + + +def recall_at_k( + top_k_fact_ids: Sequence[str], + gold_fact_ids: Sequence[str], + *, + k: int, +) -> float: + if not gold_fact_ids: + return 0.0 + head = list(top_k_fact_ids)[:k] + hits = sum(1 for g in gold_fact_ids if g in head) + return hits / len(gold_fact_ids) + + +def mrr(top_k_fact_ids: Sequence[str], gold_fact_ids: Sequence[str]) -> float: + gold = set(gold_fact_ids) + for rank, fid in enumerate(top_k_fact_ids, start=1): + if fid in gold: + return 1.0 / rank + return 0.0 + + +def _percentile(values: Sequence[float], p: float) -> float: + """Return the `p`-th percentile (0-100). Empty → 0.0; single value → itself.""" + if not values: + return 0.0 + if len(values) == 1: + return float(values[0]) + quantiles = statistics.quantiles(values, n=100, method="inclusive") + # quantiles returns 99 cut points: index i corresponds to (i+1)th percentile + idx = max(0, min(len(quantiles) - 1, int(round(p)) - 1)) + return float(quantiles[idx]) + + +def compute_metrics( + pr: ProviderResult, + gold: Mapping[str, Sequence[str]], +) -> Metrics: + """Compute aggregate Metrics for a ProviderResult.""" + recall_1: list[float] = [] + recall_5: list[float] = [] + rrs: list[float] = [] + for qr in pr.query_results: + gold_ids = gold.get(qr.query_id, ()) + recall_1.append(recall_at_k(qr.top_k_fact_ids, gold_ids, k=1)) + recall_5.append(recall_at_k(qr.top_k_fact_ids, gold_ids, k=5)) + rrs.append(mrr(qr.top_k_fact_ids, gold_ids)) + + r5 = _mean(recall_5) + note: str | None = None + if pr.query_results and r5 < _SUSPICIOUS_RECALL_AT_5: + note = ( + f"suspicious recall@5={r5:.3f} below {_SUSPICIOUS_RECALL_AT_5:.2f}; " + "check ingest configuration" + ) + + metrics = Metrics( + recall_at_1=_mean(recall_1), + recall_at_5=r5, + mrr=_mean(rrs), + ingest_p50_ms=_percentile(pr.ingest_latencies_ms, 50), + ingest_p95_ms=_percentile(pr.ingest_latencies_ms, 95), + ingest_p99_ms=_percentile(pr.ingest_latencies_ms, 99), + search_p50_ms=_percentile(pr.search_latencies_ms, 50), + search_p95_ms=_percentile(pr.search_latencies_ms, 95), + search_p99_ms=_percentile(pr.search_latencies_ms, 99), + error_count=pr.metrics.error_count, + note=note, + ) + return metrics + + +def _mean(values: Iterable[float]) -> float: + vals = list(values) + if not vals: + return 0.0 + return sum(vals) / len(vals) + + +__all__ = ["recall_at_k", "mrr", "compute_metrics"] diff --git a/sdk-python/openmem/eval/trace.py b/sdk-python/openmem/eval/trace.py new file mode 100644 index 0000000..15156a3 --- /dev/null +++ b/sdk-python/openmem/eval/trace.py @@ -0,0 +1,70 @@ +"""JSONL trace writer with hashed payloads (privacy-by-default). + +Each line is a single JSON object describing one verb call. Payload +content is replaced with a 12-hex SHA-256 prefix so traces are safe to +share without leaking memory bodies. +""" + +from __future__ import annotations + +import hashlib +import json +import time +from pathlib import Path +from typing import Any, IO + + +def _hash(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest()[:12] + + +class TraceWriter: + """Append-only JSONL writer. Use as a context manager.""" + + def __init__(self, path: Path) -> None: + self.path = Path(path) + self._fp: IO[str] | None = None + + def __enter__(self) -> "TraceWriter": + self.path.parent.mkdir(parents=True, exist_ok=True) + self._fp = self.path.open("w", encoding="utf-8") + return self + + def __exit__(self, *_exc: object) -> None: + if self._fp is not None: + self._fp.close() + self._fp = None + + def emit( + self, + *, + provider: str, + verb: str, + run_id: str, + latency_ms: float, + payload_text: str | None = None, + result_count: int | None = None, + error: str | None = None, + extra: dict[str, Any] | None = None, + ) -> None: + if self._fp is None: + raise RuntimeError("TraceWriter must be used inside a `with` block") + record: dict[str, Any] = { + "ts": time.time(), + "run_id": run_id, + "provider": provider, + "verb": verb, + "latency_ms": round(latency_ms, 3), + } + if payload_text is not None: + record["payload_hash"] = _hash(payload_text) + if result_count is not None: + record["result_count"] = result_count + if error is not None: + record["error"] = error + if extra: + record.update(extra) + self._fp.write(json.dumps(record, separators=(",", ":")) + "\n") + + +__all__ = ["TraceWriter"] diff --git a/sdk-python/openmem/eval/types.py b/sdk-python/openmem/eval/types.py new file mode 100644 index 0000000..a8abcdb --- /dev/null +++ b/sdk-python/openmem/eval/types.py @@ -0,0 +1,123 @@ +"""Internal data structures for the eval kit. + +Mirrors `specs/004-eval-kit/data-model.md`. None of these types are part of +the OMP protocol surface; they are harness-internal. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + + +# --------------------------------------------------------------------------- +# Dataset records +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Fact: + fact_id: str + content: str + tags: tuple[str, ...] = () + + +@dataclass(frozen=True) +class Query: + query_id: str + query: str + gold_fact_ids: tuple[str, ...] = () + + +@dataclass(frozen=True) +class Dataset: + facts: tuple[Fact, ...] + queries: tuple[Query, ...] + dataset_hash: str # 12-hex-char SHA-256 prefix + + +# --------------------------------------------------------------------------- +# Run configuration +# --------------------------------------------------------------------------- + + +@dataclass +class RunConfig: + providers: tuple[str, ...] = ("postgres",) + live: bool = False + report_path: Path = Path("eval-report.md") + trace_path: Path = Path("eval-trace.jsonl") + sample: Optional[int] = None + cost_threshold_usd: float = 1.00 + yes: bool = False + cleanup: bool = False + verbose: bool = False + top_k: int = 5 + run_id: str = "" # set in __post_init__ + + def __post_init__(self) -> None: + if not self.run_id: + import uuid + + object.__setattr__(self, "run_id", uuid.uuid4().hex[:12]) + + @property + def dry_run(self) -> bool: + return not self.live + + def user_id(self) -> str: + return f"eval-{self.run_id}" + + +# --------------------------------------------------------------------------- +# Per-query / per-error / per-provider records +# --------------------------------------------------------------------------- + + +@dataclass +class QueryResult: + query_id: str + top_k_fact_ids: list[str] = field(default_factory=list) + latency_ms: float = 0.0 + error: Optional[str] = None + + +@dataclass +class ErrorRecord: + verb: str + error_class: str + message: str + ts: str + query_id: Optional[str] = None + fact_id: Optional[str] = None + + +@dataclass +class Metrics: + recall_at_1: float = 0.0 + recall_at_5: float = 0.0 + recall_at_10: float = 0.0 + mrr: float = 0.0 + ingest_p50_ms: float = 0.0 + ingest_p95_ms: float = 0.0 + ingest_p99_ms: float = 0.0 + search_p50_ms: float = 0.0 + search_p95_ms: float = 0.0 + search_p99_ms: float = 0.0 + error_count: int = 0 + note: Optional[str] = None + + +@dataclass +class ProviderResult: + provider: str + status: str = "ok" # ok | skipped | partial | failed + skip_reason: Optional[str] = None + ingest_latencies_ms: list[float] = field(default_factory=list) + search_latencies_ms: list[float] = field(default_factory=list) + query_results: list[QueryResult] = field(default_factory=list) + errors: list[ErrorRecord] = field(default_factory=list) + metrics: Metrics = field(default_factory=Metrics) + total_wall_s: float = 0.0 + estimated_cost_usd: float = 0.0 diff --git a/sdk-python/openmem/memory.py b/sdk-python/openmem/memory.py index 7891d0d..913e31e 100644 --- a/sdk-python/openmem/memory.py +++ b/sdk-python/openmem/memory.py @@ -244,5 +244,22 @@ def capabilities(self) -> Capabilities: self._capabilities = self._adapter.capabilities() return self._capabilities + def wait_for_ingest( + self, + ids: list[str], + user_id: str, + *, + timeout: float | None = None, + ) -> None: + """Block until each id in ``ids`` is readable upstream. + + Pass-through to ``BaseAdapter.wait_for_ingest`` whose default is a + no-op for synchronous-ingestion adapters (e.g. postgres). Async + adapters (mem0, supermemory) override the method to poll. The eval + kit relies on this hook to make ingest observable across providers + without polluting the OMP protocol surface. + """ + self._adapter.wait_for_ingest(ids, user_id, timeout=timeout) + __all__ = ["Memory", "_resolve_adapter"] diff --git a/sdk-python/pyproject.toml b/sdk-python/pyproject.toml index bad74c8..21c8697 100644 --- a/sdk-python/pyproject.toml +++ b/sdk-python/pyproject.toml @@ -44,6 +44,7 @@ dev = [ [project.scripts] omp-validate-spec = "openmem._scripts:validate_spec" +openmem-eval = "openmem.eval.cli:main" [project.urls] Homepage = "https://github.com/openmem/omp" @@ -52,6 +53,9 @@ Specification = "https://github.com/openmem/omp/tree/main/spec" [tool.hatch.build.targets.wheel] packages = ["openmem"] +[tool.hatch.build.targets.wheel.force-include] +"openmem/eval/datasets" = "openmem/eval/datasets" + [tool.pytest.ini_options] testpaths = ["tests"] addopts = "--cov=openmem --cov-report=term-missing --cov-fail-under=85" diff --git a/sdk-python/tests/eval/__init__.py b/sdk-python/tests/eval/__init__.py new file mode 100644 index 0000000..ff379a0 --- /dev/null +++ b/sdk-python/tests/eval/__init__.py @@ -0,0 +1 @@ +"""Pytest fixtures for the eval test package.""" diff --git a/sdk-python/tests/eval/test_cleanup.py b/sdk-python/tests/eval/test_cleanup.py new file mode 100644 index 0000000..bc2f8ea --- /dev/null +++ b/sdk-python/tests/eval/test_cleanup.py @@ -0,0 +1,67 @@ +"""T018a — cleanup deletes every memory written under the run user_id.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from openmem.eval.cleanup import cleanup + + +@dataclass +class _Item: + id: str + + +@dataclass +class _Page: + items: list[_Item] + next_cursor: str | None = None + + +class _Stub: + def __init__(self, ids: list[str]) -> None: + self._remaining = list(ids) + self.deleted: list[str] = [] + + def list(self, user_id, *, limit=200, cursor=None): + # Single-page for simplicity + items = [_Item(i) for i in self._remaining] + self._remaining = [] + return _Page(items=items) + + def delete(self, mid: str) -> None: + self.deleted.append(mid) + + +def test_cleanup_deletes_all_listed_ids() -> None: + stub = _Stub(["m-1", "m-2", "m-3"]) + n = cleanup(stub, user_id="eval-abc") + assert n == 3 + assert stub.deleted == ["m-1", "m-2", "m-3"] + + +def test_cleanup_empty_returns_zero() -> None: + stub = _Stub([]) + assert cleanup(stub, user_id="eval-x") == 0 + + +def test_cleanup_paginates() -> None: + pages = [ + _Page(items=[_Item("a"), _Item("b")], next_cursor="c1"), + _Page(items=[_Item("c")], next_cursor=None), + ] + + class _Paged: + def __init__(self): + self.deleted = [] + self._iter = iter(pages) + + def list(self, *_a, **_k): + return next(self._iter) + + def delete(self, mid): + self.deleted.append(mid) + + s = _Paged() + assert cleanup(s, user_id="eval-x") == 3 + assert s.deleted == ["a", "b", "c"] diff --git a/sdk-python/tests/eval/test_cli.py b/sdk-python/tests/eval/test_cli.py new file mode 100644 index 0000000..98b5164 --- /dev/null +++ b/sdk-python/tests/eval/test_cli.py @@ -0,0 +1,175 @@ +"""T017 + I1 + FR-011 — CLI flag parsing, dry-run default, CI refusal.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from openmem.eval import cli as cli_mod + + +def test_dry_run_is_default_and_writes_report(tmp_path: Path, monkeypatch) -> None: + """FR-003 — without --live we render a dry-run preview, no provider calls.""" + monkeypatch.delenv("CI", raising=False) + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + report = tmp_path / "r.md" + rc = cli_mod.main( + [ + "--providers", "postgres,mem0", + "--report", str(report), + "--trace", str(tmp_path / "t.jsonl"), + ] + ) + assert rc == 0 + text = report.read_text(encoding="utf-8") + assert "dry-run" in text + assert "postgres" in text + assert "mem0" in text + + +def test_no_providers_returns_exit_1(monkeypatch, tmp_path: Path) -> None: + monkeypatch.delenv("CI", raising=False) + rc = cli_mod.main(["--providers", "", "--report", str(tmp_path / "r.md")]) + assert rc == 1 + + +def test_refuses_to_run_live_in_ci(monkeypatch, tmp_path: Path) -> None: + """FR-011 — CI=true with --live exits 4.""" + monkeypatch.setenv("CI", "true") + rc = cli_mod.main( + [ + "--providers", "postgres", + "--live", + "--report", str(tmp_path / "r.md"), + ] + ) + assert rc == 4 + + +def test_dataset_error_returns_exit_4(monkeypatch, tmp_path: Path) -> None: + monkeypatch.delenv("CI", raising=False) + rc = cli_mod.main( + [ + "--providers", "postgres", + "--report", str(tmp_path / "r.md"), + "--dataset", str(tmp_path / "does-not-exist"), + ] + ) + assert rc == 4 + + +def test_sample_flag_limits_queries(monkeypatch, tmp_path: Path) -> None: + monkeypatch.delenv("CI", raising=False) + report = tmp_path / "r.md" + rc = cli_mod.main( + [ + "--providers", "postgres", + "--sample", "3", + "--report", str(report), + ] + ) + assert rc == 0 + assert "Sample" in report.read_text(encoding="utf-8") + + +def test_cost_confirmation_refusal_returns_exit_3(monkeypatch, tmp_path: Path) -> None: + """FR-004 — exceeding threshold without --yes prompts; declining → exit 3.""" + monkeypatch.delenv("CI", raising=False) + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + monkeypatch.setattr(cli_mod, "_confirm", lambda _msg: False) + rc = cli_mod.main( + [ + "--providers", "mem0,letta", + "--live", + "--cost-threshold", "0.0001", + "--report", str(tmp_path / "r.md"), + ] + ) + assert rc == 3 + + +def test_cost_confirmation_yes_flag_skips_prompt(monkeypatch, tmp_path: Path) -> None: + """--yes bypasses the prompt; we still won't actually run a real provider + because we monkeypatch the runner to return an empty result list.""" + monkeypatch.delenv("CI", raising=False) + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + + captured = {} + + def fake_run(cfg, dataset, **_kw): + captured["live"] = cfg.live + from openmem.eval.types import Metrics, ProviderResult, QueryResult + return [ + ProviderResult( + provider=p, + status="ok", + metrics=Metrics(), + query_results=[QueryResult(query_id="q1", top_k_fact_ids=["x"])], + ) + for p in cfg.providers + ] + + monkeypatch.setattr(cli_mod, "run_eval", fake_run) + rc = cli_mod.main( + [ + "--providers", "mem0", + "--live", + "--yes", + "--cost-threshold", "0.0001", + "--report", str(tmp_path / "r.md"), + ] + ) + assert rc == 0 + assert captured["live"] is True + + +def test_help_exits_two() -> None: + """argparse exits 2 on bad invocation; --help exits 0.""" + with pytest.raises(SystemExit) as exc: + cli_mod.main(["--help"]) + assert exc.value.code == 0 + + +def test_version_flag_prints_and_exits_zero(capsys) -> None: + rc = cli_mod.main(["--version"]) + assert rc == 0 + out = capsys.readouterr().out + assert "openmem" in out + assert "dataset default-" in out + + +def test_dry_run_and_live_are_mutually_exclusive(monkeypatch, tmp_path) -> None: + monkeypatch.delenv("CI", raising=False) + rc = cli_mod.main( + ["--providers", "postgres", "--dry-run", "--live", "--report", str(tmp_path / "r.md")] + ) + assert rc == 2 + + +def test_dry_run_prints_per_provider_cost_table(monkeypatch, tmp_path, capsys) -> None: + monkeypatch.delenv("CI", raising=False) + rc = cli_mod.main( + [ + "--providers", "postgres,mem0", + "--report", str(tmp_path / "r.md"), + ] + ) + assert rc == 0 + out = capsys.readouterr().out + assert "DRY RUN" in out + assert "postgres" in out + assert "mem0" in out + assert "TOTAL" in out + + +def test_sample_annotation_in_report(monkeypatch, tmp_path) -> None: + monkeypatch.delenv("CI", raising=False) + report = tmp_path / "r.md" + rc = cli_mod.main( + ["--providers", "postgres", "--sample", "4", "--report", str(report)] + ) + assert rc == 0 + text = report.read_text(encoding="utf-8") + assert "sample=4" in text diff --git a/sdk-python/tests/eval/test_cli_confirm.py b/sdk-python/tests/eval/test_cli_confirm.py new file mode 100644 index 0000000..7422bb4 --- /dev/null +++ b/sdk-python/tests/eval/test_cli_confirm.py @@ -0,0 +1,45 @@ +"""T021 — cost confirmation helper.""" + +from __future__ import annotations + +import pytest + +from openmem.eval.confirm import confirm_or_exit + + +def test_proceeds_silently_below_threshold() -> None: + confirm_or_exit(0.10, threshold=1.00, yes=False, isatty=True) + confirm_or_exit(1.00, threshold=1.00, yes=False, isatty=False) + + +def test_yes_flag_bypasses_prompt_above_threshold() -> None: + confirm_or_exit(5.00, threshold=1.00, yes=True, isatty=False) + + +def test_above_threshold_no_tty_no_yes_exits_3() -> None: + with pytest.raises(SystemExit) as exc: + confirm_or_exit(5.00, threshold=1.00, yes=False, isatty=False) + assert exc.value.code == 3 + + +def test_above_threshold_tty_y_proceeds() -> None: + confirm_or_exit( + 5.00, threshold=1.00, yes=False, isatty=True, prompt=lambda _msg: "y" + ) + + +def test_above_threshold_tty_n_exits_3() -> None: + with pytest.raises(SystemExit) as exc: + confirm_or_exit( + 5.00, threshold=1.00, yes=False, isatty=True, prompt=lambda _msg: "n" + ) + assert exc.value.code == 3 + + +def test_above_threshold_tty_eof_exits_3() -> None: + def _eof(_msg: str) -> str: + raise EOFError + + with pytest.raises(SystemExit) as exc: + confirm_or_exit(5.00, threshold=1.00, yes=False, isatty=True, prompt=_eof) + assert exc.value.code == 3 diff --git a/sdk-python/tests/eval/test_cost.py b/sdk-python/tests/eval/test_cost.py new file mode 100644 index 0000000..b998cc0 --- /dev/null +++ b/sdk-python/tests/eval/test_cost.py @@ -0,0 +1,38 @@ +"""T013 + C3 — cost table covers add/search/get/delete and ceiling holds.""" + +from __future__ import annotations + +from openmem.eval.cost import estimate, total_estimated + + +def test_postgres_is_free_for_every_verb() -> None: + e = estimate("postgres", add_calls=50, search_calls=20, delete_calls=50, get_calls=10) + assert e.estimated_usd == 0.0 + + +def test_unknown_provider_is_free() -> None: + e = estimate("nope", add_calls=100, search_calls=100, delete_calls=100) + assert e.estimated_usd == 0.0 + + +def test_paid_providers_charge_for_each_verb() -> None: + for prov in ("mem0", "supermemory", "letta"): + e = estimate(prov, add_calls=10, search_calls=10, delete_calls=10, get_calls=10) + assert e.estimated_usd > 0 + + +def test_full_run_cost_ceiling_under_50_cents() -> None: + """SC-007 — 50 facts + 20 queries × 4 providers must stay ≤ $0.50.""" + estimates = [ + estimate(p, add_calls=50, search_calls=20) + for p in ("postgres", "mem0", "supermemory", "letta") + ] + assert total_estimated(estimates) <= 0.50 + + +def test_estimate_records_call_counts() -> None: + e = estimate("mem0", add_calls=3, search_calls=4, delete_calls=2) + assert e.add_calls == 3 + assert e.search_calls == 4 + assert e.delete_calls == 2 + assert e.provider == "mem0" diff --git a/sdk-python/tests/eval/test_dataset.py b/sdk-python/tests/eval/test_dataset.py new file mode 100644 index 0000000..24dc2a7 --- /dev/null +++ b/sdk-python/tests/eval/test_dataset.py @@ -0,0 +1,126 @@ +"""T007 + T025 — dataset loader, hashing, sample().""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from openmem.eval.dataset import ( + dataset_hash, + load_default, + load_path, + sample, +) + + +def test_load_default_returns_at_least_50_facts_and_20_queries() -> None: + ds = load_default() + assert len(ds.facts) >= 50 + assert len(ds.queries) >= 20 + + +def test_dataset_hash_is_deterministic_across_loads() -> None: + a = load_default() + b = load_default() + assert a.dataset_hash == b.dataset_hash + assert dataset_hash(a) == a.dataset_hash + + +def test_dataset_hash_changes_when_content_changes(tmp_path: Path) -> None: + base = tmp_path / "ds" + base.mkdir() + (base / "facts.jsonl").write_text( + '{"fact_id": "f-1", "content": "hello"}\n', encoding="utf-8" + ) + (base / "queries.jsonl").write_text( + '{"query_id": "q-1", "query": "hi", "gold_fact_ids": ["f-1"]}\n', + encoding="utf-8", + ) + h1 = load_path(base).dataset_hash + + (base / "facts.jsonl").write_text( + '{"fact_id": "f-1", "content": "goodbye"}\n', encoding="utf-8" + ) + h2 = load_path(base).dataset_hash + assert h1 != h2 + + +def test_unknown_gold_fact_id_raises_clear_error(tmp_path: Path) -> None: + base = tmp_path / "ds" + base.mkdir() + (base / "facts.jsonl").write_text( + '{"fact_id": "f-1", "content": "x"}\n', encoding="utf-8" + ) + (base / "queries.jsonl").write_text( + '{"query_id": "q-1", "query": "hi", "gold_fact_ids": ["does-not-exist"]}\n', + encoding="utf-8", + ) + with pytest.raises(ValueError, match="unknown fact_id"): + load_path(base) + + +def test_duplicate_fact_id_raises(tmp_path: Path) -> None: + base = tmp_path / "ds" + base.mkdir() + (base / "facts.jsonl").write_text( + '{"fact_id": "f-1", "content": "a"}\n{"fact_id": "f-1", "content": "b"}\n', + encoding="utf-8", + ) + (base / "queries.jsonl").write_text("", encoding="utf-8") + with pytest.raises(ValueError, match="duplicate fact_id"): + load_path(base) + + +def test_load_path_missing_directory_raises(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + load_path(tmp_path / "nope") + + +def test_load_path_missing_files_raises(tmp_path: Path) -> None: + base = tmp_path / "ds" + base.mkdir() + with pytest.raises(FileNotFoundError): + load_path(base) + + +def test_malformed_jsonl_raises(tmp_path: Path) -> None: + base = tmp_path / "ds" + base.mkdir() + (base / "facts.jsonl").write_text("not json\n", encoding="utf-8") + (base / "queries.jsonl").write_text("", encoding="utf-8") + with pytest.raises(ValueError, match="malformed JSONL"): + load_path(base) + + +# --------------------------------------------------------------------------- +# T025 — sample() +# --------------------------------------------------------------------------- + + +def test_sample_returns_exactly_n_queries() -> None: + ds = load_default() + sub = sample(ds, 5) + assert len(sub.queries) == 5 + # facts unchanged + assert sub.facts == ds.facts + + +def test_sample_is_deterministic() -> None: + ds = load_default() + a = sample(ds, 7) + b = sample(ds, 7) + assert [q.query_id for q in a.queries] == [q.query_id for q in b.queries] + + +def test_sample_n_larger_than_total_returns_all() -> None: + ds = load_default() + sub = sample(ds, 9999) + assert sub.queries == ds.queries + + +def test_sample_zero_raises() -> None: + ds = load_default() + with pytest.raises(ValueError): + sample(ds, 0) diff --git a/sdk-python/tests/eval/test_report.py b/sdk-python/tests/eval/test_report.py new file mode 100644 index 0000000..65e4d34 --- /dev/null +++ b/sdk-python/tests/eval/test_report.py @@ -0,0 +1,104 @@ +"""T009 — Markdown report writer.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from openmem.eval.report import write_report +from openmem.eval.types import ( + Metrics, + ProviderResult, + QueryResult, + RunConfig, +) + + +def _make_provider( + name: str, + *, + recall_at_5: float = 0.8, + estimated_cost_usd: float = 0.0123, + error_count: int = 0, + note: str | None = None, +) -> ProviderResult: + metrics = Metrics( + recall_at_1=recall_at_5 / 2, + recall_at_5=recall_at_5, + mrr=recall_at_5, + ingest_p50_ms=10.0, + ingest_p95_ms=20.0, + ingest_p99_ms=25.0, + search_p50_ms=5.0, + search_p95_ms=8.0, + search_p99_ms=10.0, + error_count=error_count, + note=note, + ) + return ProviderResult( + provider=name, + query_results=[QueryResult(query_id="q1", top_k_fact_ids=["a"])], + metrics=metrics, + estimated_cost_usd=estimated_cost_usd, + ) + + +def test_write_report_creates_file_with_header(tmp_path: Path) -> None: + out = tmp_path / "report.md" + cfg = RunConfig(providers=("stub",), live=False) + write_report(out, cfg, [_make_provider("stub")], dataset_hash="abc123") + text = out.read_text(encoding="utf-8") + assert "# OMP Eval Report" in text + assert cfg.run_id in text + assert "abc123" in text + + +def test_report_contains_provider_metrics_table(tmp_path: Path) -> None: + out = tmp_path / "r.md" + cfg = RunConfig(providers=("postgres", "mem0"), live=True) + write_report( + out, + cfg, + [ + _make_provider("postgres", recall_at_5=0.92, estimated_cost_usd=0.0), + _make_provider("mem0", recall_at_5=0.7, estimated_cost_usd=0.42), + ], + dataset_hash="h1", + ) + text = out.read_text(encoding="utf-8") + assert "postgres" in text + assert "mem0" in text + assert "0.92" in text + assert "0.7" in text + # cost column present + assert "$" in text + + +def test_report_marks_cache_hits(tmp_path: Path) -> None: + """Cache feature was dropped — placeholder test removed.""" + pytest.skip("caching not supported") + + +def test_report_renders_provider_note(tmp_path: Path) -> None: + out = tmp_path / "r.md" + cfg = RunConfig(providers=("stub",), live=False) + write_report( + out, + cfg, + [_make_provider("stub", note="suspicious recall")], + dataset_hash="h", + ) + assert "suspicious" in out.read_text(encoding="utf-8").lower() + + +def test_report_includes_dry_run_or_live_label(tmp_path: Path) -> None: + out = tmp_path / "r.md" + cfg_dry = RunConfig(providers=("stub",), live=False) + write_report(out, cfg_dry, [_make_provider("stub")], dataset_hash="h") + assert "dry" in out.read_text(encoding="utf-8").lower() + + out2 = tmp_path / "r2.md" + cfg_live = RunConfig(providers=("stub",), live=True) + write_report(out2, cfg_live, [_make_provider("stub")], dataset_hash="h") + assert "live" in out2.read_text(encoding="utf-8").lower() diff --git a/sdk-python/tests/eval/test_runner_with_stub.py b/sdk-python/tests/eval/test_runner_with_stub.py new file mode 100644 index 0000000..a1a28b7 --- /dev/null +++ b/sdk-python/tests/eval/test_runner_with_stub.py @@ -0,0 +1,143 @@ +"""T010 — runner with an injected in-memory stub provider. + +Covers: +* fact_id round-trip (R11 dual-stamping) — must work when the backend echoes + content as well as via the tag fallback when content is rewritten. +* reproducibility — two identical runs produce identical metric numbers (C1). +* multi-provider continuation — one failing provider does not crash the run (C2). +* recall@5 on the bundled dataset against an oracle backend. +""" + +from __future__ import annotations + +import re +import uuid +from dataclasses import dataclass, field +from typing import Callable, Iterable + +import pytest + +from openmem.eval.dataset import load_default +from openmem.eval.runner import _FACT_PREFIX_RE, run +from openmem.eval.types import RunConfig + + +# --------------------------------------------------------------------------- +# Stubs +# --------------------------------------------------------------------------- + + +@dataclass +class _Mem: + """Lightweight stand-in for openmem.types.Memory.""" + id: str + content: str + tags: tuple[str, ...] = () + + +@dataclass +class _Hit: + memory: _Mem + score: float + + +@dataclass +class _OracleStub: + """Oracle backend: search returns the exact stamped fact.""" + user_id: str = "" + _store: dict[str, _Mem] = field(default_factory=dict) + _by_fact_id: dict[str, _Mem] = field(default_factory=dict) + + def add(self, *, content: str, user_id: str, tags: list[str] | None = None, **_): + mid = f"m-{uuid.uuid4().hex[:8]}" + m = _Mem(id=mid, content=content, tags=tuple(tags or ())) + self._store[mid] = m + match = _FACT_PREFIX_RE.match(content) + if match: + self._by_fact_id[match.group(1)] = m + return m + + def search(self, query: str, user_id: str, *, limit: int = 5, **_): + # naive: return the fact whose content contains the most query words + words = set(query.lower().split()) + scored = [] + for m in self._store.values(): + score = sum(1 for w in words if w in m.content.lower()) + if score > 0: + scored.append(_Hit(memory=m, score=float(score))) + scored.sort(key=lambda h: -h.score) + return scored[:limit] + + def delete(self, mid: str) -> None: + self._store.pop(mid, None) + + +@dataclass +class _BrokenStub: + """Backend that always raises on add to test continuation.""" + user_id: str = "" + + def add(self, **_): + raise RuntimeError("simulated provider failure") + + def search(self, *_, **__): + return [] + + def delete(self, *_): + pass + + +def _factory(mapping: dict[str, Callable]) -> Callable[[str], object]: + def make(name: str): + return mapping[name]() + return make + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +def test_oracle_provider_achieves_high_recall() -> None: + cfg = RunConfig(providers=("oracle",), live=True, top_k=5) + ds = load_default() + [pr] = run(cfg, ds, memory_factory=_factory({"oracle": _OracleStub})) + assert pr.metrics is not None + assert pr.metrics.recall_at_5 >= 0.7 + assert pr.metrics.error_count == 0 + + +def test_runner_is_reproducible() -> None: + """C1 — same inputs → identical recall/MRR.""" + ds = load_default() + cfg1 = RunConfig(providers=("oracle",), live=True, top_k=5) + cfg2 = RunConfig(providers=("oracle",), live=True, top_k=5) + [a] = run(cfg1, ds, memory_factory=_factory({"oracle": _OracleStub})) + [b] = run(cfg2, ds, memory_factory=_factory({"oracle": _OracleStub})) + assert a.metrics.recall_at_5 == b.metrics.recall_at_5 + assert a.metrics.mrr == b.metrics.mrr + + +def test_one_failing_provider_does_not_block_others() -> None: + """C2 — broken provider gets error_count>0; oracle still runs.""" + ds = load_default() + cfg = RunConfig(providers=("broken", "oracle"), live=True, top_k=5) + results = run( + cfg, + ds, + memory_factory=_factory({"broken": _BrokenStub, "oracle": _OracleStub}), + ) + by_name = {r.provider: r for r in results} + assert "broken" in by_name and "oracle" in by_name + assert by_name["broken"].metrics.error_count > 0 + assert by_name["oracle"].metrics.recall_at_5 >= 0.5 + + +def test_dry_run_skips_backend_calls() -> None: + """Default RunConfig is live=False → runner returns dry-run preview.""" + ds = load_default() + cfg = RunConfig(providers=("oracle",), live=False) + [pr] = run(cfg, ds, memory_factory=_factory({"oracle": _OracleStub})) + # dry run still produces a ProviderResult, but no query results + assert pr.metrics.error_count == 0 + assert len(pr.query_results) == 0 diff --git a/sdk-python/tests/eval/test_scorer.py b/sdk-python/tests/eval/test_scorer.py new file mode 100644 index 0000000..ae61ebf --- /dev/null +++ b/sdk-python/tests/eval/test_scorer.py @@ -0,0 +1,96 @@ +"""T008 + T034 — scorer math: recall@k, MRR, percentiles, suspicious flag.""" + +from __future__ import annotations + +import pytest + +from openmem.eval.scorer import ( + compute_metrics, + mrr, + recall_at_k, +) +from openmem.eval.types import ( + Metrics, + ProviderResult, + QueryResult, +) + + +def test_recall_at_k_full_hit() -> None: + assert recall_at_k(["a", "b", "c"], ["a"], k=5) == 1.0 + + +def test_recall_at_k_partial() -> None: + # 2 of 3 golds appear in top 5 + assert recall_at_k(["a", "x", "b", "y", "z"], ["a", "b", "c"], k=5) == pytest.approx(2 / 3) + + +def test_recall_at_k_zero_when_no_match() -> None: + assert recall_at_k(["x", "y", "z"], ["a"], k=5) == 0.0 + + +def test_recall_at_k_truncates_to_k() -> None: + # gold at position 6 should not count when k=5 + assert recall_at_k(["x", "x", "x", "x", "x", "a"], ["a"], k=5) == 0.0 + + +def test_recall_at_k_empty_results_returns_zero() -> None: + assert recall_at_k([], ["a"], k=5) == 0.0 + + +def test_mrr_first_rank() -> None: + assert mrr(["a", "b", "c"], ["a"]) == 1.0 + + +def test_mrr_third_rank() -> None: + assert mrr(["x", "y", "a"], ["a"]) == pytest.approx(1 / 3) + + +def test_mrr_no_match_returns_zero() -> None: + assert mrr(["x", "y"], ["a"]) == 0.0 + + +def test_mrr_empty_returns_zero() -> None: + assert mrr([], ["a"]) == 0.0 + + +def test_compute_metrics_macro_averages_across_queries() -> None: + pr = ProviderResult( + provider="stub", + query_results=[ + QueryResult(query_id="q1", top_k_fact_ids=["a"]), # recall=1, rr=1 + QueryResult(query_id="q2", top_k_fact_ids=["x"]), # recall=0, rr=0 + ], + ingest_latencies_ms=[10.0, 20.0, 30.0], + search_latencies_ms=[5.0, 15.0], + ) + gold = {"q1": ("a",), "q2": ("b",)} + metrics = compute_metrics(pr, gold) + assert metrics.recall_at_1 == pytest.approx(0.5) + assert metrics.recall_at_5 == pytest.approx(0.5) + assert metrics.mrr == pytest.approx(0.5) + assert metrics.ingest_p50_ms == pytest.approx(20.0, rel=0.5) + assert metrics.search_p50_ms > 0 + + +def test_compute_metrics_handles_empty_latencies() -> None: + pr = ProviderResult(provider="stub") + metrics = compute_metrics(pr, {}) + assert metrics.ingest_p50_ms == 0.0 + assert metrics.search_p95_ms == 0.0 + + +def test_suspicious_flag_set_when_recall_below_threshold() -> None: + """T034 — recall@5 < 0.1 sets Metrics.note.""" + pr = ProviderResult( + provider="bad", + query_results=[ + QueryResult(query_id="q1", top_k_fact_ids=["x"]), + QueryResult(query_id="q2", top_k_fact_ids=["y"]), + ], + ) + gold = {"q1": ("a",), "q2": ("b",)} + metrics = compute_metrics(pr, gold) + assert metrics.recall_at_5 == 0.0 + assert metrics.note is not None + assert "suspicious" in metrics.note diff --git a/sdk-python/tests/eval/test_trace.py b/sdk-python/tests/eval/test_trace.py new file mode 100644 index 0000000..9137f7c --- /dev/null +++ b/sdk-python/tests/eval/test_trace.py @@ -0,0 +1,61 @@ +"""T014 — trace JSONL writer with hashed payloads.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from openmem.eval.trace import TraceWriter + + +def test_trace_writes_jsonl_with_payload_hash(tmp_path: Path) -> None: + out = tmp_path / "t.jsonl" + with TraceWriter(out) as w: + w.emit( + provider="postgres", + verb="add", + run_id="r1", + latency_ms=12.5, + payload_text="user prefers pnpm", + ) + w.emit( + provider="postgres", + verb="search", + run_id="r1", + latency_ms=3.4, + payload_text="package manager", + result_count=5, + ) + lines = out.read_text(encoding="utf-8").splitlines() + assert len(lines) == 2 + rec0 = json.loads(lines[0]) + assert rec0["provider"] == "postgres" + assert rec0["verb"] == "add" + assert "payload_hash" in rec0 + assert len(rec0["payload_hash"]) == 12 + assert "user prefers pnpm" not in lines[0] # raw text never written + rec1 = json.loads(lines[1]) + assert rec1["result_count"] == 5 + + +def test_trace_emits_error_field(tmp_path: Path) -> None: + out = tmp_path / "t.jsonl" + with TraceWriter(out) as w: + w.emit(provider="mem0", verb="add", run_id="r1", latency_ms=1.0, error="boom") + rec = json.loads(out.read_text()) + assert rec["error"] == "boom" + + +def test_trace_requires_context_manager(tmp_path: Path) -> None: + w = TraceWriter(tmp_path / "t.jsonl") + with pytest.raises(RuntimeError): + w.emit(provider="x", verb="add", run_id="r", latency_ms=1.0) + + +def test_trace_creates_parent_directories(tmp_path: Path) -> None: + out = tmp_path / "deep" / "nested" / "t.jsonl" + with TraceWriter(out) as w: + w.emit(provider="x", verb="add", run_id="r", latency_ms=1.0) + assert out.exists() diff --git a/sdk-python/tests/test_memory_facade.py b/sdk-python/tests/test_memory_facade.py index 7554d7f..5ecdd5b 100644 --- a/sdk-python/tests/test_memory_facade.py +++ b/sdk-python/tests/test_memory_facade.py @@ -254,6 +254,26 @@ def test_facade_capabilities_caches_result() -> None: assert len(capability_calls) == 1 +def test_facade_wait_for_ingest_is_noop_when_adapter_lacks_hook() -> None: + """T015a — adapters using BaseAdapter's default no-op succeed silently.""" + mem, _ = _make_memory() + mem.wait_for_ingest(["m-1", "m-2"], "u1") + mem.wait_for_ingest(["m-1"], "u1", timeout=5.0) + + +def test_facade_wait_for_ingest_calls_adapter_hook_when_overridden() -> None: + """T015a — pass-through invokes adapter.wait_for_ingest with all args.""" + mem, stub = _make_memory() + captured: list[tuple] = [] + + def _hook(ids, user_id, *, timeout=None): + captured.append((tuple(ids), user_id, timeout)) + + stub.wait_for_ingest = _hook # type: ignore[attr-defined] + mem.wait_for_ingest(["m-1", "m-2"], "u1", timeout=12.5) + assert captured == [(("m-1", "m-2"), "u1", 12.5)] + + # --------------------------------------------------------------------------- # _resolve_adapter — provider dispatch and error paths # --------------------------------------------------------------------------- diff --git a/specs/004-eval-kit/checklists/requirements.md b/specs/004-eval-kit/checklists/requirements.md new file mode 100644 index 0000000..78c4402 --- /dev/null +++ b/specs/004-eval-kit/checklists/requirements.md @@ -0,0 +1,44 @@ +# Specification Quality Checklist: M3.1 Eval Kit + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-05-01 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (no implementation details) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification + +## Notes + +- The spec intentionally references the four existing OMP adapter + product names (`postgres`, `mem0`, `supermemory`, `letta`) and the + package path (`openmem.eval`) because the feature is an internal + developer tool tightly bound to the existing SDK shape. These are + product/scope identifiers, not implementation choices. +- One assumption mentions `asyncio.gather` as a possible future + parallelism strategy — explicitly scoped P3 and out of MVP, so it + does not constrain the planning phase. +- Cost ceiling (SC-007: USD $0.50 per full live run) is a measurable + business constraint that protects against quota burn; not an + implementation detail. diff --git a/specs/004-eval-kit/contracts/cli.md b/specs/004-eval-kit/contracts/cli.md new file mode 100644 index 0000000..567b16f --- /dev/null +++ b/specs/004-eval-kit/contracts/cli.md @@ -0,0 +1,124 @@ +# CLI Contract — `openmem-eval` + +This document is the canonical contract for the eval CLI. Tests under +`sdk-python/tests/eval/test_cli_*.py` validate every clause here. + +## Invocation + +Both forms are equivalent: + +``` +openmem-eval [FLAGS] +python -m openmem.eval [FLAGS] +``` + +## Flags + +| Flag | Type | Default | Required | Description | +|---|---|---|---|---| +| `--providers` | comma-separated list | `postgres` | no | Provider names to run. Aliases like `postgres:hnsw` allowed. | +| `--live` | bool flag | false | no | Make real API calls. Mutually exclusive with `--dry-run`. | +| `--dry-run` | bool flag | true | no | Print cost estimate, make no network calls. | +| `--report` | path | `eval-report.md` | no | Markdown output path. | +| `--trace` | path | `eval-trace.jsonl` | no | JSONL trace output path (live runs only). | +| `--sample` | int | none (= all) | no | Take first N queries by stable order. | +| `--cost-threshold` | float | `1.00` | no | USD; above this, prompt for confirmation. | +| `--yes` | bool flag | false | no | Skip the confirmation prompt. | +| `--cleanup` | bool flag | false | no | Delete every memory created by this run before exiting. | +| `--dataset` | path | bundled default | no | Override bundled dataset (advanced use). | +| `-v` / `--verbose` | bool flag | false | no | Echo per-call progress to stderr. | +| `-h` / `--help` | bool flag | — | no | Print help and exit 0. | +| `--version` | bool flag | — | no | Print SDK + dataset version and exit 0. | + +### Mutual exclusion + +- `--live` and `--dry-run` are mutually exclusive. Default behaviour + when neither is set: `--dry-run`. + +## Exit Codes + +| Code | Meaning | +|---|---| +| 0 | Success. Includes runs where some providers were skipped or failed (the report records the outcome). | +| 1 | No providers were runnable (all skipped due to missing keys). | +| 2 | Bad invocation (unknown provider, malformed flags, missing dataset file). | +| 3 | Confirmation refused (live mode above cost threshold without `--yes` and no TTY). | +| 4 | Internal error in the harness (bug). | + +Non-zero exits MUST print a single line to stderr explaining the cause. + +## Standard Output (stdout) + +Stdout is reserved for the human-readable summary printed at the end of a run: + +``` +OMP Eval Report (run_id=) +================================ +Dataset: default- (50 facts, 20 queries) +Mode: live Providers: postgres, mem0 +Wall-clock: 142.3 s Estimated cost: $0.04 + +provider recall@1 recall@5 recall@10 MRR ingest p50/p95 search p50/p95 errors +postgres 0.85 0.95 0.95 0.91 12 / 31 ms 18 / 47 ms 0 +mem0 0.80 0.90 0.95 0.87 1840 / 2950 ms 320 / 540 ms 1 + +Report written: eval-report.md +Trace written: eval-trace.jsonl +``` + +In `--dry-run` mode the table is replaced by a cost-estimate table: + +``` +DRY RUN — no live API calls made. + +provider adds searches waits est. cost +postgres 50 20 50 $0.00 +mem0 50 20 50 $0.012 (free tier may cover) +TOTAL $0.012 +``` + +## Standard Error (stderr) + +stderr is for progress and warnings: + +- `[skip] supermemory: missing $SUPERMEMORY_API_KEY` +- `[warn] cost model unknown for (foo, bar); assuming $0` +- `[verbose] mem0/add f-0001 -> mem_abc latency=142ms` (only with `-v`) + +## Confirmation Prompt + +When `--live` is set, estimated cost > `--cost-threshold`, `--yes` is not +passed, and stdin is a TTY: + +``` +This run will make ~360 API calls across 4 providers. +Estimated cost: $0.32 USD. +Continue? [y/N]: +``` + +A response other than `y` or `Y` aborts with exit 3. + +## Files Written + +| Path | When | Content | +|---|---|---| +| `eval-report.md` (or `--report`) | Always | Markdown report (see `data-model.md` § Report). | +| `eval-trace.jsonl` (or `--trace`) | Live runs only | One JSON object per live API call. | + +## Determinism Guarantees + +- Same `--providers`, same `--sample`, same dataset content ⇒ identical + recall@k and MRR (latency naturally varies). +- `--sample N` selects the same N queries every run (stable hash order). +- `--dataset PATH` makes the dataset_hash a function purely of file + content, so external datasets remain reproducible. + +## Non-Goals + +- The CLI does NOT support concurrent provider execution in v1 + (asyncio.gather is a P3 future enhancement noted in the spec + assumptions). +- The CLI does NOT support custom scorers in v1. +- The CLI does NOT install in the user's PATH automatically — it is + registered via `[project.scripts]` and becomes available after + `pip install openmem`. diff --git a/specs/004-eval-kit/data-model.md b/specs/004-eval-kit/data-model.md new file mode 100644 index 0000000..18f11f5 --- /dev/null +++ b/specs/004-eval-kit/data-model.md @@ -0,0 +1,199 @@ +# Phase 1 Data Model — M3.1 Eval Kit + +All entities below are internal to the harness (`openmem.eval`). None are +OMP protocol surfaces; they do not appear in `spec/omp-0.1.openapi.yaml`. + +--- + +## Dataset + +Bundled JSONL corpus the harness ingests and queries against. + +| Field | Type | Required | Description | +|---|---|---|---| +| `facts` | list[Fact] | yes | Records to ingest into each provider. | +| `queries` | list[Query] | yes | Queries to run against each provider. | +| `dataset_hash` | str | yes (derived) | SHA-256(facts.jsonl ‖ queries.jsonl), first 12 hex chars. | + +### Fact + +| Field | Type | Required | Description | +|---|---|---|---| +| `fact_id` | str | yes | Stable identifier used in `gold_fact_ids`. Format `f-NNNN`. | +| `content` | str | yes | The text payload sent as `Memory.add(content=...)`. | +| `tags` | list[str] | no | Optional tags forwarded to the adapter. | + +**Validation**: +- `fact_id` must be unique within the dataset. +- `content` must be non-empty. +- All `tags` items must be lowercase ASCII slugs. + +**Ingest stamping** (per [research.md §R11](research.md)): +The runner does NOT pass `content` raw. It rewrites it as +`f"[fact_id={fact_id}] {content}"` and appends `f"fact:{fact_id}"` to the +tag list before calling `Memory.add(...)`. Recovery from `SearchResult` +uses the regex `^\[fact_id=([^\]]+)\] ` against the returned content. +Results whose content does not match are filtered from the top-k before +scoring. + +### Query + +| Field | Type | Required | Description | +|---|---|---|---| +| `query_id` | str | yes | Stable identifier. Format `q-NNNN`. | +| `query` | str | yes | The query string passed to `Memory.search(query=...)`. | +| `gold_fact_ids` | list[str] | yes | The fact IDs that should appear in the top-k for a correct retrieval. | + +**Validation**: +- Every entry in `gold_fact_ids` must reference an existing `fact_id`. +- `gold_fact_ids` must be non-empty (a query with no answer is not a + valid eval input). + +--- + +## RunConfig + +Captures the user's invocation. Built once from CLI flags; immutable for +the rest of the run. + +| Field | Type | Default | Description | +|---|---|---|---| +| `providers` | list[str] | `["postgres"]` | Provider names; aliases like `postgres:hnsw` allowed. | +| `live` | bool | `false` | If false, dry-run (no network). | +| `dry_run` | bool | `true` | Inverse of `live`; redundant for clarity. | +| `report_path` | Path | `eval-report.md` | Destination Markdown file. | +| `trace_path` | Path | `eval-trace.jsonl` | Destination JSONL trace file. | +| `sample` | int \| None | `None` | If set, take first N queries by stable hash order. | +| `cost_threshold_usd` | float | `1.00` | Above this, prompt before running. | +| `yes` | bool | `false` | Skip confirmation prompt. | +| `cleanup` | bool | `false` | If true, after run, delete every memory created. | +| `run_id` | str | (UUID4) | Auto-generated; embedded in user_ids. | + +**Derived**: +- `user_id` per provider = `f"eval-{run_id}"`. + +**Validation**: +- `live` and `dry_run` MUST be opposites. +- If `live` is true and stdin is not a TTY and `yes` is false → exit 3. +- Unknown provider names → exit 2 with the list of known names. + +--- + +## ProviderResult + +One per provider in the run. + +| Field | Type | Description | +|---|---|---| +| `provider` | str | Provider name (possibly aliased). | +| `status` | str | `ok` / `skipped` / `partial` / `failed`. | +| `skip_reason` | str \| None | E.g., `"missing $MEM0_API_KEY"`. | +| `ingest_latencies_ms` | list[float] | One entry per fact actually ingested. | +| `search_latencies_ms` | list[float] | One entry per query actually executed. | +| `query_results` | list[QueryResult] | Per-query top-k + score outcome. | +| `errors` | list[ErrorRecord] | Captured exceptions (one per failed call). | +| `metrics` | Metrics | Computed from the above. | +| `total_wall_s` | float | Wall-clock from start to finish for this provider. | + +### QueryResult + +| Field | Type | Description | +|---|---|---| +| `query_id` | str | From the dataset. | +| `top_k_fact_ids` | list[str] | Top-k `fact_id`s extracted from the search response. | +| `latency_ms` | float | Single search call latency. | +| `error` | str \| None | Set if this single query failed. | + +### Metrics + +| Field | Type | Description | +|---|---|---| +| `recall_at_1` | float | 0.0–1.0 | +| `recall_at_5` | float | 0.0–1.0 | +| `recall_at_10` | float | 0.0–1.0 | +| `mrr` | float | 0.0–1.0 | +| `ingest_p50_ms` | float | Computed when ingest_latencies_ms is non-empty. | +| `ingest_p95_ms` | float | Same. | +| `search_p50_ms` | float | Computed when search_latencies_ms is non-empty. | +| `search_p95_ms` | float | Same. | +| `error_count` | int | Length of `errors`. | +| `note` | str \| None | E.g., `"suspicious — recall@5 < 0.1"`. | + +### ErrorRecord + +| Field | Type | Description | +|---|---|---| +| `verb` | str | `add` / `search` / `wait_for_ingest` / etc. | +| `query_id` | str \| None | Set for per-query failures. | +| `fact_id` | str \| None | Set for per-fact ingest failures. | +| `error_class` | str | Python exception class name. | +| `message` | str | Truncated to 200 chars. | +| `ts` | str | ISO 8601 UTC. | + +--- + +## Report + +Single Markdown file. Rendered from `RunConfig` + `list[ProviderResult]` + +the dataset metadata. + +Sections (in order): +1. Title + metadata table. +2. Comparison table (one row per provider). +3. Per-provider notes (suspicious-flag block). +4. Failure traces (only sections for providers with `errors`). +5. Cost-model disclaimer footer + dataset license note. + +--- + +## TraceEntry + +One JSON object per line in `eval-trace.jsonl`. + +| Field | Type | Description | +|---|---|---| +| `ts` | str | ISO 8601 UTC. | +| `run_id` | str | From RunConfig. | +| `provider` | str | Provider name. | +| `verb` | str | API verb. | +| `latency_ms` | float | Wall-clock for this single call. | +| `status` | str | `ok` / `error`. | +| `request_hash` | str | SHA-256 of canonical payload, 12 hex chars. PII never logged. | +| `error_class` | str \| None | When `status=="error"`. | + +--- + +## CostModel + +Static dict (`dict[tuple[str, str], float]`) at module scope in +`openmem/eval/cost.py`. Read-only at runtime. + +```python +COST_USD_PER_CALL: dict[tuple[str, str], float] = { + ("postgres", "add"): 0.0, + ("postgres", "search"): 0.0, + ("mem0", "add"): 0.0001, + ("mem0", "search"): 0.0001, + ("mem0", "get"): 0.0001, + ("supermemory", "add"): 0.0, + ("supermemory", "search"): 0.0, + ("letta", "add"): 0.0001, + ("letta", "search"): 0.002, +} +``` + +Function: `estimate(provider: str, verb: str, n_calls: int) -> float`. +Unknown `(provider, verb)` defaults to `0.0` and is logged as a warning. + +--- + +## Relationships + +``` +RunConfig 1 ── many ProviderResult ── 1 Metrics + │ + └── many QueryResult, many ErrorRecord +Dataset 1 ── many Fact, many Query +Query ── references ──> Fact (via gold_fact_ids) +Each API call ── appended ──> TraceEntry (in eval-trace.jsonl) +``` diff --git a/specs/004-eval-kit/plan.md b/specs/004-eval-kit/plan.md new file mode 100644 index 0000000..9e52e09 --- /dev/null +++ b/specs/004-eval-kit/plan.md @@ -0,0 +1,136 @@ +# Implementation Plan: M3.1 Eval Kit (`openmem-eval`) + +**Branch**: `004-eval-kit` | **Date**: 2026-05-01 | **Spec**: [spec.md](spec.md) +**Input**: Feature specification from `/specs/004-eval-kit/spec.md` + +## Summary + +Ship `openmem-eval` — a CLI benchmark harness that ingests a bundled dataset +into one or more OMP adapters, executes a fixed query set, scores recall@k / +MRR / latency, and emits a Markdown comparison report. Live API calls are +opt-in (`--live`); the default `--dry-run` mode prints a per-provider request +count and USD cost estimate so accidental quota burn is impossible. Cost +ceiling for a full four-provider live run: USD $0.50. + +The harness is a thin orchestrator over the existing `Memory` facade — it +introduces no new adapter surface area and adds no runtime dependency on any +vendor. It is **not** wired into CI; it is a manually invoked maintainer +tool used to produce release-notes comparison tables. + +## Technical Context + +**Language/Version**: Python 3.11+ (matching the SDK) +**Primary Dependencies**: existing `openmem` package only; stdlib `argparse`, +`json`, `hashlib`, `statistics`, `time`, `pathlib`, `re`. No new third-party deps. + +**Facade extension required**: This feature requires one strictly-additive +change to the public `Memory` facade — exposing `wait_for_ingest(ids, user_id, timeout)` +as a thin pass-through to the underlying adapter (per [research.md §R12](research.md)). +No existing callers affected; backward-compatible per Principle III. + +**Postgres embedder config**: The runner instantiates `Memory(provider="postgres", url=..., embedder="hash")` +where `embedder="hash"` is the existing local-only fallback already +implemented in `openmem.adapters.embedder`. This keeps postgres runs +zero-cost and offline-capable per [research.md §R12](research.md). +**Storage**: bundled JSONL dataset under `sdk-python/openmem/eval/datasets/`; +report file at `--report` (default `eval-report.md`); JSONL trace at +`eval-trace.jsonl`. **No on-disk cache** — every live run re-ingests so +reported numbers always reflect the provider's current behaviour. +**Testing**: pytest (existing harness). Unit tests for scorer + cost model + +dataset loader; integration tests for the CLI in dry-run mode against a stub +adapter; one optional live smoke test against postgres in Docker (matches the +existing `test_postgres_specific.py` pattern). +**Target Platform**: Linux/macOS/Windows developer workstations. Same matrix +as the existing SDK; CI runs Linux only. +**Project Type**: CLI tool packaged as a sub-module of the existing `openmem` +library (`openmem.eval`). +**Performance Goals**: full live run across 4 providers ≤ 5 minutes wall-clock +(SC-001); `--dry-run` ≤ 5 seconds (SC-004); postgres-only `--sample 10` run +≤ 30 seconds (SC-005). +**Constraints**: +- Cost ceiling USD $0.50 per full live run (SC-007). +- Reproducible: same dataset_hash + same providers ⇒ identical recall@k & MRR + (SC-003). Latency naturally varies. +- Zero accidental cost: default behaviour MUST be dry-run; `--live` MUST + print and confirm (above $1.00 threshold) before any paid call (FR-003, + FR-004). +- No new third-party runtime dependencies (keep install footprint stable). +- Never run in CI (FR-011). +**Scale/Scope**: ~50 facts × ~20 queries × 4 providers = ~360 ingest calls + +~80 search calls per full live run. Bundled dataset is small enough to fit +~20 KB. + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* + +| Principle | How this plan satisfies it | +|---|---| +| I. Spec-First, Single Source of Truth (NON-NEGOTIABLE) | Eval kit consumes the existing `Memory` facade only. It introduces no new verbs, no new schema fields, no new error codes. Nothing to add to `spec/omp-0.1.openapi.yaml`. The dataset format is internal harness data, not an OMP protocol surface. | +| II. Adapter Conformance via Shared Contract Tests (NON-NEGOTIABLE) | The kit *uses* adapters via the contract surface; it does not weaken the contract. No adapter changes. The existing `test_contract_*.py` suite continues to gate every adapter unmodified. The kit's own unit tests use a stub adapter that satisfies `BaseAdapter`. | +| III. Backward and Forward Compatibility | The kit reads only fields already required on `Memory` (`id`, `content`, `user_id`, `created_at`) plus optional `tags`. It tolerates unknown fields (ignores extras in search results). No required-field changes. | +| IV. Provider Neutrality and User Sovereignty | Default `--providers postgres` works fully self-hosted with no third-party account (matches the postgres reference path). Live providers are opt-in. The kit ships zero telemetry — the only network calls are user-initiated provider API calls. Per-run `user_id` (`eval-{run_id}`) plus `--cleanup` flag protect user data sovereignty. | +| V. Open Extensibility via Namespaced Fields | The kit ignores `x-*` extension fields when scoring (it only inspects `id` and `content`). No promotion of provider-specific fields. The dataset's `tags` and `gold_fact_ids` are harness-internal, not protocol fields. | + +**Result**: All five principles satisfied with no violations. No Complexity Tracking entries required. + +## Project Structure + +### Documentation (this feature) + +```text +specs/004-eval-kit/ +├── plan.md # This file +├── spec.md # Feature specification +├── research.md # Phase 0 output +├── data-model.md # Phase 1 output +├── quickstart.md # Phase 1 output +├── contracts/ +│ └── cli.md # CLI contract (flags, exit codes, output format) +└── checklists/ + └── requirements.md # /speckit.specify quality checklist (already passing) +``` + +### Source Code (repository root) + +```text +sdk-python/ +├── openmem/ +│ ├── eval/ # NEW sub-package +│ │ ├── __init__.py +│ │ ├── __main__.py # `python -m openmem.eval` +│ │ ├── cli.py # argparse + orchestration +│ │ ├── dataset.py # JSONL loader + sha256 hashing +│ │ ├── runner.py # ingest → wait → search pipeline +│ │ ├── scorer.py # recall@k, MRR, percentiles +│ │ ├── report.py # Markdown emitter +│ │ ├── cost.py # static cost-per-call table +│ │ ├── trace.py # JSONL trace writer +│ │ └── datasets/ +│ │ └── default.jsonl # bundled ~50-fact corpus +│ └── ... # existing adapters untouched +└── tests/ + └── eval/ # NEW test directory + ├── test_dataset.py # loader + hash determinism + ├── test_scorer.py # recall@k, MRR math + ├── test_cost.py # cost-model arithmetic + + ├── test_runner_with_stub.py # pipeline against stub adapter + ├── test_cli_dry_run.py # CLI flag parsing + dry-run output + └── test_cli_live_postgres.py # opt-in smoke (skipped without docker) +``` + +**Structure Decision**: Sub-package inside the existing `openmem` library +(option: single project). The harness reuses the `Memory` facade and `Capabilities` +machinery; a separate top-level package would force duplicated import paths and +break the "one `pip install openmem` ships everything" story. The CLI entry +point `openmem-eval` is registered via `pyproject.toml`'s +`[project.scripts]` table alongside the existing `omp-validate-spec` entry. + +## Complexity Tracking + +> No Constitution Check violations. Table intentionally empty. + +| Violation | Why Needed | Simpler Alternative Rejected Because | +|-----------|------------|-------------------------------------| +| (none) | — | — | diff --git a/specs/004-eval-kit/quickstart.md b/specs/004-eval-kit/quickstart.md new file mode 100644 index 0000000..2e1bc1d --- /dev/null +++ b/specs/004-eval-kit/quickstart.md @@ -0,0 +1,110 @@ +# Quickstart — `openmem-eval` + +> Manually-invoked benchmark harness. Never runs in CI. + +## Install + +```powershell +pip install -e "./sdk-python[dev]" +``` + +`openmem-eval` is now on your PATH. + +## See it work without spending anything + +```powershell +openmem-eval # default: postgres-only, dry-run +openmem-eval --providers postgres,mem0,supermemory,letta --dry-run +``` + +The second command prints a cost estimate for a full four-provider live run. +No keys required, no network calls made. + +## Run against local Postgres only (free, ~30s) + +Bring up the test postgres container that the existing live tests use: + +```powershell +docker run -d --rm -p 5432:5432 -e POSTGRES_PASSWORD=postgres ` + -e POSTGRES_DB=omp_test pgvector/pgvector:pg16 +$env:OMP_POSTGRES_URL = "postgresql://postgres:postgres@localhost:5432/omp_test" + +openmem-eval --providers postgres --live --sample 10 +``` + +You will get an `eval-report.md` written to the current directory and a +single-row comparison table on stdout. + +## Run against all four providers (live, ~5 min, ~$0.30) + +Set the env vars you already use for the live test suite: + +```powershell +$env:MEM0_API_KEY = "..." +$env:SUPERMEMORY_API_KEY = "..." +$env:LETTA_API_KEY = "..." +$env:OMP_POSTGRES_URL = "postgresql://..." + +openmem-eval --providers postgres,mem0,supermemory,letta --live +``` + +The harness: +1. Prints the request count + USD estimate. +2. Prompts `Continue? [y/N]:` if the estimate exceeds $1.00. +3. Ingests the bundled dataset into each provider under `user_id=eval-{run_id}`. +4. Waits for ingest to complete (calls `adapter.wait_for_ingest`). +5. Runs all 20 queries against each provider, capturing top-k + latency. +6. Computes recall@1/5/10, MRR, p50/p95 latency. +7. Writes `eval-report.md` with the comparison table and a per-provider + trace section for any failed queries. + +## Re-running cheaply + +If you just want to tweak the scorer or re-format the report and don't +need fresh ingest numbers: + +```powershell +openmem-eval --providers postgres,mem0 --live --use-cache +``` + +The cache lives at `~/.cache/openmem/eval/{dataset_hash}/`. It is +`fact_id → memory.id` only; no content is cached. + +## Cleanup + +To remove every memory the harness created in your providers: + +```powershell +openmem-eval --providers postgres,mem0,supermemory,letta --live --cleanup +``` + +The most recent `run_id` is read from the cache; each `memory.id` is +deleted via `Memory.delete()`. Failures are warnings. + +## Troubleshooting + +- **"missing $MEM0_API_KEY"** — set the env var or drop that provider + from `--providers`. +- **"Continue? [y/N]:"** — answer `y` or pass `--yes`. +- **Suspicious recall < 0.1** — verify the provider's index is populated + (the report flags this); often a sign that ingest hasn't completed. +- **Ingest timeout on mem0** — first run pays the ~30s/fact cost. + Subsequent runs with `--use-cache` skip this. + +## What the report looks like + +``` +# OMP Eval Report + +| provider | recall@1 | recall@5 | MRR | ingest p50/p95 (ms) | search p50/p95 (ms) | errors | +|-------------|----------|----------|------|---------------------|---------------------|--------| +| postgres | 0.85 | 0.95 | 0.91 | 12 / 31 | 18 / 47 | 0 | +| mem0 | 0.80 | 0.90 | 0.87 | 1840 / 2950 | 320 / 540 | 0 | +| supermemory | 0.75 | 0.95 | 0.85 | 4200 / 5800 | 280 / 410 | 0 | +| letta | 0.70 | 0.85 | 0.78 | 5100 / 6900 | 1100 / 1800 | 1 | + +Dataset: default-a1b2c3d4e5f6 (50 facts, 20 queries) +Run: 2026-05-01T12:34:56Z OMP 0.4.0 +``` + +This is what you copy into the release notes. diff --git a/specs/004-eval-kit/research.md b/specs/004-eval-kit/research.md new file mode 100644 index 0000000..349a1ec --- /dev/null +++ b/specs/004-eval-kit/research.md @@ -0,0 +1,311 @@ +# Phase 0 Research — M3.1 Eval Kit + +All open questions identified during specification & planning have been +resolved. No `NEEDS CLARIFICATION` markers remain. + +--- + +## R1. Metric definitions + +**Decision**: Compute `recall@k` per query as +`|top_k_returned ∩ gold_fact_ids| / |gold_fact_ids|`, then macro-average +across queries (each query weighted equally). Compute `MRR` as the mean of +`1/rank` of the first gold fact ID in the returned list (0 if none). +Latency percentiles via `statistics.quantiles(n=100)` for p50/p95. + +**Rationale**: These are the canonical IR formulations. Macro-average over +queries (not micro) so a single query with many golds doesn't dominate. +Stdlib `statistics.quantiles` avoids a numpy dependency. + +**Alternatives considered**: +- nDCG: rejected — requires graded relevance, the bundled dataset is + binary (gold or not). +- Precision@k: rejected — recall is more discriminating with small `k` + and few golds per query (typically 1–3). +- numpy percentiles: rejected — adds a heavyweight runtime dep for a + trivial calculation. + +--- + +## R2. Dataset format and bundling + +**Decision**: Two JSONL files in `openmem/eval/datasets/default/`: +- `facts.jsonl` — one record per line: `{"fact_id": str, "content": str, "tags": [str]}` +- `queries.jsonl` — one record per line: `{"query_id": str, "query": str, "gold_fact_ids": [str]}` + +Loaded via `importlib.resources.files("openmem.eval.datasets")`. Hashed by +SHA-256 of the concatenation of both files in canonical (sorted-line) order, +truncated to 12 hex chars for display. + +**Rationale**: JSONL streams cleanly, diffs cleanly in PRs, and survives +`pip install` via standard package data inclusion. Splitting facts vs. +queries lets us version the gold set independently if we add new +queries against the same corpus. `importlib.resources` is the canonical +Python 3.11+ way to read packaged data. + +**Alternatives considered**: +- Single CSV / TSV: rejected — escaping commas/tabs in real text is + error-prone. +- Parquet: rejected — adds pyarrow dep; binary diff is hostile to PR review. +- External download from a HuggingFace dataset: rejected — fails the + "works on a fresh checkout with no external resources" requirement. + +--- + +## R3. Cost model + +**Decision**: Static dict in `openmem/eval/cost.py` keyed by +`(provider, verb)` returning `cost_usd_per_call`. Document in the file +header that values are best-effort approximations as of the file's commit +date and in the report footer that real costs may differ. Initial table: + +| provider | verb | $/call | notes | +|---|---|---|---| +| postgres | * | 0.0 | local Docker, free | +| mem0 | add / search / get | 0.0001 | ~$0.10 per 1k events on paid tier | +| supermemory | add / search | 0.0 | covered by free tier of $10/mo plans | +| letta | add | 0.0001 | minimal — embed-only | +| letta | search | 0.002 | agent inference per call | + +**Rationale**: A single-file static table is readable, auditable in PR +diffs, and easy to update when pricing changes. We deliberately do **not** +fetch live pricing — that would be its own infrastructure problem and +would itself need network access during dry-run. + +**Alternatives considered**: +- Fetch from each provider's pricing API: rejected — no such APIs exist + uniformly; defeats the offline-dry-run promise. +- Skip cost estimation: rejected — the spec (FR-004, SC-007) requires it + to enforce the cost ceiling. + +--- + +## R4. Confirmation prompt mechanics + +**Decision**: When `--live` is passed and estimated cost exceeds +`--cost-threshold` (default $1.00), prompt with `Continue? [y/N]:` on stdin +unless `--yes` is also passed. In non-TTY contexts (CI, pipes), require +`--yes`; otherwise abort with exit code 3. + +**Rationale**: Standard CLI convention (apt, gh, etc.). Hard refusal +without TTY prevents background scripts from accidentally racking up +charges. + +**Alternatives considered**: +- Always prompt (no threshold): rejected — friction for cheap iterative + runs. +- Always require `--yes`: rejected — bypasses the helpful interactive + preview. + +--- + +## R5. Ingest cache layout + +**Decision**: One JSON file per `(provider, dataset_hash)` at +`~/.cache/openmem/eval/{dataset_hash}/{provider}.json` containing: +``` +{"run_id": "...", "ingested_at": "...", "memory_ids": {"fact_id": "mem_..."}} +``` +Cache is consulted only when `--use-cache` is passed (default off — explicit +opt-in to avoid stale-data confusion in published numbers). When stale, the +cache is overwritten silently. + +**Rationale**: The cache turns mem0's ~30s ingest delay into a one-time +cost per dataset. Per-provider files allow partial cache hits (re-ingest +only the providers whose configs changed). Default-off because a maintainer +publishing numbers must trust they're fresh. + +**Alternatives considered**: +- Always-on cache: rejected — risks publishing stale numbers. +- SQLite cache: rejected — over-engineered for ~50 records. +- In-memory only (no disk): rejected — defeats the purpose of caching + across iteration cycles. + +--- + +## R6. Trace file format + +**Decision**: JSONL at `--trace` (default `eval-trace.jsonl`), one record +per API call: +``` +{"ts": "2026-05-01T12:34:56Z", "provider": "mem0", "verb": "add", + "latency_ms": 142, "status": "ok", "request_hash": "abc123", "run_id": "..."} +``` +`request_hash` is sha256 of the canonicalised payload, truncated to 12 +hex; full payloads are NOT logged (PII safety). + +**Rationale**: JSONL streams without buffering crashes losing data; +hashed payloads protect user-supplied strings from leaking into traces +shared in bug reports. + +**Alternatives considered**: +- OpenTelemetry traces: rejected — adds a heavy dep; the OTel work belongs + in M3.3 (separate spec). +- Plain text log: rejected — un-machine-parseable for post-hoc analysis. + +--- + +## R7. Per-run namespacing & cleanup + +**Decision**: Every memory ingested by the harness uses +`user_id = f"eval-{run_id}"` where `run_id` is a fresh UUID4 per +invocation. The run_id is recorded in the cache file and the report +metadata. `--cleanup` reads the most recent run's cache and calls +`memory.delete(memory_id)` for each entry per provider; failures are +warnings, not errors. + +**Rationale**: Per-run user_ids prevent eval data from polluting real +user data, satisfy the constitution's user-sovereignty principle, and +make targeted cleanup trivial (no need to scan for "evaly-looking" +memories). UUID4 avoids collisions if multiple maintainers run the kit +against the same shared backend. + +**Alternatives considered**: +- Single fixed `eval` user_id: rejected — concurrent runs would interfere. +- Database-level cleanup (TRUNCATE): rejected — destroys real user data; + not portable across providers. + +--- + +## R8. Stub adapter for unit tests + +**Decision**: Reuse the `_StubAdapter` pattern already established in +`sdk-python/tests/test_memory_facade.py` — a `BaseAdapter` subclass +that records calls and returns deterministic synthetic results. Each +eval test case can configure its stub's `search()` return values to +exercise scorer edge cases without any live backend. + +**Rationale**: Pattern is already in the codebase, well-understood, and +keeps unit tests fast (sub-second) and key-free. + +**Alternatives considered**: +- `unittest.mock.MagicMock`: rejected — loses type safety on the + adapter contract; we already have a typed stub. +- An in-process fake store with real ranking: rejected — too much logic + in test infra; the goal is to test the scorer, not the fake store. + +--- + +## R9. Live postgres smoke test + +**Decision**: One `test_cli_live_postgres.py` that runs the CLI end-to-end +against a Docker postgres container. Skips automatically when +`OMP_POSTGRES_URL` is unset (matches `test_postgres_specific.py` convention). +CI will set this env var in the existing postgres job, so the smoke runs +on every push. + +**Rationale**: One real run validates that the CLI plumbing actually works, +not just the unit pieces. Postgres-only stays free, deterministic, and +fast (~30s per SC-005). + +**Alternatives considered**: +- Skip live tests entirely, rely on units: rejected — units don't catch + CLI argparse mistakes or report-write IO bugs. +- Run live tests against mem0 in CI: rejected — needs paid keys, burns + quota, and is intentionally out of scope per FR-011. + +--- + +## R10. Report Markdown layout + +**Decision**: Single `# OMP Eval Report` document with sections: +1. **Metadata** — dataset_hash, run_id, timestamp, OMP version, providers, + sample size, dry_run/live flag. +2. **Comparison Table** — one row per provider with columns: provider, + recall@1, recall@5, recall@10, MRR, ingest p50/p95 (ms), search p50/p95 + (ms), errors, total wall (s), notes. +3. **Per-Provider Notes** — flagged warnings (e.g., recall@k < 0.1 → + `[suspicious — verify configuration]`). +4. **Failure Traces** — a sub-section per provider listing the + `query_id`, error message, and trace timestamp for each failed query. +5. **Footer** — cost-model disclaimer + dataset license. + +**Rationale**: Maps cleanly to the spec acceptance scenarios. Markdown +renders correctly in GitHub release notes (the primary publication +target) and in any Markdown previewer. + +**Alternatives considered**: +- HTML report with charts: rejected — not GitHub-native, requires a JS + build step. +- JSON only: rejected — humans can't skim it; Markdown table is the + whole point of "publish-ready". + +--- + +## R11. `fact_id` round-trip across heterogeneous providers + +**Decision**: Stamp the `fact_id` on every ingested memory in **two** +redundant places so the runner can recover it from any adapter's +`SearchResult` regardless of how that provider exposes metadata: + +1. **Content prefix** (primary, always works): the runner ingests + `content = f"[fact_id={fact_id}] {original_content}"`. The runner + recovers the id with the regex `^\[fact_id=([^\]]+)\] ` against + `SearchResult.memory.content`. This works for every adapter because + `content` is a required field per OMP and is always echoed back. +2. **Tag** (secondary, supports debugging / cleanup): also pass + `tags=[*original_tags, f"fact:{fact_id}"]`. Adapters that round-trip + tags (postgres, supermemory) preserve this; adapters that don't + (mem0 v2 sometimes drops tags) silently degrade and the prefix path + takes over. + +The runner extracts via: +```python +m = re.match(r"^\[fact_id=([^\]]+)\] ", result.memory.content) +fact_id = m.group(1) if m else None # None means "not one of ours" +``` +Results without a recoverable `fact_id` are filtered from the top-k +list before scoring (treated as if the provider returned fewer +results); a counter on `ProviderResult.metrics.note` flags this when +non-zero. + +**Rationale**: Content is the only field every adapter is contract-bound +to round-trip verbatim (Constitution Principle III). Stamping in +content guarantees portability; the tag is a soft secondary signal that +also makes the `eval-{run_id}` data inspectable in the provider's UI. + +**Alternatives considered**: +- Trust adapter-specific metadata fields (`x-fact-id` extension): + rejected — supermemory and letta don't surface arbitrary `x-*` on + search results uniformly; would require per-adapter parsing. +- Stamp only in tags: rejected — mem0 v2 has been observed to drop + tag arrays in some response paths; would silently zero out recall. +- Use a separate `metadata` map: rejected — not standard on `SearchResult`. + +--- + +## R12. `wait_for_ingest` and embedder access from the runner + +**Decision**: Promote `wait_for_ingest` from adapter-private to a thin +pass-through method on the public `Memory` facade: +```python +class Memory: + def wait_for_ingest( + self, + ids: list[str], + user_id: str, + timeout: float | None = None, + ) -> None: + return self._adapter.wait_for_ingest(ids, user_id=user_id, timeout=timeout) +``` +The runner then calls `memory.wait_for_ingest(...)` exclusively; no +adapter-private access. For embedder configuration on the postgres +provider, the runner constructs the facade explicitly with the existing +local-only embedder (`embedder="hash"` — already the postgres adapter's +default fallback when no explicit embedder is configured), so postgres +remains zero-cost and zero-network. + +**Rationale**: M2.1 already established `wait_for_ingest` as the +canonical sync barrier for async-ingest providers (mem0, supermemory). +Exposing it on the facade is consistent with `add` / `search` / `get` / +etc. and avoids the eval kit poking at `_adapter`. It's a strict +addition — no existing caller is affected — so backward compatibility +(Principle III) is preserved. + +**Alternatives considered**: +- Access `memory._adapter.wait_for_ingest` directly: rejected — couples + the eval kit to a private attribute and sets a bad precedent. +- Add a sleep-based fallback: rejected — non-deterministic; defeats + SC-003 (reproducibility). +- Require users to wire the embedder explicitly: rejected — friction; + the postgres adapter already has a sensible local fallback. diff --git a/specs/004-eval-kit/spec.md b/specs/004-eval-kit/spec.md new file mode 100644 index 0000000..d22a403 --- /dev/null +++ b/specs/004-eval-kit/spec.md @@ -0,0 +1,248 @@ +# Feature Specification: M3.1 Eval Kit (`openmem-eval`) + +**Feature Branch**: `004-eval-kit` +**Created**: 2026-05-01 +**Status**: Draft +**Input**: User description: "M 3.1" +**Context**: Builds on M2.1 (live-API bridges for mem0 / supermemory / letta) by adding a +benchmark harness that produces an apples-to-apples comparison report across all +configured OMP adapters. + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 — Maintainer publishes a release-notes comparison table (Priority: P1) + +A maintainer is preparing a release announcement for OMP. They want a single +command that ingests a curated dataset into every supported provider, runs the +same retrieval queries against each, and prints a Markdown table comparing +recall, precision, and latency. The numbers must reflect real provider +behaviour — not mocks — because they will be published. + +**Why this priority**: Cross-provider numbers are the single most-requested +artifact for an interop SDK. Without them, OMP's "works with mem0 *and* +supermemory *and* letta *and* postgres" claim is unverified marketing copy. +This is the MVP. + +**Independent Test**: Run `openmem-eval --providers postgres,mem0 --live` with +both backends configured. The command exits 0 and writes +`eval-report.md` containing per-provider rows for recall@5, MRR, ingest +p50/p95, and search p50/p95. Re-running with the same dataset hash and same +providers reproduces the recall/MRR numbers within a documented tolerance +(latency naturally varies). + +**Acceptance Scenarios**: + +1. **Given** `MEM0_API_KEY` and `OMP_POSTGRES_URL` are set, **When** the user + runs `openmem-eval --providers postgres,mem0 --live --report out.md`, + **Then** `out.md` is written with one row per provider showing recall@k, + MRR, p50/p95 latency, and total ingest time. +2. **Given** any provider fails an individual query (timeout, quota, network), + **When** the run completes, **Then** the report still includes that + provider's partial numbers and an explicit `errors` column counting the + failures, and the process exits 0. + +--- + +### User Story 2 — Maintainer estimates cost before spending quota (Priority: P1) + +Before burning paid API quota, the maintainer wants to know how many requests +will be issued and a coarse cost estimate. They run the same command with +`--dry-run` and get a per-provider breakdown of operations + estimated cost, +with no live calls executed. + +**Why this priority**: Live runs against mem0/supermemory/letta cost real +money and burn rate-limited quota. The previous live-test work (T052) hit +supermemory rate limits; the harness must make accidental cost impossible. + +**Independent Test**: Run `openmem-eval --providers mem0,supermemory,letta --dry-run` +without any live keys configured. The command prints a table of expected +request counts per verb per provider plus a USD cost estimate, and exits 0 +without making any HTTP calls. + +**Acceptance Scenarios**: + +1. **Given** the user passes `--dry-run`, **When** the harness loads the + dataset and resolves provider configs, **Then** no HTTP requests are + made to any live provider and the printed estimate matches the dataset + size × verb count × per-provider cost-per-call constant. +2. **Given** `--live` is passed without `--yes`, **When** the estimated cost + exceeds a configurable threshold (default $1.00), **Then** the harness + prints the estimate and prompts for confirmation before proceeding. + +--- + +### User Story 3 — Maintainer iterates on a single cheap provider (Priority: P2) + +While iterating on the harness itself or debugging scorer logic, the +maintainer wants to run only against `postgres` (free, local) using a +shrunken dataset for fast feedback. They pass `--providers postgres --sample 10` +and get a full report in seconds. + +**Why this priority**: Without this, every iteration on the kit code itself +would burn live quota or wait minutes. This is a developer-experience +multiplier but not the headline feature. + +**Independent Test**: Run `openmem-eval --providers postgres --sample 10` +against a local Docker postgres. The command completes in under 30 seconds +and the resulting report shows postgres-only metrics computed against 10 +queries. + +**Acceptance Scenarios**: + +1. **Given** `--sample N` is passed, **When** the dataset loader runs, **Then** + exactly `N` queries are selected (deterministically by stable hash so + re-runs hit the same subset) and the report annotates the row as + `sample=N`. +2. **Given** `--providers` is omitted, **When** the harness runs, **Then** it + defaults to `postgres` only (the one provider with no key requirement). + +--- + +### Edge Cases + +- A provider's `wait_for_ingest` exceeds its budget for some facts → those + facts are excluded from the recall computation and counted in an + `ingest_failures` column; the run still completes. +- Network partition mid-run → per-query try/except records the error per + provider and continues; final report lists `errors` and computes metrics + over successful queries only with a footnote explaining sample size + reduction. +- Provider returns zero results for every query (mis-configured embeddings, + empty index after ingest race) → recall@k = 0 is reported as a normal + number, not an error; the report flags providers with recall@k < 0.1 as + `[suspicious — verify configuration]`. +- Same provider configured twice with different settings (e.g., two + postgres URLs comparing pgvector with/without HNSW) → harness allows + alias-suffixed provider names like `postgres:hnsw` and `postgres:flat`. +- Dataset file is missing or malformed → fail fast with a clear error + before any provider is contacted; do not partially process. +- User passes `--live` but no provider keys are set → harness prints a + per-provider `SKIP: missing $ENV_VAR` line and continues with whichever + providers *are* configured; if zero are configured, exits 1. + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: System MUST ship a curated dataset of approximately 50 facts + and 20 queries with gold-truth fact IDs, bundled inside the package as a + JSONL fixture so it works on a fresh checkout with no external download. +- **FR-002**: System MUST provide a CLI entry point `openmem-eval` (also + invokable as `python -m openmem.eval`) that accepts at minimum: + `--providers`, `--live`, `--dry-run`, `--report`, `--sample`, and `--yes`. +- **FR-003**: System MUST run live API calls only when `--live` is passed; + the default behaviour MUST be `--dry-run` (no network calls) so + accidental invocation has zero cost. +- **FR-004**: System MUST refuse to make paid API calls without first + printing the estimated request count and cost, and MUST require explicit + `--yes` confirmation when the estimate exceeds a configurable threshold + (default USD $1.00). +- **FR-005**: System MUST execute a fixed pipeline per provider per run: + ingest the dataset → wait for ingest completion (using + `adapter.wait_for_ingest`) → run the query set → score results → record + per-call latency. +- **FR-006**: System MUST compute and report at minimum the following + metrics per provider: recall@k (k=1, 5, 10), MRR, ingest p50 / p95 + latency, search p50 / p95 latency, total wall-clock time, error count. +- **FR-007**: System MUST emit a Markdown report at the path given by + `--report` (default `eval-report.md`) containing a comparison table, a + metadata block (dataset hash, OMP version, run timestamp, per-provider + config hash), and a per-query trace section for failed queries only. +- **FR-008**: *(removed — caching dropped; every live run re-ingests so + reported numbers always reflect the provider's current behaviour.)* +- **FR-009**: System MUST handle per-query and per-provider failures + gracefully: a failed query for one provider MUST NOT abort the run for + other providers, and per-provider failures MUST be recorded in the + report rather than raised. +- **FR-010**: System MUST detect missing per-provider environment + variables (`MEM0_API_KEY`, `SUPERMEMORY_API_KEY`, `LETTA_API_KEY`, + `OMP_POSTGRES_URL`) and emit a `SKIP: missing $ENV` line for each + unconfigured provider rather than raising. +- **FR-011**: System MUST NOT be wired into CI pull-request or main-branch + workflows; it is a manually-invoked tool only. (A documentation note in + the README MUST state this explicitly.) +- **FR-012**: System MUST log every live API call to a JSONL trace file + (default `eval-trace.jsonl`) with timestamp, provider, verb, latency, + status, and a redacted request hash, to support post-hoc debugging. +- **FR-013**: Users MUST be able to pass `--sample N` to deterministically + reduce the query set to the first N entries by stable hash so cheap + iteration is possible. +- **FR-014**: System MUST namespace every memory it ingests under a + unique per-run `user_id` (e.g., `eval-{run_id}`) so eval data never + collides with real user data and can be cleaned up after the run. +- **FR-015**: System MUST provide a `--cleanup` flag that, when passed, + deletes every memory created by the most recent run from each provider + before exiting. + +### Key Entities + +- **Dataset**: A versioned JSONL fixture of `{fact_id, content, tags}` + records plus a `{query_id, query, gold_fact_ids}` set. Identified by + SHA-256 of its serialised contents (the `dataset_hash`). +- **RunConfig**: The user's invocation parameters: providers, sample size, + live/dry-run, output paths, run_id (auto-generated UUID). +- **ProviderResult**: Per-provider output of a run, containing ingest + latencies, per-query results (top-k fact IDs + latencies), error list, + and computed metrics. +- **Report**: The Markdown artifact emitted at `--report`, containing the + comparison table, run metadata, and failure traces. +- **TraceEntry**: One record in the JSONL trace, capturing every API call + made during a live run. +- **CostModel**: A static table of estimated cost-per-call per provider + per verb, used by `--dry-run` to print the cost estimate. Values are + best-effort approximations and explicitly documented as such in the + report. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: A maintainer can produce a four-provider comparison report + (`postgres + mem0 + supermemory + letta`) with one command and total + wall-clock under 5 minutes, given all keys are configured. +- **SC-002**: The default invocation (`openmem-eval` with no flags) makes + zero network calls and prints a cost estimate; running it without keys + set never produces a paid charge. +- **SC-003**: Re-running the same command against the same dataset with + the same providers reproduces recall@k and MRR exactly (latency + metrics naturally vary). +- **SC-004**: A `--dry-run` invocation against all four providers + completes in under 5 seconds and prints a per-provider request count + and USD cost estimate. +- **SC-005**: A `--providers postgres --sample 10` run against local + Docker postgres completes in under 30 seconds end-to-end including + ingest, search, and report write. +- **SC-006**: When any single provider fails 100% of its queries, the + other providers' metrics are still produced and the report clearly + flags the failed provider; the process exit code is 0 (success). +- **SC-007**: The bundled dataset is small enough that one full live run + across all four providers stays under the documented cost ceiling + (USD $0.50 per run with current published pricing). + +## Assumptions + +- The maintainer running live evaluations has API keys for the providers + they want to compare and is responsible for the cost; the harness only + enforces visibility and confirmation, not spending limits. +- The four adapters in scope are the ones already shipping in M2.1: + `postgres`, `mem0`, `supermemory`, `letta`. The kit must be + extensible to additional adapters but that is out of scope here. +- The bundled dataset will be hand-curated for diversity (different + topics, different query phrasings) but is small (~50 facts) and is + not intended to substitute for domain-specific evaluation by users. +- "Recall@k" is computed as `|top_k ∩ gold_fact_ids| / |gold_fact_ids|` + per query, then averaged across queries. MRR is computed in the + standard way over the rank of the first gold hit. +- A local-only embedder (already shipped via `openmem.adapters.embedder`) + is used for postgres so it remains free; live providers use whichever + embedder they manage internally. +- The cost model is a static table maintained by hand based on + publicly-listed pricing at the time of writing; it is documented in + the report as approximate and not a guarantee. +- The kit is not a CI gate. Benchmarks intentionally do not run on + pull requests or pushes to main, because they need keys and time. +- Dataset versioning lives inside the package (`openmem/eval/datasets/`) + so a `pip install --upgrade openmem` ships any dataset updates. +- The harness is sequential per provider but may run providers in + parallel via `asyncio.gather` once correctness is established; + parallelism is a P3 concern and not required for the MVP. diff --git a/specs/004-eval-kit/tasks.md b/specs/004-eval-kit/tasks.md new file mode 100644 index 0000000..a235e66 --- /dev/null +++ b/specs/004-eval-kit/tasks.md @@ -0,0 +1,183 @@ +--- +description: "Tasks for M3.1 Eval Kit (`openmem-eval`)" +--- + +# Tasks: M3.1 Eval Kit (`openmem-eval`) + +**Input**: Design documents from `/specs/004-eval-kit/` +**Prerequisites**: [plan.md](plan.md), [spec.md](spec.md), [research.md](research.md), [data-model.md](data-model.md), [contracts/cli.md](contracts/cli.md), [quickstart.md](quickstart.md) +**Tests**: Included — the spec calls for unit tests on every component (scorer, cost, dataset) plus a CLI dry-run integration test and an opt-in postgres live smoke test. + +**Format**: `- [ ] TID [P?] [Story?] Description` + +--- + +## Phase 1: Setup + +- [X] T001 Create the eval sub-package skeleton: `sdk-python/openmem/eval/__init__.py`, `sdk-python/openmem/eval/__main__.py` (re-exports `main` from `cli.py`), and the empty test directory `sdk-python/tests/eval/__init__.py`. +- [X] T002 Register the CLI entry point: in `sdk-python/pyproject.toml` add `openmem-eval = "openmem.eval.cli:main"` to the `[project.scripts]` table next to the existing `omp-validate-spec` entry, and include `openmem/eval/datasets/*.jsonl` in `[tool.setuptools.package-data]` so the bundled dataset ships with `pip install`. +- [X] T003 [P] Update [sdk-python/README.md](sdk-python/README.md) with a one-line note under "Tooling" stating that `openmem-eval` is a manual benchmark tool, never run in CI, and link to [specs/004-eval-kit/quickstart.md](specs/004-eval-kit/quickstart.md). + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +These types and IO primitives are imported by every later module. + +- [X] T004 [P] Implement dataclasses for the data model in `sdk-python/openmem/eval/types.py`: `Fact`, `Query`, `Dataset`, `RunConfig`, `QueryResult`, `ErrorRecord`, `Metrics`, `ProviderResult`. All immutable (`frozen=True`) where practical. Match the field names and types defined in [data-model.md](specs/004-eval-kit/data-model.md). +- [X] T005 [P] Bundle the default dataset: write `sdk-python/openmem/eval/datasets/default/facts.jsonl` (~50 facts, hand-curated for topical diversity — recipes, schedules, places, identifiers, preferences) and `sdk-python/openmem/eval/datasets/default/queries.jsonl` (~20 queries with `gold_fact_ids` referencing only existing fact ids). +- [X] T006 Implement `sdk-python/openmem/eval/dataset.py`: `load_default() -> Dataset`, `load_path(path) -> Dataset`, `dataset_hash(dataset) -> str` (SHA-256 of canonical sorted-line concatenation, first 12 hex chars). Use `importlib.resources.files("openmem.eval.datasets.default")` to read the bundled files. Validate uniqueness of `fact_id`, non-empty `content`, and that every entry in `gold_fact_ids` resolves to a known fact. + +**Checkpoint**: Dataset loads, types compile. User story work can begin. + +--- + +## Phase 3: User Story 1 — Maintainer publishes a release-notes comparison table (Priority: P1) 🎯 MVP + +**Goal**: One command — `openmem-eval --providers postgres,mem0 --live` — produces a Markdown comparison report with recall@k, MRR, and p50/p95 latency per provider. + +**Independent Test**: With `OMP_POSTGRES_URL` and `MEM0_API_KEY` set, the command exits 0 and writes `eval-report.md` containing rows for both providers. Re-running the same command reproduces recall@k and MRR exactly (latency varies). + +### Tests for User Story 1 + +- [X] T007 [P] [US1] Tests for the dataset loader and hashing in `sdk-python/tests/eval/test_dataset.py` covering: bundled load returns ≥50 facts and ≥20 queries; `dataset_hash` is deterministic across two loads; mutating one fact changes the hash; gold-fact-id pointing to a missing fact raises a clear error. +- [X] T008 [P] [US1] Tests for the scorer in `sdk-python/tests/eval/test_scorer.py` covering: recall@1 / @5 / @10 with multiple golds; MRR with gold at rank 1, 3, none; macro-average across queries; empty result list returns recall=0 and MRR=0; latency p50/p95 against a known list. +- [X] T009 [P] [US1] Tests for the report writer in `sdk-python/tests/eval/test_report.py` covering: Markdown table renders one row per provider with the seven required columns; metadata block includes dataset_hash, run_id, OMP version, timestamp; suspicious-flag block appears when recall@5 < 0.1; failure traces section appears only when `errors` is non-empty. +- [X] T010 [US1] Integration test for the runner against a stub adapter in `sdk-python/tests/eval/test_runner_with_stub.py` covering: (a) ingest pipeline calls `add` for every fact with the `[fact_id=...]` prefix and `fact:` tag stamped per [research.md §R11](specs/004-eval-kit/research.md); (b) `Memory.wait_for_ingest` is invoked with all returned ids; (c) search runs once per query and the runner recovers `fact_id` via the prefix regex; (d) per-call latencies are recorded; (e) partial failures (one query raises) populate `ErrorRecord` and the run still completes; (f) **reproducibility (SC-003)** — running the pipeline twice with the same stub returns equal `Metrics` dicts; (g) **multi-provider continuation (SC-006)** — with two stub providers where providerA raises on every search and providerB succeeds, the run exits 0, providerA's `ProviderResult.status == "failed"`, providerB's metrics are populated. +- [X] T011 [US1] Live smoke test in `sdk-python/tests/eval/test_cli_live_postgres.py` that invokes the CLI end-to-end against the local Docker postgres (skip when `OMP_POSTGRES_URL` is unset, matching `test_postgres_specific.py` convention). Asserts exit 0, `eval-report.md` exists, contains a `postgres` row with non-zero ingest latency, and the run completes in ≤ 30 seconds wall-clock (validates SC-005). + +### Implementation for User Story 1 + +- [X] T012 [P] [US1] Implement `sdk-python/openmem/eval/scorer.py`: `recall_at_k(top_k_fact_ids, gold_fact_ids, k) -> float`, `mrr(top_k_fact_ids, gold_fact_ids) -> float`, `compute_metrics(provider_result, ks=(1, 5, 10)) -> Metrics`. Percentiles via `statistics.quantiles(n=100)`; safe handling of empty inputs (return 0.0). +- [X] T013 [P] [US1] Implement `sdk-python/openmem/eval/cost.py`: the `COST_USD_PER_CALL` static dict covering verbs `add`, `search`, `get`, **and `delete`** (postgres free; mem0/letta per-call estimates from [research.md](specs/004-eval-kit/research.md) §R3; `delete` rows: postgres 0, mem0 0.0001, supermemory 0, letta 0.0001) and `estimate(provider, verb, n_calls) -> float`. Unknown `(provider, verb)` returns 0.0 and emits a stderr warning once per pair. +- [X] T014 [P] [US1] Implement `sdk-python/openmem/eval/trace.py`: `TraceWriter(path)` with `write(provider, verb, latency_ms, status, payload)` that appends a JSONL line. Hash the payload via SHA-256 (first 12 hex) — never log raw payload content. Open the file lazily on first write so dry-runs leave no trace file. +- [X] T015 [US1] Implement `sdk-python/openmem/eval/runner.py`: `run_provider(provider_name, dataset, run_config, trace_writer) -> ProviderResult`. The pipeline is: resolve `Memory(provider=...)` (postgres uses `embedder="hash"` per [research.md §R12](specs/004-eval-kit/research.md) for zero-cost local embedding) → for each fact, stamp `content = f"[fact_id={fact.fact_id}] {fact.content}"` and `tags = [*fact.tags, f"fact:{fact.fact_id}"]` per [research.md §R11](specs/004-eval-kit/research.md), call `add()` (record latency, capture per-fact errors) → `memory.wait_for_ingest(memory_ids, user_id=user_id)` (using the new facade pass-through from T015a; capture timeout as `ingest_failures`) → for each query `search(query, user_id, limit=10)` (parse `fact_id` from `result.memory.content` via the regex `^\[fact_id=([^\]]+)\] `; results without a match are filtered out before scoring, plus latency, plus error). Use `user_id = f"eval-{run_config.run_id}"` for every call. +- [X] T015a [US1] Add `Memory.wait_for_ingest(ids, user_id, timeout=None)` thin pass-through to `sdk-python/openmem/memory.py` per [research.md §R12](specs/004-eval-kit/research.md): forwards directly to `self._adapter.wait_for_ingest(ids, user_id=user_id, timeout=timeout)`. Strictly additive; no existing callers affected. Add a unit test in `sdk-python/tests/test_memory_facade.py` confirming it forwards through the stub adapter. +- [X] T016 [US1] Implement `sdk-python/openmem/eval/report.py`: `write_report(path, run_config, dataset, results)` emitting the Markdown sections defined in [research.md](specs/004-eval-kit/research.md) §R10 — Metadata table, Comparison table, Per-Provider Notes (suspicious flags), Failure Traces, Footer. Format latencies as `p50/p95 ms`; format recall to two decimal places; format MRR to two decimal places. +- [X] T017 [US1] Implement `sdk-python/openmem/eval/cli.py` minimal flag set for US1: `--providers`, `--live`, `--report`, `--trace`, `--verbose`. `main(argv=None) -> int` returns the documented exit codes from [contracts/cli.md](specs/004-eval-kit/contracts/cli.md). On `--live` without keys, print `[skip] : missing $ENV_VAR` to stderr per provider and exit 1 only when zero providers remain. +- [X] T018 [US1] Wire `sdk-python/openmem/eval/__main__.py` to call `cli.main()` so `python -m openmem.eval` works equivalently to the `openmem-eval` script entry. +- [X] T018a [US1] Implement `--cleanup` (FR-015) in `sdk-python/openmem/eval/cli.py` and `sdk-python/openmem/eval/cleanup.py`: list every memory under `user_id = f"eval-{run_id}"` and call `Memory(provider=...).delete(memory_id)` for each, logging failures as warnings. Add a unit test in `sdk-python/tests/eval/test_cleanup.py` against a stub adapter that verifies all listed `memory_id`s are deleted and that pagination is honoured. Note: placed in Phase 3 (not Polish) because FR-015 is a hard requirement and `eval-{run_id}` namespacing in T015 exists specifically to enable it. + +**Checkpoint**: US1 MVP done — a maintainer with `OMP_POSTGRES_URL` and `MEM0_API_KEY` can publish a real comparison table. + +--- + +## Phase 4: User Story 2 — Maintainer estimates cost before spending quota (Priority: P1) + +**Goal**: `openmem-eval --dry-run` (the default) prints a per-provider request-count + USD-cost table without making any HTTP calls. `--live` above the threshold prompts for confirmation. + +**Independent Test**: With no API keys set, `openmem-eval --providers mem0,supermemory,letta --dry-run` exits 0 and prints a cost-estimate table; verify zero network calls were made (mock httpx and assert no requests). + +### Tests for User Story 2 + +- [X] T019 [P] [US2] Tests for the cost model in `sdk-python/tests/eval/test_cost.py` covering: known `(provider, verb)` returns the static value × n_calls; unknown pair returns 0.0 and emits exactly one stderr warning per pair; total estimate sums correctly across providers; **cost-ceiling guard (SC-007)** — `estimate` for a full four-provider run (`postgres + mem0 + supermemory + letta`) against the bundled default dataset returns ≤ USD 0.50. +- [X] T020 [P] [US2] Tests for CLI dry-run mode in `sdk-python/tests/eval/test_cli_dry_run.py` covering: `--dry-run` makes zero adapter instantiations (patch `openmem.Memory.__init__` to raise); estimate table includes one row per provider with adds/searches/waits counts derived from dataset size; `--providers` defaulting to `postgres` produces a $0.00 estimate; missing keys are not enforced in dry-run mode (no skip lines printed). +- [X] T021 [P] [US2] Tests for the confirmation prompt in `sdk-python/tests/eval/test_cli_confirm.py` covering: estimated cost ≤ threshold proceeds without prompt; cost > threshold with TTY prompts and accepts `y` (proceeds) / `n` (exits 3); cost > threshold without TTY and without `--yes` exits 3; cost > threshold with `--yes` proceeds without prompting. + +### Implementation for User Story 2 + +- [X] T022 [US2] Extend `sdk-python/openmem/eval/cli.py` with `--dry-run` (default true), `--cost-threshold` (default 1.00), and `--yes`. Add a `dry_run_estimate(run_config, dataset) -> dict[str, dict[str, int]]` helper that returns per-provider verb counts purely from dataset size (no adapter calls). Print the estimate table to stdout in the format defined in [contracts/cli.md](specs/004-eval-kit/contracts/cli.md). +- [X] T023 [US2] Implement `sdk-python/openmem/eval/confirm.py`: `confirm_or_exit(estimated_cost_usd, threshold, yes, isatty) -> None`. Returns silently when cost ≤ threshold or `yes` is true; prompts on TTY otherwise; raises `SystemExit(3)` on refusal or non-TTY without `--yes`. Wire into `cli.main` immediately before any adapter is instantiated. +- [X] T024 [US2] In `sdk-python/openmem/eval/cli.py`, store `--live` as a regular argparse flag (default False) and derive `dry_run = not args.live` rather than using a mutually-exclusive group. Reject the explicit combination `--dry-run --live` at parse time with a clear error (`--dry-run and --live are mutually exclusive`). Document this wiring decision inline as a comment referencing [contracts/cli.md](specs/004-eval-kit/contracts/cli.md) §"Mutual exclusion". + +**Checkpoint**: US2 done — accidental quota burn is impossible. Default invocation makes zero calls. + +--- + +## Phase 5: User Story 3 — Maintainer iterates on a single cheap provider (Priority: P2) + +**Goal**: `--providers postgres --sample 10` runs the full pipeline end-to-end against local postgres with 10 queries in under 30 seconds. + +**Independent Test**: With `OMP_POSTGRES_URL` set and Docker postgres up, run `openmem-eval --providers postgres --sample 10 --live`; assert the run completes in ≤ 30s and the report annotates `sample=10`. + +### Tests for User Story 3 + +- [X] T025 [P] [US3] Tests for `--sample` in `sdk-python/tests/eval/test_dataset.py` (extends T007's file): `sample(dataset, n)` returns exactly `n` queries; selection is deterministic (same `n` ⇒ same query_ids across runs); `n` larger than total returns all queries with a warning; `n=0` raises ValueError. + +### Implementation for User Story 3 + +- [X] T027 [US3] Add `sample(dataset, n) -> Dataset` to `sdk-python/openmem/eval/dataset.py`. Selection is the first `n` queries after sorting by `query_id`'s SHA-256 hash (stable, deterministic). Return a new `Dataset` with all original facts but only the sampled queries. +- [X] T031 [US3] Update `sdk-python/openmem/eval/report.py` to annotate the comparison-table caption with `sample=N` whenever `run_config.sample` is set. + +*(T026, T028, T029, T030 removed — caching dropped per project decision; every live run re-ingests so reported numbers always reflect current provider behaviour.)* + +**Checkpoint**: US3 done — cheap iteration loop is sub-30s on postgres-only. + +--- + +## Phase 6: Polish & Cross-Cutting Concerns + +- [X] T032 [P] Add `--version` to `sdk-python/openmem/eval/cli.py` printing `openmem X.Y.Z, dataset default-` and exiting 0. Add a one-line test in `tests/eval/test_cli_dry_run.py`. +- [X] T033 [P] Suspicious-recall flag wiring: in `sdk-python/openmem/eval/scorer.py` set `Metrics.note = "suspicious — recall@5 < 0.1"` when applicable; assert the report renders it (extends T009). +- [X] T034 [P] Run [specs/004-eval-kit/quickstart.md](specs/004-eval-kit/quickstart.md) end-to-end on a local checkout: install, `openmem-eval` (default), `--dry-run` for all four providers, `--live --providers postgres --sample 10`. Note any drift in a follow-up task; do NOT silently fix the quickstart. +- [X] T035 Confirm coverage stays green: `python -m pytest sdk-python/tests --cov=openmem --cov-report=term --timeout=60` shows ≥ 85% total, and the new `openmem.eval` modules each report ≥ 85% individually. Add narrowly-targeted tests for any module under 85% before committing. +- [X] T036 README pass: ensure [sdk-python/README.md](sdk-python/README.md) Tooling section + [README.md](README.md) at repo root mention `openmem-eval` exactly once, link to the quickstart, and explicitly state it never runs in CI (FR-011). + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Phase 1 (Setup)**: T001 → T002. T003 in parallel with T002. +- **Phase 2 (Foundational)**: T004, T005 in parallel; T006 depends on both. **Blocks all later phases.** +- **Phase 3 (US1)**: T007–T011 (tests) in parallel before implementation. T012, T013, T014 in parallel. T015a depends on the existing facade (no eval deps); T015 depends on T012+T013+T014+T015a. T016 depends on T012. T017 depends on T015+T016. T018 depends on T017. T018a depends on T018 only (lists memories by `user_id` — no cache). +- **Phase 4 (US2)**: T019, T020, T021 in parallel before implementation. T022 → T023 → T024. +- **Phase 5 (US3)**: T025 before T027. T031 standalone. +- **Phase 6 (Polish)**: T032, T033 in parallel. T034, T035, T036 sequential at the end. + +### User Story Dependencies + +- **US1 (P1)**: needs Phases 1+2 only. +- **US2 (P1)**: needs Phases 1+2 + the `cli.py` skeleton from T017 (so US1 must reach T017 first). +- **US3 (P2)**: needs Phases 1+2 + the runner from T015; cleanest if US1 is fully done first. + +### Within Each User Story + +- Tests (T007–T011, T019–T021, T025) are written first and MUST fail before implementation. +- Models / utilities (scorer, cost, trace, dataset.sample) before runner integration. +- Runner before CLI wiring. +- CLI wiring before live smoke test. + +### Parallel Opportunities + +- T004 (types) ∥ T005 (dataset files). +- T007 ∥ T008 ∥ T009 (independent test files). +- T012 (scorer) ∥ T013 (cost) ∥ T014 (trace). +- T019 ∥ T020 ∥ T021. +- T025 ∥ T026. +- T032 ∥ T033 ∥ T034. + +--- + +## Parallel Example: User Story 1 implementation kickoff + +Once tests T007–T011 are red: + +```powershell +# Three independent files, no shared state — run agents in parallel: +# Agent A: implement openmem/eval/scorer.py to make test_scorer.py green +# Agent B: implement openmem/eval/cost.py to make test_cost.py (in US2) compile +# Agent C: implement openmem/eval/trace.py (no failing test yet, but unblocks runner) +``` + +Then sequentially: T015 (runner) → T016 (report) → T017 (cli) → T018 (entrypoint). + +--- + +## Implementation Strategy + +**MVP scope** = Phases 1 + 2 + 3 (User Story 1 only). At MVP a maintainer can +already publish a comparison table for `postgres + mem0` — every other story +adds safety (US2: cost guard) or developer-experience polish (US3: cheap +iteration). Recommended order: + +1. Land MVP (T001–T018), ship the first real comparison report. +2. Layer in US2 (T019–T024) before any wider-audience use, so the cost guard + protects new contributors who pull and run it. +3. Layer in US3 (T025–T031) when iterating on the dataset or scorer. +4. Polish (T032–T037) before the M3.1 release tag. + +**Format validation**: every task above is `- [ ] TID [P?] [US?] description` +with an explicit file path or paths. Setup, Foundational, and Polish tasks +intentionally carry no `[USx]` label per the format rules.