diff --git a/.gitignore b/.gitignore index bd2c35c..9900946 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,6 @@ Thumbs.db .env.* !.env.example -# Eval kit run artifacts +# Eval kit run artifacts (committed samples live in docs/eval/sample-*) eval-*.md eval-*.jsonl diff --git a/README.md b/README.md index a8fee66..3470d3d 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ print(ctx.text) ## Tooling -- `openmem-eval` — manual benchmark harness comparing recall, MRR, and latency across configured providers. **Never runs in CI.** Default invocation is a dry-run that makes zero network calls. See [specs/004-eval-kit/quickstart.md](specs/004-eval-kit/quickstart.md). +- `openmem-eval` — manual benchmark harness comparing recall, MRR, and latency across configured providers. **Never runs in CI.** Default invocation is a dry-run that makes zero network calls. See [specs/004-eval-kit/quickstart.md](specs/004-eval-kit/quickstart.md) for usage and [docs/eval/](docs/eval/README.md) for a sample report and trace from a real postgres run. ## License diff --git a/docs/eval/README.md b/docs/eval/README.md new file mode 100644 index 0000000..567a996 --- /dev/null +++ b/docs/eval/README.md @@ -0,0 +1,50 @@ +# OMP Eval Kit — sample outputs + +This directory contains **committed sample artifacts** from the +`openmem-eval` benchmarking kit so you can see what a run looks like +without installing or running anything. + +## Files + +| File | What it is | How it was produced | +|---|---|---| +| [sample-report.md](sample-report.md) | Markdown results table — recall@k, MRR, ingest/search latency percentiles, error count, cost estimate | `openmem-eval --providers postgres --live --yes --cleanup` against a local `pgvector/pgvector:pg16` container | +| [sample-trace.jsonl](sample-trace.jsonl) | One JSON line per verb call: `add` × 50 + `search` × 20 = 70 lines. Payloads are SHA-256 hashed (12-hex prefix) so the trace is safe to share. | Same run as above, written via `--trace` | + +## Reproducing locally + +```bash +# 1. Start a local pgvector container +docker run -d --name omp-postgres -p 5432:5432 \ + -e POSTGRES_PASSWORD=postgres pgvector/pgvector:pg16 + +# 2. Point the eval at it and run the bundled 50-fact / 20-query dataset +export OMP_POSTGRES_URL=postgresql://postgres:postgres@localhost:5432/postgres +openmem-eval --providers postgres --live --yes --cleanup \ + --report eval-report.md --trace eval-trace.jsonl +``` + +Cost: **$0.00** (postgres + offline `FakeEmbedder`). Wall time: ~1s. + +## Reading the report + +``` +| Provider | Status | recall@1 | recall@5 | MRR | ingest p50/p95/p99 (ms) | search p50/p95/p99 (ms) | errors | est. cost | +| postgres | ok | 0.750 | 0.800 | 0.775 | 8.6/12.4/24.5 | 5.1/6.4/6.5 | 0 | $0.0000 | +``` + +- **recall@k** — fraction of queries where any gold fact appears in the top-`k` results. +- **MRR** — Mean Reciprocal Rank across all queries. +- **Latency percentiles** — wall-clock time for each `add`/`search` call, in milliseconds. +- **est. cost** — derived from per-verb pricing in `openmem.eval.cost`. Postgres is $0; paid providers (mem0, supermemory, letta) are estimated **before** the run and gated by a confirmation prompt above `--cost-threshold` (default $1.00). + +## Notes on the bundled dataset + +- 50 facts × 20 queries; SHA-256 dataset hash `5c2a0d95915a` is recorded in every report so two runs of the same dataset are directly comparable. +- The hash-based offline embedder (`FakeEmbedder`) caps recall@5 around 0.80 because some queries share few literal tokens with their gold facts. Production deployments using `OpenAIEmbedder` typically reach ~0.95+ recall on the same dataset. + +## See also + +- [Spec & design](../../specs/004-eval-kit/) — full requirements, plan, tasks +- [Quickstart](../../specs/004-eval-kit/quickstart.md) — full CLI reference and exit-code table +- [Source](../../sdk-python/openmem/eval/) — runner, scorer, trace, report, cost, cleanup diff --git a/docs/eval/sample-report.md b/docs/eval/sample-report.md new file mode 100644 index 0000000..e2c6740 --- /dev/null +++ b/docs/eval/sample-report.md @@ -0,0 +1,13 @@ +# OMP Eval Report + +- **Run ID**: `d4c7881a1364` +- **Mode**: live +- **Dataset hash**: `5c2a0d95915a` +- **Generated**: 2026-05-01T14:58:30.132399+00:00 +- **Providers**: postgres + +## Results + +| Provider | Status | recall@1 | recall@5 | MRR | ingest p50/p95/p99 (ms) | search p50/p95/p99 (ms) | errors | est. cost | +|---|---|---:|---:|---:|---|---|---:|---:| +| postgres | ok | 0.750 | 0.800 | 0.775 | 8.6/12.4/24.5 | 5.1/6.4/6.5 | 0 | $0.0000 | diff --git a/docs/eval/sample-trace.jsonl b/docs/eval/sample-trace.jsonl new file mode 100644 index 0000000..40e7dc8 --- /dev/null +++ b/docs/eval/sample-trace.jsonl @@ -0,0 +1,70 @@ +{"ts":1777647509.574082,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":33.461,"payload_hash":"bb603645dbb2","fact_id":"f-0001"} +{"ts":1777647509.5886862,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":15.088,"payload_hash":"5fc23df8bf0d","fact_id":"f-0002"} +{"ts":1777647509.6000168,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":11.242,"payload_hash":"4d273b69d664","fact_id":"f-0003"} +{"ts":1777647509.6111352,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":10.237,"payload_hash":"f23b1e4a4366","fact_id":"f-0004"} +{"ts":1777647509.6202528,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":11.983,"payload_hash":"dccc8c4b9928","fact_id":"f-0005"} +{"ts":1777647509.6302006,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":12.566,"payload_hash":"f2badb4a1dcf","fact_id":"f-0006"} +{"ts":1777647509.6428952,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":10.799,"payload_hash":"0673389325dd","fact_id":"f-0007"} +{"ts":1777647509.652893,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":9.263,"payload_hash":"079c3514556a","fact_id":"f-0008"} +{"ts":1777647509.6601303,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":11.96,"payload_hash":"6b8b493ac0d4","fact_id":"f-0009"} +{"ts":1777647509.6781976,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":9.668,"payload_hash":"fe0ffc7b1484","fact_id":"f-0010"} +{"ts":1777647509.690411,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":11.763,"payload_hash":"539519f347fd","fact_id":"f-0011"} +{"ts":1777647509.700334,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":11.189,"payload_hash":"f3dbc00cb9ec","fact_id":"f-0012"} +{"ts":1777647509.7104833,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":9.598,"payload_hash":"df08c35d82eb","fact_id":"f-0013"} +{"ts":1777647509.718719,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":8.125,"payload_hash":"cf5272b89ab0","fact_id":"f-0014"} +{"ts":1777647509.7302148,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":10.418,"payload_hash":"d85dedf3bda0","fact_id":"f-0015"} +{"ts":1777647509.7382276,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":8.531,"payload_hash":"5272ad0a369f","fact_id":"f-0016"} +{"ts":1777647509.748475,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":9.975,"payload_hash":"e60a32415247","fact_id":"f-0017"} +{"ts":1777647509.7603064,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":12.268,"payload_hash":"b92fd338ddaf","fact_id":"f-0018"} +{"ts":1777647509.7703865,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":11.291,"payload_hash":"07d54ff990eb","fact_id":"f-0019"} +{"ts":1777647509.7831478,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":11.156,"payload_hash":"6c95931a18ba","fact_id":"f-0020"} +{"ts":1777647509.7943177,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":10.43,"payload_hash":"badec07ce02f","fact_id":"f-0021"} +{"ts":1777647509.8048534,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":11.045,"payload_hash":"9521151c9f33","fact_id":"f-0022"} +{"ts":1777647509.8102825,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":10.698,"payload_hash":"5ff0c313e1e9","fact_id":"f-0023"} +{"ts":1777647509.8277512,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":11.225,"payload_hash":"b7cb8df8c722","fact_id":"f-0024"} +{"ts":1777647509.8387074,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":11.728,"payload_hash":"ff1fdf5dc866","fact_id":"f-0025"} +{"ts":1777647509.8502166,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":10.669,"payload_hash":"406ef2045889","fact_id":"f-0026"} +{"ts":1777647509.857707,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":7.815,"payload_hash":"9a15098b5efd","fact_id":"f-0027"} +{"ts":1777647509.8624995,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":8.742,"payload_hash":"662963ad89a2","fact_id":"f-0028"} +{"ts":1777647509.8750973,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":8.305,"payload_hash":"abfc48989ba9","fact_id":"f-0029"} +{"ts":1777647509.880285,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":7.835,"payload_hash":"7f37ad57e2c9","fact_id":"f-0030"} +{"ts":1777647509.880285,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":6.866,"payload_hash":"f93f450ffd89","fact_id":"f-0031"} +{"ts":1777647509.8976698,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":7.795,"payload_hash":"c8e5d7b24e25","fact_id":"f-0032"} +{"ts":1777647509.904722,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":6.49,"payload_hash":"8905f18a1fb0","fact_id":"f-0033"} +{"ts":1777647509.9103463,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":7.543,"payload_hash":"de6ca38ec1de","fact_id":"f-0034"} +{"ts":1777647509.9103463,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":5.916,"payload_hash":"1ae46b5236e1","fact_id":"f-0035"} +{"ts":1777647509.92356,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":6.386,"payload_hash":"e81d5adacbcf","fact_id":"f-0036"} +{"ts":1777647509.930929,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":6.669,"payload_hash":"6dd331b7721f","fact_id":"f-0037"} +{"ts":1777647509.9370384,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":6.127,"payload_hash":"0b94417dd1f6","fact_id":"f-0038"} +{"ts":1777647509.9439006,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":6.496,"payload_hash":"f72d638e87e5","fact_id":"f-0039"} +{"ts":1777647509.950408,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":6.283,"payload_hash":"8f0c1863d7af","fact_id":"f-0040"} +{"ts":1777647509.9560988,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":5.953,"payload_hash":"fe21b6595f79","fact_id":"f-0041"} +{"ts":1777647509.9600883,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":7.597,"payload_hash":"3307b689c723","fact_id":"f-0042"} +{"ts":1777647509.9702249,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":6.444,"payload_hash":"3c2e915f19ed","fact_id":"f-0043"} +{"ts":1777647509.9782808,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":8.012,"payload_hash":"93388db3b74f","fact_id":"f-0044"} +{"ts":1777647509.9857953,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":7.043,"payload_hash":"1b00d902e59c","fact_id":"f-0045"} +{"ts":1777647509.994153,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":8.08,"payload_hash":"6dfb451ea98f","fact_id":"f-0046"} +{"ts":1777647510.0016747,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":8.348,"payload_hash":"6b8bcf303bb3","fact_id":"f-0047"} +{"ts":1777647510.0101273,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":8.495,"payload_hash":"0b1548f41eca","fact_id":"f-0048"} +{"ts":1777647510.0101273,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":8.117,"payload_hash":"5d6669d0054f","fact_id":"f-0049"} +{"ts":1777647510.0273886,"run_id":"d4c7881a1364","provider":"postgres","verb":"add","latency_ms":7.991,"payload_hash":"178b28818d0e","fact_id":"f-0050"} +{"ts":1777647510.033952,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":6.537,"payload_hash":"d9b0659f6de5","result_count":5,"query_id":"q-0001"} +{"ts":1777647510.038954,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":5.396,"payload_hash":"152e58d46e21","result_count":5,"query_id":"q-0002"} +{"ts":1777647510.0442173,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":5.096,"payload_hash":"d7f4c29e1719","result_count":5,"query_id":"q-0003"} +{"ts":1777647510.0489588,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":4.461,"payload_hash":"9ae71db0e2b6","result_count":5,"query_id":"q-0004"} +{"ts":1777647510.0524812,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":3.827,"payload_hash":"c820862687e8","result_count":5,"query_id":"q-0005"} +{"ts":1777647510.0569952,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":4.62,"payload_hash":"7d4693c3b8d6","result_count":5,"query_id":"q-0006"} +{"ts":1777647510.0627866,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":5.625,"payload_hash":"5468c5963c0e","result_count":5,"query_id":"q-0007"} +{"ts":1777647510.0684743,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":4.813,"payload_hash":"f2279116ca7a","result_count":5,"query_id":"q-0008"} +{"ts":1777647510.0704832,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":4.866,"payload_hash":"774288b64350","result_count":5,"query_id":"q-0009"} +{"ts":1777647510.078312,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":5.365,"payload_hash":"90820120524b","result_count":5,"query_id":"q-0010"} +{"ts":1777647510.0833874,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":5.153,"payload_hash":"dbcf27d64abd","result_count":5,"query_id":"q-0011"} +{"ts":1777647510.0893867,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":5.369,"payload_hash":"15194f16f7b3","result_count":5,"query_id":"q-0012"} +{"ts":1777647510.093969,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":5.016,"payload_hash":"281c5b0c3bf3","result_count":5,"query_id":"q-0013"} +{"ts":1777647510.0986938,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":4.218,"payload_hash":"d6fd7664031b","result_count":5,"query_id":"q-0014"} +{"ts":1777647510.1022139,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":3.904,"payload_hash":"193708f38b75","result_count":5,"query_id":"q-0015"} +{"ts":1777647510.1067388,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":4.29,"payload_hash":"97a6af835837","result_count":5,"query_id":"q-0016"} +{"ts":1777647510.11112,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":5.478,"payload_hash":"f7f8e1814a95","result_count":5,"query_id":"q-0017"} +{"ts":1777647510.1153057,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":4.679,"payload_hash":"0a339a384ee6","result_count":5,"query_id":"q-0018"} +{"ts":1777647510.120213,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":5.092,"payload_hash":"0af52e366bc9","result_count":5,"query_id":"q-0019"} +{"ts":1777647510.1283789,"run_id":"d4c7881a1364","provider":"postgres","verb":"search","latency_ms":6.424,"payload_hash":"d52f2a7c632a","result_count":5,"query_id":"q-0020"} diff --git a/sdk-python/README.md b/sdk-python/README.md index 2470159..a058b52 100644 --- a/sdk-python/README.md +++ b/sdk-python/README.md @@ -57,4 +57,6 @@ green your adapter is a drop-in replacement for any other. - `openmem-eval` — manual benchmark harness comparing recall, MRR, and latency across configured providers. **Never runs in CI.** Default invocation makes zero network calls. See - [specs/004-eval-kit/quickstart.md](../specs/004-eval-kit/quickstart.md). + [specs/004-eval-kit/quickstart.md](../specs/004-eval-kit/quickstart.md) + for usage and [docs/eval/](../docs/eval/README.md) for a committed + sample report + trace from a real postgres run. diff --git a/sdk-python/openmem/eval/runner.py b/sdk-python/openmem/eval/runner.py index 3a9a05f..02d4489 100644 --- a/sdk-python/openmem/eval/runner.py +++ b/sdk-python/openmem/eval/runner.py @@ -13,6 +13,7 @@ from openmem.eval import cost as cost_mod from openmem.eval.scorer import compute_metrics +from openmem.eval.trace import TraceWriter from openmem.eval.types import ( Dataset, ErrorRecord, @@ -108,121 +109,153 @@ def run( results: list[ProviderResult] = [] gold = {q.query_id: q.gold_fact_ids for q in dataset.queries} - for provider in cfg.providers: - pr = ProviderResult(provider=provider) - run_started = time.perf_counter() + with TraceWriter(cfg.trace_path) as trace: + for provider in cfg.providers: + pr = _run_provider(provider, cfg, dataset, gold, factory, trace) + results.append(pr) + + return results + + +def _run_provider( + provider: str, + cfg: RunConfig, + dataset: Dataset, + gold: dict[str, tuple[str, ...]], + factory: Callable[[str], Any], + trace: TraceWriter, +) -> ProviderResult: + pr = ProviderResult(provider=provider) + run_started = time.perf_counter() + try: + mem = factory(provider) + except Exception as exc: # pragma: no cover - defensive + pr.status = "failed" + pr.errors.append( + ErrorRecord( + verb="init", + error_class=type(exc).__name__, + message=str(exc), + ts=_iso_now(), + ) + ) + pr.metrics = Metrics(error_count=1) + trace.emit(provider=provider, verb="init", run_id=cfg.run_id, + latency_ms=0.0, error=f"{type(exc).__name__}: {exc}") + return pr + + # ---- ingest ---- + added_ids: list[str] = [] + for fact in dataset.facts: + t0 = time.perf_counter() + err: Optional[str] = None try: - mem = factory(provider) - except Exception as exc: # pragma: no cover - defensive - pr.status = "failed" + rec = mem.add( + content=_stamp(fact.fact_id, fact.content), + user_id=cfg.user_id(), + tags=[*fact.tags, f"fact:{fact.fact_id}"], + ) + rid = getattr(rec, "id", None) + if rid: + added_ids.append(rid) + except Exception as exc: + err = f"{type(exc).__name__}: {exc}" pr.errors.append( ErrorRecord( - verb="init", + verb="add", error_class=type(exc).__name__, message=str(exc), ts=_iso_now(), + fact_id=fact.fact_id, ) ) - pr.metrics = Metrics(error_count=1) - results.append(pr) - continue - - # ---- ingest ---- - added_ids: list[str] = [] - for fact in dataset.facts: - t0 = time.perf_counter() - try: - rec = mem.add( - content=_stamp(fact.fact_id, fact.content), - user_id=cfg.user_id(), - tags=[*fact.tags, f"fact:{fact.fact_id}"], - ) - rid = getattr(rec, "id", None) - if rid: - added_ids.append(rid) - except Exception as exc: - pr.errors.append( - ErrorRecord( - verb="add", - error_class=type(exc).__name__, - message=str(exc), - ts=_iso_now(), - fact_id=fact.fact_id, - ) - ) - finally: - pr.ingest_latencies_ms.append((time.perf_counter() - t0) * 1000) - - # ---- wait for ingest (no-op for sync adapters; polling for async) ---- - wait = getattr(mem, "wait_for_ingest", None) - if callable(wait) and added_ids: - try: - wait(added_ids, cfg.user_id(), timeout=30.0) - except Exception: # pragma: no cover - defensive - pass - - # ---- search ---- - for query in dataset.queries: - t0 = time.perf_counter() - top_ids: list[str] = [] - err: Optional[str] = None - try: - hits = mem.search(query.query, cfg.user_id(), limit=cfg.top_k) - for h in hits: - fid = _recover_fact_id( - content=getattr(h.memory, "content", None), - tags=getattr(h.memory, "tags", None), - ) - if fid: - top_ids.append(fid) - except Exception as exc: - err = f"{type(exc).__name__}: {exc}" - pr.errors.append( - ErrorRecord( - verb="search", - error_class=type(exc).__name__, - message=str(exc), - ts=_iso_now(), - query_id=query.query_id, - ) - ) + finally: latency_ms = (time.perf_counter() - t0) * 1000 - pr.search_latencies_ms.append(latency_ms) - pr.query_results.append( - QueryResult( + pr.ingest_latencies_ms.append(latency_ms) + trace.emit( + provider=provider, verb="add", run_id=cfg.run_id, + latency_ms=latency_ms, + payload_text=fact.content, + error=err, + extra={"fact_id": fact.fact_id}, + ) + + # ---- wait for ingest (no-op for sync adapters; polling for async) ---- + wait = getattr(mem, "wait_for_ingest", None) + if callable(wait) and added_ids: + try: + wait(added_ids, cfg.user_id(), timeout=30.0) + except Exception: # pragma: no cover - defensive + pass + + # ---- search ---- + for query in dataset.queries: + t0 = time.perf_counter() + top_ids: list[str] = [] + err: Optional[str] = None + try: + hits = mem.search(query.query, cfg.user_id(), limit=cfg.top_k) + for h in hits: + fid = _recover_fact_id( + content=getattr(h.memory, "content", None), + tags=getattr(h.memory, "tags", None), + ) + if fid: + top_ids.append(fid) + except Exception as exc: + err = f"{type(exc).__name__}: {exc}" + pr.errors.append( + ErrorRecord( + verb="search", + error_class=type(exc).__name__, + message=str(exc), + ts=_iso_now(), query_id=query.query_id, - top_k_fact_ids=top_ids, - latency_ms=latency_ms, - error=err, ) ) - - pr.total_wall_s = round(time.perf_counter() - run_started, 3) - pr.metrics = Metrics(error_count=len(pr.errors)) - pr.metrics = compute_metrics(pr, gold) - # carry over error_count after compute_metrics overwrites - pr.metrics.error_count = len(pr.errors) - - # cost accounting (only matters when --live and provider is paid) - est = cost_mod.estimate( - provider, - add_calls=len(dataset.facts), - search_calls=len(dataset.queries), - delete_calls=len(dataset.facts) if cfg.cleanup else 0, + latency_ms = (time.perf_counter() - t0) * 1000 + pr.search_latencies_ms.append(latency_ms) + pr.query_results.append( + QueryResult( + query_id=query.query_id, + top_k_fact_ids=top_ids, + latency_ms=latency_ms, + error=err, + ) + ) + trace.emit( + provider=provider, verb="search", run_id=cfg.run_id, + latency_ms=latency_ms, + payload_text=query.query, + result_count=len(top_ids), + error=err, + extra={"query_id": query.query_id}, ) - pr.estimated_cost_usd = est.estimated_usd - - # status roll-up - if pr.errors and pr.query_results and any(qr.top_k_fact_ids for qr in pr.query_results): - pr.status = "partial" - elif pr.errors and not any(qr.top_k_fact_ids for qr in pr.query_results): - pr.status = "failed" - else: - pr.status = "ok" - - results.append(pr) - return results + pr.total_wall_s = round(time.perf_counter() - run_started, 3) + pr.metrics = Metrics(error_count=len(pr.errors)) + pr.metrics = compute_metrics(pr, gold) + # carry over error_count after compute_metrics overwrites + pr.metrics.error_count = len(pr.errors) + + # cost accounting (only matters when --live and provider is paid) + est = cost_mod.estimate( + provider, + add_calls=len(dataset.facts), + search_calls=len(dataset.queries), + delete_calls=len(dataset.facts) if cfg.cleanup else 0, + ) + pr.estimated_cost_usd = est.estimated_usd + + # status roll-up + if pr.errors and pr.query_results and any(qr.top_k_fact_ids for qr in pr.query_results): + pr.status = "partial" + elif pr.errors and not any(qr.top_k_fact_ids for qr in pr.query_results): + pr.status = "failed" + else: + pr.status = "ok" + + return pr def _iso_now() -> str: