From f0b697064ad97fda6ca567556619b19546a7b1e7 Mon Sep 17 00:00:00 2001 From: hrw991009 Date: Thu, 10 Sep 2026 11:09:15 +0800 Subject: [PATCH] feat(benchmark): add LoCoMo V2 evidence chain and report --- benchmark/locomo/.env.example | 12 +- benchmark/locomo/.gitignore | 4 + benchmark/locomo/README.md | 241 ++-- .../locomo/docs/locomo-v2-test-report.md | 109 ++ benchmark/locomo/smoke-test.ts | 2 +- benchmark/locomo/src/dataset.test.ts | 37 + benchmark/locomo/src/dataset.ts | 68 +- benchmark/locomo/src/diagnostics.test.ts | 207 ++++ benchmark/locomo/src/diagnostics.ts | 314 +++++ benchmark/locomo/src/evaluator.test.ts | 200 +++- benchmark/locomo/src/evaluator.ts | 1047 ++++++++++------- benchmark/locomo/src/index.ts | 391 +++--- benchmark/locomo/src/metrics.ts | 86 +- .../locomo/src/opencontext-client.test.ts | 31 + benchmark/locomo/src/opencontext-client.ts | 299 +++-- benchmark/locomo/src/scorer.test.ts | 16 + benchmark/locomo/src/scorer.ts | 12 +- benchmark/locomo/src/types.ts | 164 ++- benchmark/locomo/start-daemon.ps1 | 120 ++ 19 files changed, 2485 insertions(+), 875 deletions(-) create mode 100644 benchmark/locomo/docs/locomo-v2-test-report.md create mode 100644 benchmark/locomo/src/dataset.test.ts create mode 100644 benchmark/locomo/src/diagnostics.test.ts create mode 100644 benchmark/locomo/src/diagnostics.ts create mode 100644 benchmark/locomo/src/opencontext-client.test.ts create mode 100644 benchmark/locomo/src/scorer.test.ts create mode 100644 benchmark/locomo/start-daemon.ps1 diff --git a/benchmark/locomo/.env.example b/benchmark/locomo/.env.example index b1e30b40..9eff1031 100644 --- a/benchmark/locomo/.env.example +++ b/benchmark/locomo/.env.example @@ -1,4 +1,4 @@ -# OpenContext memory daemon (started via `opencontext http`) +# OpenContext daemon (started via `start-daemon.ps1` or `opencontext http`) # OPENCONTEXT_URL=http://127.0.0.1:7421 # Answerer LLM — Anthropic-compatible endpoint (e.g. MiniMax) @@ -6,5 +6,13 @@ ANTHROPIC_AUTH_TOKEN= ANTHROPIC_BASE_URL=https://api.minimaxi.com/anthropic ANSWER_MODEL=MiniMax-M3-highspeed -# Judge LLM (OpenRouter) — used by the LLM judge in metrics.ts +# OpenRouter is required for the judge and is the answerer fallback when +# ANTHROPIC_AUTH_TOKEN is unset. OPENROUTER_API_KEY=your_openrouter_api_key_here +OPENROUTER_ANSWER_MODEL=deepseek/deepseek-v4-flash-0731 +OPENROUTER_JUDGE_MODEL=qwen/qwen3.7-flash + +# Optional benchmark controls +LOCOMO_TOP_K=8 +# LOCOMO_MODEL_REQUEST_TIMEOUT_MS=90000 +# LOCOMO_CHECKPOINT_DIR=checkpoints/locomo diff --git a/benchmark/locomo/.gitignore b/benchmark/locomo/.gitignore index 9af3f11b..ae42b217 100644 --- a/benchmark/locomo/.gitignore +++ b/benchmark/locomo/.gitignore @@ -1,2 +1,6 @@ # Evaluation checkpoints (resume cache) checkpoints/ +dataset/*.json +results/ +results.json +runtime/ diff --git a/benchmark/locomo/README.md b/benchmark/locomo/README.md index c4ccc3fb..a059c302 100644 --- a/benchmark/locomo/README.md +++ b/benchmark/locomo/README.md @@ -1,153 +1,156 @@ # LoCoMo Benchmark -> **Note:** this directory is opencontext's **own** scoring pipeline (custom -> BLEU/F1 + custom LLM-judge prompts). Its numbers are **not** comparable to -> the AML (Agent Memory Leaderboard) binary CORRECT/WRONG scoring. For the -> AML-comparable LoCoMo path, use [`../aml-local/`](../aml-local/), which -> drives the vendored official AML pipeline. +OpenContext's local LoCoMo V2 evaluation harness. It measures answer quality with +the existing LoCoMo answer prompt, token-overlap metrics, and an LLM judge while +preserving a question-level evidence trail for ingestion, retrieval, answerer, +and judge stages. -Benchmark suite for evaluating the OpenContext long-term memory retrieval system using the [LoCoMo](https://github.com/StonyBrookUniversity/LoCoMo) dataset. +This is an OpenContext-specific evaluation pipeline. Its scores are not directly +comparable to the Agent Memory Leaderboard's official CORRECT/WRONG pipeline. -## Overview +## Evaluation boundary -This benchmark evaluates how well the memory system answers questions from conversation history across different retrieval modes. It tests temporal reasoning, multi-hop inference, and single-hop fact retrieval capabilities. +For the recommended `dialog` mode, the benchmark converts each upstream LoCoMo +conversation session into exactly one `RawMessage` and sends it to +`POST /v1/raw-messages` with `embedOnInsert: true`. Dialog-turn identifiers such +as `D1:3` remain in the raw text for later evidence attribution. -## Dataset - -This benchmark uses [LoCoMo V2](https://github.com/BrianV1981/locomo-v2) (the `locomo_v2_minicpm.json` variant — text-only with MiniCPM-V OCR baked in) instead of the original V1. V2 fixes V1's 99 score-corrupting ground-truth hallucinations and ~75 dead image URLs, and is drop-in compatible with the loader (no `evidence` field on QA items, plus a new category 5 "abstention" set that the loader skips automatically). - -The dataset contains 10 conversation samples (`conv-26`, `conv-30`, `conv-41`–`conv-50`) with 1,922 total QA pairs. The loader drops category-5 items without an answer (430 of 438) and keeps the 8 category-5 items that do have an answer, so the effective question set is **1,492 questions**. - -Each sample includes: +After that boundary, the daemon owns all core memory behavior: -- **Conversation history** - Raw dialog between speakers -- **Observations** - Summarized observations with dialog references -- **Session summaries** - High-level summaries of each session -- **QA pairs** - Questions with ground truth answers across 5 categories +- chunking and parent-child relationships; +- embedding and indexing; +- semantic/lexical retrieval, fusion, and reranking; +- the final Top-K returned by `POST /v1/search`. -### Question Categories +The benchmark does not pre-chunk, embed, insert derived retrieval records, alter +rankings, or replace daemon results. It only formats the returned Top-K with the +existing benchmark prompt, calls the configured answer model, and scores that +answer. -| Category | Name | Description | -| -------- | ----------- | ------------------------------------------ | -| 1 | single_hop | Simple factual recall from a single memory | -| 2 | temporal | Questions requiring date/time reasoning | -| 3 | multi_hop | Multi-step inference across sessions | -| 4 | open_domain | Open-ended questions requiring synthesis | -| 5 | abstention | Questions without an answer (loader skips) | +`observation` and `session_summary` remain available for exploratory comparisons, +but a formal raw-conversation run should use `dialog`. -## Retrieval Modes - -| Mode | Description | -| ----------------- | ---------------------------------------------- | -| `dialog` | Raw conversation history | -| `observation` | Summarized observations with dialog references | -| `session_summary` | Session-level summaries only | +## Dataset -## Setup +Use the text-only `locomo_v2_minicpm.json` file from +[LoCoMo V2](https://github.com/BrianV1981/locomo-v2). Put it under `dataset/` +(dataset JSON files are intentionally ignored by Git). -```bash -# Install dependencies -pnpm install +The downloaded V2 MiniCPM file contains 10 conversation samples and 1,922 raw +QA rows. The harness retains 1,492 rows with usable `answer` values, including +the eight answerable category-5 rows; its remaining category-5 rows use the +separate adversarial-answer format and are outside this answer-and-judge path. +Its 834 image descriptions are stored beside their dialog turns as +`minicpm_caption`; the dialog mapper includes those descriptions in the same +source RawMessage as the corresponding text and turn ID. The loader validates +sample IDs, conversation objects, QA fields, category, and any optional evidence +arrays before ingestion. -# Copy environment file -cp .env.example .env +LoCoMo V2 does not provide a QA-level `evidence` field. Consequently, exact gold +dialog-turn Recall@K, Hit@K, MRR, and Precision@K are recorded as `null` for V2; +they must not be inferred from the answer string. The complete daemon retrieval +trace is still retained. If an evidence-bearing LoCoMo-compatible file is used, +gold metrics are computed only from exact turn IDs present in the returned child +chunk text; parent metadata alone does not count as a hit. -# Start the OpenContext memory daemon (from repo root, after build) -node packages/opencontext/dist/cli/opencontext.js http --embedding-provider local --memory-backend sqlite-vec -# or, if the global bin is installed: -opencontext http -# → serves http://127.0.0.1:7421, no auth -``` +## Configuration -Edit `.env` to add your API keys: +From `benchmark/locomo`, copy `.env.example` to `.env` and configure the models: ```env -# Answerer LLM (Anthropic-compatible endpoint, e.g. MiniMax) -ANTHROPIC_AUTH_TOKEN=your_anthropic_token_here -ANTHROPIC_BASE_URL=https://api.minimaxi.com/anthropic -ANSWER_MODEL=MiniMax-M3-highspeed - -# Judge LLM (OpenRouter); also the answerer fallback if ANTHROPIC_AUTH_TOKEN is unset OPENROUTER_API_KEY=your_openrouter_api_key_here +OPENROUTER_ANSWER_MODEL=deepseek/deepseek-v4-flash-0731 +OPENROUTER_JUDGE_MODEL=qwen/qwen3.8-flash +LOCOMO_TOP_K=8 ``` -## Usage +An Anthropic-compatible answer endpoint remains supported through +`ANTHROPIC_AUTH_TOKEN`, `ANTHROPIC_BASE_URL`, and `ANSWER_MODEL`. When that token +is absent, the answerer uses OpenRouter. The judge always uses OpenRouter and +requires `OPENROUTER_JUDGE_MODEL`; no judge model is hard-coded. -```bash -# Run full benchmark with observation mode -pnpm benchmark -- --dataset dataset/locomo_v2.json --mode observation +## Start a clean daemon -# Quick mode (first 5 questions per sample) -pnpm benchmark -- --dataset dataset/locomo_v2.json --mode observation --quick +Build the repository once, then start the benchmark-owned local daemon: -# Run with specific samples -pnpm benchmark -- --dataset dataset/locomo_v2.json --mode dialog --samples conv-26,conv-30 - -# Save results to file -pnpm benchmark -- --dataset dataset/locomo_v2.json --mode observation --output results.json +```powershell +pnpm build +./benchmark/locomo/start-daemon.ps1 -Port 7421 ``` -### CLI Options - -| Flag | Short | Description | Default | -| ----------- | ----- | --------------------------------------------------- | ------------------ | -| `--dataset` | `-d` | Path to LoCoMo JSON dataset | Required | -| `--mode` | `-m` | Retrieval mode (dialog/observation/session_summary) | observation | -| `--samples` | `-s` | Comma-separated sample IDs to run | All | -| `--quick` | `-q` | Limit to first 5 questions per sample | false | -| `--output` | `-o` | Save results JSON to path | None | -| `--port` | `-p` | OpenContext daemon port (env: `OPENCONTEXT_PORT` / `OPENCONTEXT_URL`) | 7421 | -| `--resume` | | Reuse completed checkpoints for the same models | true | -| `--no-resume` | | Ignore checkpoints and run every selected question | false | - -Before ingest or model calls, the CLI checks the dataset and selected samples, -daemon, credentials, output/checkpoint paths, and arguments. It reports all -detected failures together and never prints credential values. `--help` does not -run these checks. - -With `--resume`, both correct and incorrect completed judge results are reused; -only execution failures are retried. `--no-resume` always starts a fresh run. +Without `-DatabasePath`, every launch creates a timestamped fresh SQLite database +under `benchmark/locomo/runtime/`. The script refuses to replace an existing port +listener, waits for `/health`, and writes `daemon.json` plus stdout/stderr logs. +To resume the exact same database deliberately: -## Output - -The benchmark outputs: +```powershell +./benchmark/locomo/start-daemon.ps1 -Port 7421 -DatabasePath D:\path\to\store.db +``` -- **Overall accuracy** - LLM judge accuracy across all categories -- **Per-category metrics** - F1, BLEU-1, BLEU-4 scores by category -- **Per-sample results** - Accuracy and token usage per sample -- **Token usage** - Real provider usage when available; otherwise `null` -- **Run manifest** - Git commit, dataset identity, models, retrieval mode/top-k, - selection parameters, resume mode, and wall-clock time +The launcher uses local MiniLM embeddings, `sqlite-vec`, and the local MiniLM +reranker. The returned `daemon.json` freezes those settings for the run record. -With `--output results.json`, the manifest is written to -`results.json.manifest.json`. Without `--output`, it is written under `results/`. +## Preflight and run -### Metrics +Run commands from `benchmark/locomo`: -- **LLM Judge Accuracy** - Whether the LLM judge considers the answer correct -- **F1 Score** - Token-level precision/recall -- **BLEU-1/4** - N-gram overlap with brevity penalty +```powershell +# Validate dataset, daemon, credentials, arguments, and writable artifacts. +pnpm benchmark -- --dataset dataset/locomo_v2_minicpm.json --mode dialog --preflight-only -## Architecture +# Small smoke run: first five answerable questions per sample. +pnpm benchmark -- --dataset dataset/locomo_v2_minicpm.json --mode dialog --quick --no-resume --output results/smoke.json +# Formal full run against a fresh daemon database. +pnpm benchmark -- --dataset dataset/locomo_v2_minicpm.json --mode dialog --no-resume --output results/locomo-v2-dialog.json ``` -src/ -├── index.ts # CLI entry point -├── evaluator.ts # LoCoMoEvaluator - loads samples, runs QA evaluation -├── opencontext-client.ts # OpenContext daemon client + answerer LLM calls -├── dataset.ts # LoCoMo JSON parsing -├── metrics.ts # BLEU, F1, LLM judge evaluation -├── scorer.ts # Category name mapping -├── prompts.ts # LLM judge prompt template -├── contracts.ts # MemoryStorageAdapter interface -├── types.ts # TypeScript types -└── prompts.ts # Evaluation prompts -``` - -## Requirements -- Node.js 18+ -- pnpm -- OpenContext memory daemon running on localhost (default http://127.0.0.1:7421, no auth; override with `--port` / `OPENCONTEXT_URL`) -- Anthropic-compatible API token for the answerer (or OpenRouter as fallback) -- OpenRouter API key (for LLM judge evaluation) +Useful options: + +- `--samples conv-26,conv-30` selects sample IDs. +- `--port 7421` overrides `OPENCONTEXT_PORT`/`OPENCONTEXT_URL`. +- `--resume` reuses only completed checkpoints whose schema, dataset, question, + retrieval mode/Top-K, answer model, and judge model all match. +- `--no-resume` reruns every selected question. Use it with a fresh database for + comparable formal results. +- `LOCOMO_CHECKPOINT_DIR` moves the checkpoint directory. +- `LOCOMO_MODEL_REQUEST_TIMEOUT_MS` configures answer/judge request timeout. + +Execution failures are checkpointed separately and retried on a resumed run; +they are never counted as completed judge failures or retrieval-metric misses. + +## Evidence artifacts + +With `--output results/run.json`, the harness writes: + +- `run.json`: compact results, category metrics, completed-only accuracy, + execution-error rate, diagnostic summary, and run manifest reference; +- `run.trace.jsonl`: one complete record per QA, including exact dataset/question + hashes, final Top-K full text, candidate channels, fused pre-rerank order, + reranker metadata, answer prompt/response, raw judge output, attempts, latency, + token usage, and failure-stage classification; +- `run.sessions.jsonl`: one record per mapped source session, including message ID, + source evidence IDs, content hash/size, ingest batch, status, latency, warnings, + and error details; +- `run.json.manifest.json`: Git state, dataset path/hash/size, models, retrieval + configuration, selection flags, resume mode, timestamps, and token usage. + +Candidate hits keep hashes and excerpts to limit artifact size; final Top-K hits +keep the full text actually supplied to the answer prompt. When the daemon does +not return pre-merge diagnostics, the trace says so explicitly instead of +inventing candidate-stage evidence. + +## Metrics and failure attribution + +The primary answer metric is LLM-judge accuracy. F1 and BLEU-1/4 are also +reported. Diagnostics keep these cases separate: + +- ingestion/indexing, retrieval, answerer, judge, and provider execution errors; +- missing or partial dataset evidence references; +- retrieval miss or partial retrieval when exact gold evidence exists; +- answer failure despite complete retrieved evidence; +- answer failure with V2 gold evidence unavailable. + +This separation prevents provider outages from being reported as retrieval +failures and prevents session-level or parent-level metadata from being promoted +to child-chunk recall evidence. diff --git a/benchmark/locomo/docs/locomo-v2-test-report.md b/benchmark/locomo/docs/locomo-v2-test-report.md new file mode 100644 index 00000000..3be5d9e3 --- /dev/null +++ b/benchmark/locomo/docs/locomo-v2-test-report.md @@ -0,0 +1,109 @@ +# LoCoMo V2 OpenContext Evaluation Report + +- Report version: v1 +- Status: completed; 1,492 / 1,492 Answerer-and-Judge records persisted, with no final execution errors +- Dataset: `locomo_v2_minicpm.json` (LoCoMo V2 MiniCPM, 10 samples) +- Final recovery window: 2026-09-10T00:42:33.356Z to 2026-09-10T01:35:33.633Z +- Result: [locomo-v2-minicpm-dialog-20260909-r2-recovery3.json](../results/locomo-v2-minicpm-dialog-20260909-r2-recovery3.json) +- Run manifest: [locomo-v2-minicpm-dialog-20260909-r2-recovery3.json.manifest.json](../results/locomo-v2-minicpm-dialog-20260909-r2-recovery3.json.manifest.json) +- Question trace: [locomo-v2-minicpm-dialog-20260909-r2-recovery3.trace.jsonl](../results/locomo-v2-minicpm-dialog-20260909-r2-recovery3.trace.jsonl) +- Session-ingest trace: [locomo-v2-minicpm-dialog-20260909-r2-recovery3.sessions.jsonl](../results/locomo-v2-minicpm-dialog-20260909-r2-recovery3.sessions.jsonl) + +> `results/`, checkpoints, runtime databases, and model caches are excluded by `.gitignore`. This report is the committable summary; retain the linked raw artifacts for response-, source-, and trace-level audit. It intentionally does not create a separate question index. + +## 1. Purpose and scope + +This report records the completed LoCoMo V2 MiniCPM OpenContext run. The harness maps every upstream conversation session to one `RawMessage` and submits it to the local OpenContext daemon. The daemon, rather than the evaluator, owns chunking, parent-child construction, embedding, indexing, semantic and lexical retrieval, fusion, reranking, and the final Top-12 returned by search. The evaluator supplies only that final returned context to the existing LoCoMo answer prompt, calls the Answerer, calls the Judge, and records the evidence trail. + +The result is an end-to-end diagnostic, not a pure retrieval score: it includes daemon retrieval behavior, the final answer context, Answerer behavior, provider reliability, and judging. It is not an AML or official leaderboard LoCoMo score; this workspace uses its OpenContext harness, answer prompt, lexical metrics, and LLM judge. + +## 2. System and evaluation configuration + +| Item | Configuration | +| ---------------------- | ------------------------------------------------------------------------------------ | +| Dataset | `dataset/locomo_v2_minicpm.json` | +| Dataset size / SHA-256 | 3,198,216 bytes / `874685e3fd3ecefe0562637b948f8470b73e32fc72c02407a2797b09dce83959` | +| Dataset rows | 10 samples; 1,922 raw QA rows; 1,492 answerable rows evaluated | +| Answerer | `openrouter:deepseek/deepseek-v4-flash-0731` | +| Judge | `openrouter:qwen/qwen3.7-flash` | +| Store | isolated local `sqlite-vec` daemon with lexical search available | +| Embedding | local `Xenova/all-MiniLM-L6-v2`, 384 dimensions | +| Reranker | local `Xenova/ms-marco-MiniLM-L-6-v2` | +| Parent record | one complete upstream conversation session per `RawMessage` | +| Retrieval | daemon-default final Top-12, `dialog` mode | +| Trace schema | `1.0` | +| Manifest commit | `2f2df851a02849bc1f295fa7b9bda3cb6866041f` | + +The final manifest records `resume: true` and `git_dirty: true`. The run began as a fresh run and later used compatible checkpoints to reuse completed judged records and retry provider failures. The final recovery used a 300-second model-request timeout and resolved the two remaining execution errors. This is a complete end-to-end artifact, but not a clean fresh-database, `--no-resume` baseline; do not use it directly for version-to-version or leaderboard comparison. + +## 3. Evidence completeness and execution + +| Metric | Result | +| --------------------------------------------- | ---------------------------------: | +| Scheduled questions / persisted predictions | 1,492 / 1,492 | +| Completed Answerer + Judge records | 1,492 / 1,492 | +| Final execution errors | 0 / 1,492 (0.00%) | +| Question traces | 1,492; 10 unique sample IDs | +| Retrieval, Answerer, and Judge traces present | 1,492 / 1,492 each | +| Session-ingest records | 272; 272 unique daemon message IDs | +| Final session-ingest errors | 0 | +| Recorded prompt tokens | 18,366,412 | +| Recorded completion tokens | 3,203,813 | +| Recorded total tokens | 21,570,225 | +| Mean recorded total tokens / question | 14,457 | +| Final recovery wall-clock | 3,180,277 ms (about 53m) | + +Two Answerer attempts timed out during recovery, but their in-question retries completed successfully. They are not terminal execution errors. Recorded token totals cover completed provider responses present in final checkpoints; failed attempts can carry provider-side usage that is not represented, so the totals are not a billing total. + +## 4. Overall result + +All scheduled records completed, so the all-record and completed-only views are identical. + +| Metric | Result | +| ------------------ | -------------------: | +| LLM-judge accuracy | 931 / 1,492 (62.40%) | +| Mean token F1 | 0.1549 | +| Mean BLEU-1 | 0.1158 | +| Mean BLEU-4 | 0.0172 | + +LLM-judge accuracy is the primary end-to-end measure in this harness. F1 and BLEU are lexical-overlap diagnostics and do not replace the Judge result. + +## 5. Results by category + +| Category | Questions | LLM-judge accuracy | Mean F1 | Mean BLEU-1 | +| ------------- | --------: | -----------------: | ------: | ----------: | +| `multi_hop` | 261 | 190 / 261 (72.80%) | 0.0740 | 0.0475 | +| `temporal` | 308 | 175 / 308 (56.82%) | 0.1006 | 0.0696 | +| `open_domain` | 94 | 49 / 94 (52.13%) | 0.0984 | 0.0811 | +| `single_hop` | 821 | 512 / 821 (62.36%) | 0.2084 | 0.1595 | +| `adversarial` | 8 | 5 / 8 (62.50%) | 0.0534 | 0.0360 | + +`multi_hop` is the strongest category in this artifact, while `open_domain` is the weakest. The category results are end-to-end outcomes, not isolated retrieval measurements. + +## 6. Retrieval evidence and its limit + +The question trace retains the exact final Top-12 text supplied to the Answerer, plus available candidate, fused-before-rerank, reranker, Answerer, and Judge evidence. Pre-merge diagnostics are present for all 1,492 questions. + +LoCoMo V2 MiniCPM does not supply QA-level `evidence` turn IDs. Therefore no question is retrieval-evaluable against gold turns, and the following retrieval metrics are correctly recorded as `null`: Recall@K, retrievable Recall@K, Hit@K, Precision@K, MRR, all-evidence-retrieved rate, and candidate-channel evidence recall. A session-level match or parent metadata must not be substituted for child-turn gold evidence. + +This artifact proves that daemon-owned retrieval and its final returned context were recorded for every question. It does not prove a numeric retrieval-recall claim. An evidence-bearing LoCoMo-compatible dataset is required for that claim. + +## 7. End-to-end failure interpretation + +| Diagnostic classification | Count | +| --------------------------- | ----: | +| `none` (judge-correct) | 931 | +| `gold_evidence_unavailable` | 561 | +| Execution/provider errors | 0 | + +The 561 non-passing records are labelled `gold_evidence_unavailable` because this V2 file has no QA-level gold turns. This is an evidence-availability limit, not a causal finding that retrieval, Answerer, or Judge behavior caused those outcomes. + +## 8. Artifact boundary and follow-up + +This artifact proves that every scheduled question reached a terminal judged record with ingestion, retrieval, Answerer, and Judge evidence. It does not establish a clean-baseline comparison, a standalone retrieval score, or an official leaderboard score. + +The narrow next steps are: + +1. Use a fresh database, `--no-resume`, and a clean recorded revision or explicit patch identity before comparing systems. +2. Use a LoCoMo-compatible file with QA-level evidence IDs before making Recall@K, Hit@K, Precision@K, or MRR claims. +3. Analyze the 561 non-passing traces by question category and final context quality, while keeping ingestion and retrieval daemon-owned. diff --git a/benchmark/locomo/smoke-test.ts b/benchmark/locomo/smoke-test.ts index 0d624c7c..fde47187 100644 --- a/benchmark/locomo/smoke-test.ts +++ b/benchmark/locomo/smoke-test.ts @@ -15,7 +15,7 @@ function assert(condition: boolean, message: string): void { console.log(`✓ ${message}`); } -const datasetPath = "./dataset/locomo_v2.json"; +const datasetPath = "./dataset/locomo_v2_minicpm.json"; console.log(`Loading ${datasetPath}...\n`); const samples = await loadLoCoMoDatasetFromJson(datasetPath); diff --git a/benchmark/locomo/src/dataset.test.ts b/benchmark/locomo/src/dataset.test.ts new file mode 100644 index 00000000..c7f52aae --- /dev/null +++ b/benchmark/locomo/src/dataset.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; + +import { createLoCoMoSample } from "./dataset"; + +describe("LoCoMo dataset validation", () => { + it("accepts V2 questions without evidence and skips rows without answers", () => { + const sample = createLoCoMoSample({ + sample_id: "conv-26", + conversation: { session_1: [] }, + qa: [ + { question: "Where did they go?", answer: "Paris", category: 4 }, + { question: "Unsupported premise?", category: 5 }, + ], + }); + + expect(sample.qa_pairs).toEqual([ + { question: "Where did they go?", answer: "Paris", category: 4, evidence: [] }, + ]); + }); + + it("rejects malformed evidence instead of silently fabricating attribution", () => { + expect(() => + createLoCoMoSample({ + sample_id: "conv-26", + conversation: { session_1: [] }, + qa: [ + { + question: "Where did they go?", + answer: "Paris", + category: 4, + evidence: ["D1:3", 7] as unknown as string[], + }, + ], + }), + ).toThrow("invalid evidence"); + }); +}); diff --git a/benchmark/locomo/src/dataset.ts b/benchmark/locomo/src/dataset.ts index a796c4e4..ed8de5ca 100644 --- a/benchmark/locomo/src/dataset.ts +++ b/benchmark/locomo/src/dataset.ts @@ -13,27 +13,44 @@ interface RawQAPair { } interface RawSample { - sample_id: string; - conversation: Record; + sample_id?: string; + conversation?: Record; observation?: Record; session_summary?: Record; event_summary?: Record; qa?: RawQAPair[]; } +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + /** * Create QAPair from raw dictionary. */ -function createQAPair(qa: RawQAPair): QAPair | null { +function createQAPair(qa: RawQAPair, sampleIndex: number, questionIndex: number): QAPair | null { // Skip QA pairs that don't have an answer field (e.g., adversarial questions) if (!qa.answer) { return null; } + if (typeof qa.question !== "string" || qa.question.trim().length === 0) { + throw new Error(`sample ${sampleIndex} question ${questionIndex} has invalid question`); + } + const category = typeof qa.category === "string" ? Number.parseInt(qa.category, 10) : (qa.category ?? 0); + if (!Number.isInteger(category) || category < 1 || category > 5) { + throw new Error(`sample ${sampleIndex} question ${questionIndex} has invalid category`); + } + if ( + qa.evidence !== undefined && + (!Array.isArray(qa.evidence) || !qa.evidence.every((item) => typeof item === "string")) + ) { + throw new Error(`sample ${sampleIndex} question ${questionIndex} has invalid evidence`); + } return { question: qa.question, answer: String(qa.answer), - category: typeof qa.category === "string" ? Number.parseInt(qa.category, 10) : (qa.category ?? 0), + category, evidence: qa.evidence ?? [], }; } @@ -41,21 +58,41 @@ function createQAPair(qa: RawQAPair): QAPair | null { /** * Create LoCoMoSample from raw dictionary. */ -export function createLoCoMoSample(data: RawSample): LoCoMoSample { +export function createLoCoMoSample(data: RawSample, sampleIndex = 0): LoCoMoSample { + if (typeof data.sample_id !== "string" || data.sample_id.trim().length === 0) { + throw new Error(`sample ${sampleIndex} has invalid sample_id`); + } + if (!isRecord(data.conversation)) { + throw new Error(`sample ${sampleIndex} has invalid conversation`); + } + for (const [field, value] of [ + ["observation", data.observation], + ["session_summary", data.session_summary], + ["event_summary", data.event_summary], + ] as const) { + if (value !== undefined && !isRecord(value)) { + throw new Error(`sample ${sampleIndex} has invalid ${field}`); + } + } + if (data.qa !== undefined && !Array.isArray(data.qa)) { + throw new Error(`sample ${sampleIndex} has invalid qa`); + } const qaPairs: QAPair[] = []; if (data.qa) { - for (const qa of data.qa) { - const qaPair = createQAPair(qa); + for (const [questionIndex, qa] of data.qa.entries()) { + if (!isRecord(qa)) throw new Error(`sample ${sampleIndex} question ${questionIndex} must be an object`); + const qaPair = createQAPair(qa as unknown as RawQAPair, sampleIndex, questionIndex); if (qaPair) { qaPairs.push(qaPair); } } } + if (qaPairs.length === 0) throw new Error(`sample ${sampleIndex} has no answerable questions`); return { sample_id: data.sample_id, - conversation: data.conversation ?? {}, + conversation: data.conversation, observation: data.observation ?? {}, session_summary: data.session_summary ?? {}, event_summary: data.event_summary ?? {}, @@ -68,11 +105,18 @@ export function createLoCoMoSample(data: RawSample): LoCoMoSample { */ export async function loadLoCoMoDatasetFromJson(jsonPath: string): Promise { const content = await readFile(jsonPath, "utf-8"); - const data = JSON.parse(content); + const data: unknown = JSON.parse(content); // Handle both single sample and list of samples - if (Array.isArray(data)) { - return data.map((sample) => createLoCoMoSample(sample as RawSample)); + const rawSamples = Array.isArray(data) ? data : [data]; + const samples = rawSamples.map((sample, index) => { + if (!isRecord(sample)) throw new Error(`sample ${index} must be an object`); + return createLoCoMoSample(sample as RawSample, index); + }); + const ids = new Set(); + for (const sample of samples) { + if (ids.has(sample.sample_id)) throw new Error(`duplicate sample_id: ${sample.sample_id}`); + ids.add(sample.sample_id); } - return [createLoCoMoSample(data as RawSample)]; + return samples; } diff --git a/benchmark/locomo/src/diagnostics.test.ts b/benchmark/locomo/src/diagnostics.test.ts new file mode 100644 index 00000000..74d19835 --- /dev/null +++ b/benchmark/locomo/src/diagnostics.test.ts @@ -0,0 +1,207 @@ +import { describe, expect, it } from "vitest"; + +import { + buildRetrievalErrorTrace, + buildRetrievalTrace, + calculateDiagnosticSummary, + deriveFailureStage, +} from "./diagnostics"; +import type { MemorySearchResponse } from "./opencontext-client"; +import { + LOCOMO_TRACE_SCHEMA_VERSION, + type LoCoMoSessionTrace, + type Prediction, + type QAPair, + RetrievalMode, +} from "./types"; + +function qa(evidence: string[] = ["D1:3", "D2:1"]): QAPair { + return { question: "What pets were adopted?", answer: "Luna and Milo", category: 3, evidence }; +} + +function session(messageId: string, sessionId: string, evidenceIds: string[]): LoCoMoSessionTrace { + return { + schema_version: LOCOMO_TRACE_SCHEMA_VERSION, + sample_id: "fixture-sample", + retrieval_mode: RetrievalMode.DIALOG, + message_id: messageId, + session_id: sessionId, + session_index: 0, + ingest_batch_index: 0, + session_date: "2024-01-01", + evidence_ids: evidenceIds, + content_sha256: "hash", + content_characters: 10, + ingest_status: "completed", + ingest_latency_ms: 5, + ingest_warnings: [], + }; +} + +describe("LoCoMo retrieval evidence", () => { + it("records daemon candidates, reranking, and exact turn-level gold matches", () => { + const response: MemorySearchResponse = { + query: "What pets were adopted?", + sources: ["memory"], + count: 2, + warnings: [], + results: [ + { + id: "child-2", + content: "[D2:1] I adopted Milo.", + similarity: 0.91, + metadata: { parentMessageId: "message-2", sessionId: "2" }, + }, + { + id: "child-noise", + content: "Unrelated text", + similarity: 0.7, + metadata: { sessionId: "1", evidenceIds: ["D1:3"] }, + }, + ], + retrievalDiagnostics: { + mergeStrategy: "rrf", + candidateLimit: 20, + backend: "raw-message", + candidateCounts: { semantic: 2, lexical: 1, hybrid: 0, entity: 0, fused: 2, final: 2 }, + channels: { + semantic: [{ id: "child-1", content: "[D1:3] I adopted Luna.", similarity: 0.8, metadata: {} }], + lexical: [], + }, + fusedBeforeRerank: [ + { id: "child-1", content: "[D1:3] I adopted Luna.", similarity: 0.8, metadata: {} }, + { id: "child-2", content: "[D2:1] I adopted Milo.", similarity: 0.7, metadata: {} }, + ], + reranker: { + enabled: true, + provider: "local-transformers", + model: "reranker", + inputCount: 2, + outputCount: 2, + latencyMs: 12, + orderChanged: true, + }, + }, + }; + const sessions = new Map([ + ["message-1", session("message-1", "1", ["D1:3"])], + ["message-2", session("message-2", "2", ["D2:1"])], + ]); + + const trace = buildRetrievalTrace({ + qa: qa(), + response, + sessionsByMessageId: sessions, + availableEvidenceIds: ["D1:3", "D2:1"], + userId: "locomo_fixture-sample", + topK: 8, + latencyMs: 20, + }); + + expect(trace.hits[0]).toMatchObject({ + rank: 1, + matched_evidence_ids: ["D2:1"], + relevant: true, + }); + expect(trace.hits[0]?.content).toBe("[D2:1] I adopted Milo."); + expect(trace.candidate_channels.semantic[0]?.content).toBeUndefined(); + expect(trace.semantic_evidence_recall_at_candidate_k).toBe(0.5); + expect(trace.evidence_recall_at_k).toBe(0.5); + expect(trace.hit_at_k).toBe(1); + expect(trace.mrr).toBe(1); + expect(trace.reranker).toMatchObject({ enabled: true, order_changed: true }); + expect(deriveFailureStage({ qa: qa(), retrieval: trace, correct: false })).toBe("retrieval_partial"); + }); + + it("does not count parent metadata when the returned child text lacks the gold turn", () => { + const trace = buildRetrievalTrace({ + qa: qa(["D1:3"]), + response: { + query: "fixture", + sources: ["memory"], + count: 1, + warnings: [], + results: [ + { + id: "child-noise", + content: "A different child chunk", + similarity: 0.9, + metadata: { evidenceIds: ["D1:3"] }, + }, + ], + }, + sessionsByMessageId: new Map(), + availableEvidenceIds: ["D1:3"], + userId: "locomo_fixture-sample", + topK: 8, + latencyMs: 5, + }); + + expect(trace.hits[0]).toMatchObject({ matched_evidence_ids: [], relevant: false }); + expect(trace.evidence_recall_at_k).toBe(0); + }); + + it("marks V2 questions without gold evidence as unavailable", () => { + const trace = buildRetrievalTrace({ + qa: qa([]), + response: { query: "fixture", sources: ["memory"], results: [], count: 0, warnings: [] }, + sessionsByMessageId: new Map(), + availableEvidenceIds: [], + userId: "locomo_fixture-sample", + topK: 8, + latencyMs: 5, + }); + + expect(trace).toMatchObject({ + retrieval_applicable: false, + evidence_granularity: "unavailable", + evidence_recall_at_k: null, + mrr: null, + }); + expect(deriveFailureStage({ qa: qa([]), retrieval: trace, correct: false })).toBe( + "gold_evidence_unavailable", + ); + }); + + it("keeps execution errors separate in the diagnostic summary", () => { + const prediction = { + status: "execution_error", + failure_stage: "answerer_error", + trace: { retrieval: null, answerer: null, judge: null }, + } as Prediction; + + expect(calculateDiagnosticSummary([prediction])).toMatchObject({ + completed: 0, + execution_errors: 1, + execution_error_rate: 1, + questions_with_gold_evidence: 0, + retrieval_evaluable_questions: 0, + mean_evidence_recall_at_k: null, + failure_stages: { answerer_error: 1 }, + }); + }); + + it("excludes failed retrieval calls from retrieval metric denominators", () => { + const retrieval = buildRetrievalErrorTrace({ + qa: qa(["D1:3"]), + availableEvidenceIds: ["D1:3"], + userId: "locomo_fixture-sample", + topK: 8, + latencyMs: 5, + error: "timeout", + }); + const prediction = { + status: "execution_error", + evidence: ["D1:3"], + failure_stage: "retrieval_error", + trace: { retrieval, answerer: null, judge: null }, + } as Prediction; + + expect(calculateDiagnosticSummary([prediction])).toMatchObject({ + questions_with_gold_evidence: 1, + retrieval_evaluable_questions: 0, + mean_evidence_recall_at_k: null, + all_evidence_retrieved_rate: null, + }); + }); +}); diff --git a/benchmark/locomo/src/diagnostics.ts b/benchmark/locomo/src/diagnostics.ts new file mode 100644 index 00000000..87bb9eae --- /dev/null +++ b/benchmark/locomo/src/diagnostics.ts @@ -0,0 +1,314 @@ +import { createHash } from "node:crypto"; + +import type { MemorySearchResponse } from "./opencontext-client"; +import type { + LoCoMoFailureStage, + LoCoMoRetrievalTrace, + LoCoMoSessionTrace, + Prediction, + QAPair, +} from "./types"; + +export function sha256Text(text: string): string { + return createHash("sha256").update(text, "utf8").digest("hex"); +} + +function stringArray(value: unknown): string[] { + if (!Array.isArray(value)) return []; + return value.filter((item): item is string => typeof item === "string"); +} + +function optionalString(value: unknown): string[] { + return typeof value === "string" ? [value] : []; +} + +function unique(values: string[]): string[] { + return [...new Set(values)]; +} + +/** Extract LoCoMo dialog-turn references such as D1:3 from source text. */ +export function extractEvidenceIds(content: string): string[] { + return unique(content.match(/\b[A-Za-z]\d+:\d+\b/g) ?? []); +} + +export function buildRetrievalTrace(input: { + qa: QAPair; + response: MemorySearchResponse; + sessionsByMessageId: ReadonlyMap; + availableEvidenceIds: readonly string[]; + userId: string; + topK: number; + latencyMs: number; +}): LoCoMoRetrievalTrace { + const requiredIds = unique(input.qa.evidence); + const requiredSet = new Set(requiredIds); + const availableSet = new Set(input.availableEvidenceIds); + const availableRequiredIds = requiredIds.filter((id) => availableSet.has(id)); + const missingEvidenceIds = requiredIds.filter((id) => !availableSet.has(id)); + const retrievedRequiredIds = new Set(); + + const mapHits = (sourceHits: MemorySearchResponse["results"], trackRetrieved: boolean) => + sourceHits.map((hit, index) => { + const localSession = input.sessionsByMessageId.get(hit.id); + const sessionIds = unique([ + ...optionalString(hit.metadata.sessionId), + ...optionalString(hit.metadata.session_id), + ...stringArray(hit.metadata.sessionIds), + ...stringArray(hit.metadata.session_ids), + ...(localSession ? [localSession.session_id] : []), + ]); + // Match gold references against the returned chunk text. Parent-level + // metadata can contain every turn in a session and would overstate + // child-level recall after daemon chunking. + const evidenceIds = extractEvidenceIds(hit.content); + const matchedEvidenceIds = evidenceIds.filter((id) => requiredSet.has(id)); + if (trackRetrieved) { + for (const evidenceId of matchedEvidenceIds) retrievedRequiredIds.add(evidenceId); + } + return { + rank: index + 1, + id: hit.id, + similarity: hit.similarity, + signals: hit.signals ?? null, + metadata: hit.metadata, + content_sha256: sha256Text(hit.content), + content_characters: hit.content.length, + content_excerpt: hit.content.replace(/\s+/g, " ").trim().slice(0, 240), + ...(trackRetrieved ? { content: hit.content } : {}), + session_ids: sessionIds, + evidence_ids: evidenceIds, + matched_evidence_ids: matchedEvidenceIds, + relevant: requiredIds.length === 0 ? null : matchedEvidenceIds.length > 0, + }; + }); + + const hits = mapHits(input.response.results, true); + const diagnostics = input.response.retrievalDiagnostics; + const candidateChannels = { + semantic: mapHits(diagnostics?.channels.semantic ?? [], false), + lexical: mapHits(diagnostics?.channels.lexical ?? [], false), + hybrid: mapHits(diagnostics?.channels.hybrid ?? [], false), + entity: mapHits(diagnostics?.channels.entity ?? [], false), + }; + const fusedBeforeRerank = mapHits(diagnostics?.fusedBeforeRerank ?? [], false); + const channelRecall = (channelHits: typeof candidateChannels.semantic): number | null => { + if (requiredIds.length === 0) return null; + const matched = new Set(channelHits.flatMap((hit) => hit.matched_evidence_ids)); + return matched.size / requiredIds.length; + }; + + const relevantHits = hits.filter((hit) => hit.relevant === true); + const firstRelevantRank = relevantHits[0]?.rank ?? null; + const retrievalApplicable = requiredIds.length > 0; + const retrievedEvidenceIds = requiredIds.filter((id) => retrievedRequiredIds.has(id)); + const missedEvidenceIds = requiredIds.filter((id) => !retrievedRequiredIds.has(id)); + + return { + status: "completed", + query: input.qa.question, + user_id: input.userId, + top_k: input.topK, + candidate_k: diagnostics?.candidateLimit ?? null, + strategy: "daemon-default", + merge_strategy: diagnostics?.mergeStrategy ?? null, + threshold: null, + backend: diagnostics?.backend ?? null, + semantic_degraded_reason: diagnostics?.semanticDegradedReason ?? null, + candidate_counts: diagnostics?.candidateCounts ?? null, + latency_ms: input.latencyMs, + response_query: input.response.query, + response_sources: input.response.sources, + response_count: input.response.count, + response_warnings: input.response.warnings, + response_reasoning: input.response.reasoning ?? null, + premerge_diagnostics_available: diagnostics !== undefined, + candidate_channels: candidateChannels, + fused_before_rerank: fusedBeforeRerank, + reranker: diagnostics?.reranker + ? { + enabled: diagnostics.reranker.enabled, + provider: diagnostics.reranker.provider ?? null, + model: diagnostics.reranker.model ?? null, + input_count: diagnostics.reranker.inputCount, + output_count: diagnostics.reranker.outputCount, + latency_ms: diagnostics.reranker.latencyMs, + order_changed: diagnostics.reranker.orderChanged, + } + : null, + hits, + retrieval_applicable: retrievalApplicable, + evidence_granularity: retrievalApplicable ? "dialog_turn" : "unavailable", + required_evidence_ids: requiredIds, + available_evidence_ids: availableRequiredIds, + missing_evidence_ids: missingEvidenceIds, + retrieved_evidence_ids: retrievedEvidenceIds, + missed_evidence_ids: missedEvidenceIds, + dataset_evidence_coverage: retrievalApplicable ? availableRequiredIds.length / requiredIds.length : null, + evidence_recall_at_k: retrievalApplicable ? retrievedRequiredIds.size / requiredIds.length : null, + retrievable_evidence_recall_at_k: + availableRequiredIds.length > 0 ? retrievedRequiredIds.size / availableRequiredIds.length : null, + all_evidence_retrieved: retrievalApplicable ? retrievedRequiredIds.size === requiredIds.length : null, + hit_at_k: retrievalApplicable ? (firstRelevantRank === null ? 0 : 1) : null, + first_relevant_rank: firstRelevantRank, + mrr: firstRelevantRank === null ? (retrievalApplicable ? 0 : null) : 1 / firstRelevantRank, + precision_at_k: retrievalApplicable ? relevantHits.length / input.topK : null, + relevant_hit_precision: retrievalApplicable + ? hits.length === 0 + ? 0 + : relevantHits.length / hits.length + : null, + semantic_evidence_recall_at_candidate_k: channelRecall(candidateChannels.semantic), + lexical_evidence_recall_at_candidate_k: channelRecall(candidateChannels.lexical), + hybrid_evidence_recall_at_candidate_k: channelRecall(candidateChannels.hybrid), + }; +} + +export function buildRetrievalErrorTrace(input: { + qa: QAPair; + availableEvidenceIds: readonly string[]; + userId: string; + topK: number; + latencyMs: number; + error: string; +}): LoCoMoRetrievalTrace { + const requiredIds = unique(input.qa.evidence); + const availableSet = new Set(input.availableEvidenceIds); + const availableIds = requiredIds.filter((id) => availableSet.has(id)); + const missingIds = requiredIds.filter((id) => !availableSet.has(id)); + return { + status: "execution_error", + query: input.qa.question, + user_id: input.userId, + top_k: input.topK, + candidate_k: null, + strategy: "daemon-default", + merge_strategy: null, + threshold: null, + backend: null, + semantic_degraded_reason: null, + candidate_counts: null, + latency_ms: input.latencyMs, + response_query: input.qa.question, + response_sources: [], + response_count: 0, + response_warnings: [], + response_reasoning: null, + premerge_diagnostics_available: false, + candidate_channels: { semantic: [], lexical: [], hybrid: [], entity: [] }, + fused_before_rerank: [], + reranker: null, + hits: [], + retrieval_applicable: requiredIds.length > 0, + evidence_granularity: requiredIds.length > 0 ? "dialog_turn" : "unavailable", + required_evidence_ids: requiredIds, + available_evidence_ids: availableIds, + missing_evidence_ids: missingIds, + retrieved_evidence_ids: [], + missed_evidence_ids: requiredIds, + dataset_evidence_coverage: requiredIds.length > 0 ? availableIds.length / requiredIds.length : null, + evidence_recall_at_k: null, + retrievable_evidence_recall_at_k: null, + all_evidence_retrieved: null, + hit_at_k: null, + first_relevant_rank: null, + mrr: null, + precision_at_k: null, + relevant_hit_precision: null, + semantic_evidence_recall_at_candidate_k: null, + lexical_evidence_recall_at_candidate_k: null, + hybrid_evidence_recall_at_candidate_k: null, + error: input.error, + }; +} + +export function deriveFailureStage(input: { + qa: QAPair; + retrieval: LoCoMoRetrievalTrace | null; + correct: boolean; + executionStage?: "ingest" | "retrieval" | "answerer" | "judge" | "provider"; +}): LoCoMoFailureStage { + if (input.executionStage === "ingest") return "ingest_or_index_error"; + if (input.executionStage === "retrieval") return "retrieval_error"; + if (input.executionStage === "answerer") return "answerer_error"; + if (input.executionStage === "judge") return "judge_error"; + if (input.executionStage === "provider") return "provider_error"; + if (input.correct) return "none"; + if (input.qa.evidence.length === 0) return "gold_evidence_unavailable"; + if ((input.retrieval?.missing_evidence_ids.length ?? 0) > 0) { + return input.retrieval?.available_evidence_ids.length === 0 + ? "dataset_reference_missing" + : "dataset_reference_partial"; + } + if (!input.retrieval || input.retrieval.evidence_recall_at_k === 0) return "retrieval_miss"; + if ((input.retrieval.evidence_recall_at_k ?? 0) < 1) return "retrieval_partial"; + return "context_present_answer_failed"; +} + +export function calculateDiagnosticSummary(predictions: Prediction[]): Record { + const failureStages: Record = {}; + const recalls: number[] = []; + const retrievableRecalls: number[] = []; + const sourceCoverages: number[] = []; + const hitAtK: number[] = []; + const reciprocalRanks: number[] = []; + const precisionAtK: number[] = []; + const semanticCandidateRecalls: number[] = []; + const lexicalCandidateRecalls: number[] = []; + const hybridCandidateRecalls: number[] = []; + let completed = 0; + let executionErrors = 0; + let questionsWithGoldEvidence = 0; + let retrievalEvaluable = 0; + let allEvidenceRetrieved = 0; + let premergeDiagnostics = 0; + + for (const prediction of predictions) { + failureStages[prediction.failure_stage] = (failureStages[prediction.failure_stage] ?? 0) + 1; + if (prediction.status === "completed") completed++; + else executionErrors++; + if ((prediction.evidence?.length ?? 0) > 0) questionsWithGoldEvidence++; + const retrieval = prediction.trace.retrieval; + if (retrieval?.premerge_diagnostics_available) premergeDiagnostics++; + if (!retrieval?.retrieval_applicable) continue; + if (retrieval.status !== "completed") continue; + retrievalEvaluable++; + if (retrieval.all_evidence_retrieved) allEvidenceRetrieved++; + if (retrieval.evidence_recall_at_k !== null) recalls.push(retrieval.evidence_recall_at_k); + if (retrieval.retrievable_evidence_recall_at_k !== null) + retrievableRecalls.push(retrieval.retrievable_evidence_recall_at_k); + if (retrieval.dataset_evidence_coverage !== null) + sourceCoverages.push(retrieval.dataset_evidence_coverage); + if (retrieval.hit_at_k !== null) hitAtK.push(retrieval.hit_at_k); + if (retrieval.mrr !== null) reciprocalRanks.push(retrieval.mrr); + if (retrieval.precision_at_k !== null) precisionAtK.push(retrieval.precision_at_k); + if (retrieval.semantic_evidence_recall_at_candidate_k !== null) + semanticCandidateRecalls.push(retrieval.semantic_evidence_recall_at_candidate_k); + if (retrieval.lexical_evidence_recall_at_candidate_k !== null) + lexicalCandidateRecalls.push(retrieval.lexical_evidence_recall_at_candidate_k); + if (retrieval.hybrid_evidence_recall_at_candidate_k !== null) + hybridCandidateRecalls.push(retrieval.hybrid_evidence_recall_at_candidate_k); + } + + const mean = (values: number[]): number | null => + values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length; + return { + completed, + execution_errors: executionErrors, + execution_error_rate: predictions.length === 0 ? 0 : executionErrors / predictions.length, + failure_stages: failureStages, + questions_with_gold_evidence: questionsWithGoldEvidence, + questions_without_gold_evidence: predictions.length - questionsWithGoldEvidence, + retrieval_evaluable_questions: retrievalEvaluable, + premerge_diagnostics_questions: premergeDiagnostics, + mean_semantic_evidence_recall_at_candidate_k: mean(semanticCandidateRecalls), + mean_lexical_evidence_recall_at_candidate_k: mean(lexicalCandidateRecalls), + mean_hybrid_evidence_recall_at_candidate_k: mean(hybridCandidateRecalls), + mean_evidence_recall_at_k: mean(recalls), + mean_retrievable_evidence_recall_at_k: mean(retrievableRecalls), + mean_dataset_evidence_coverage: mean(sourceCoverages), + hit_at_k_rate: mean(hitAtK), + mean_precision_at_k: mean(precisionAtK), + mean_reciprocal_rank: mean(reciprocalRanks), + all_evidence_retrieved_rate: retrievalEvaluable === 0 ? null : allEvidenceRetrieved / retrievalEvaluable, + }; +} diff --git a/benchmark/locomo/src/evaluator.test.ts b/benchmark/locomo/src/evaluator.test.ts index cf8e7103..e276df01 100644 --- a/benchmark/locomo/src/evaluator.test.ts +++ b/benchmark/locomo/src/evaluator.test.ts @@ -5,33 +5,45 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; vi.mock("./opencontext-client", async (importOriginal) => { const actual = await importOriginal(); - return { - ...actual, - generateAnswer: vi.fn(), - searchMemory: vi.fn(), - }; + return { ...actual, generateAnswer: vi.fn(), searchMemory: vi.fn() }; }); vi.mock("./metrics", async (importOriginal) => { const actual = await importOriginal(); - return { - ...actual, - evaluateLLMJudge: vi.fn(), - }; + return { ...actual, evaluateLLMJudge: vi.fn() }; }); -import { LoCoMoEvaluator } from "./evaluator"; -import { JUDGE_MODEL, evaluateLLMJudge, parseLLMJudgeResponse } from "./metrics"; +import { buildSampleMessages, LoCoMoEvaluator } from "./evaluator"; +import { evaluateLLMJudge, getJudgeModelIdentity, parseLLMJudgeResponse } from "./metrics"; import { generateAnswer, searchMemory } from "./opencontext-client"; import type { LoCoMoSample } from "./types"; +import { RetrievalMode } from "./types"; const originalAuthToken = process.env.ANTHROPIC_AUTH_TOKEN; const originalAnswerModel = process.env.ANSWER_MODEL; +const originalJudgeModel = process.env.OPENROUTER_JUDGE_MODEL; const temporaryDirectories: string[] = []; const fixtureUsage = { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }; function judgeResult(score: number) { - return { score, token_usage: fixtureUsage }; + return { + score, + token_usage: fixtureUsage, + status: "completed" as const, + attempt: 1, + latency_ms: 1, + prompt_version: "locomo-judge-v1", + prompt_sha256: "fixture-hash", + prompt_characters: 10, + system_prompt: "fixture system", + prompt: "fixture prompt", + raw_response: score === 1 ? '{"label":"CORRECT"}' : '{"label":"WRONG"}', + parse_status: "parsed" as const, + }; +} + +function searchResponse() { + return { query: "fixture", sources: ["memory"], results: [], count: 0, warnings: [] }; } function restoreEnvironment(name: string, value: string | undefined): void { @@ -46,7 +58,12 @@ async function createCheckpointDir(): Promise { } function createEvaluator(checkpointDir: string, resume = true): LoCoMoEvaluator { - const evaluator = new LoCoMoEvaluator("observation", "http://fixture.invalid", undefined, resume); + const evaluator = new LoCoMoEvaluator( + RetrievalMode.OBSERVATION, + "http://fixture.invalid", + undefined, + resume, + ); Object.assign(evaluator, { checkpointDir, ingestedCount: 1 }); return evaluator; } @@ -71,20 +88,26 @@ beforeEach(() => { vi.clearAllMocks(); process.env.ANTHROPIC_AUTH_TOKEN = "fixture-token"; process.env.ANSWER_MODEL = "answerer-a"; - vi.mocked(searchMemory).mockResolvedValue([]); - vi.mocked(generateAnswer).mockResolvedValue({ text: "fixture response", token_usage: fixtureUsage }); + process.env.OPENROUTER_JUDGE_MODEL = "judge-a"; + vi.mocked(searchMemory).mockResolvedValue(searchResponse()); + vi.mocked(generateAnswer).mockResolvedValue({ + text: "fixture response", + token_usage: fixtureUsage, + attempt: 1, + }); }); afterEach(async () => { restoreEnvironment("ANTHROPIC_AUTH_TOKEN", originalAuthToken); restoreEnvironment("ANSWER_MODEL", originalAnswerModel); + restoreEnvironment("OPENROUTER_JUDGE_MODEL", originalJudgeModel); await Promise.all( temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })), ); }); describe("LoCoMo checkpoint resume", () => { - it("reuses both correct and incorrect completed results across repeated resumes", async () => { + it("reuses both correct and incorrect completed results", async () => { const checkpointDir = await createCheckpointDir(); const sample = createSample(2); vi.mocked(evaluateLLMJudge).mockResolvedValueOnce(judgeResult(1)).mockResolvedValueOnce(judgeResult(0)); @@ -92,21 +115,35 @@ describe("LoCoMo checkpoint resume", () => { const first = await createEvaluator(checkpointDir).evaluateQA(sample); expect(first.correct_answers).toBe(1); expect(first.token_usage).toEqual({ prompt_tokens: 40, completion_tokens: 20, total_tokens: 60 }); - expect(first.predictions.map((prediction) => prediction.status)).toEqual(["completed", "completed"]); vi.mocked(generateAnswer).mockClear(); vi.mocked(evaluateLLMJudge).mockClear(); - const second = await createEvaluator(checkpointDir).evaluateQA(sample); - const third = await createEvaluator(checkpointDir).evaluateQA(sample); + const resumed = await createEvaluator(checkpointDir).evaluateQA(sample); expect(generateAnswer).not.toHaveBeenCalled(); expect(evaluateLLMJudge).not.toHaveBeenCalled(); - expect(second.correct_answers).toBe(1); - expect(third.correct_answers).toBe(1); - expect(second.predictions.map((prediction) => prediction.correct)).toEqual([true, false]); + expect(resumed.predictions.map((prediction) => prediction.correct)).toEqual([true, false]); + }); + + it("restores session evidence when an entire sample is checkpoint-complete", async () => { + const checkpointDir = await createCheckpointDir(); + const sample = createSample(); + sample.observation = { session_1_observation: { Alice: [["Adopted Luna", "D1:3"]] } }; + sample.conversation = { + session_1_date_time: "2024-01-02", + session_1: [{ speaker: "Alice", text: "I adopted Luna.", dia_id: "D1:3" }], + }; + vi.mocked(evaluateLLMJudge).mockResolvedValueOnce(judgeResult(1)); + await createEvaluator(checkpointDir).evaluateQA(sample); + + const resumed = createEvaluator(checkpointDir); + expect(await resumed.reuseCompletedSample(sample)).toBe(true); + expect(resumed.getSessionTraces()).toMatchObject([ + { session_id: "1", evidence_ids: ["D1:3"], ingest_status: "completed", ingest_latency_ms: null }, + ]); }); - it("retries execution errors and increments the attempt", async () => { + it("retries execution errors and increments the benchmark attempt", async () => { const checkpointDir = await createCheckpointDir(); const sample = createSample(); vi.mocked(evaluateLLMJudge).mockRejectedValueOnce(new Error("judge parse failure")); @@ -115,7 +152,7 @@ describe("LoCoMo checkpoint resume", () => { expect(failed.predictions[0]).toMatchObject({ status: "execution_error", attempt: 1, - error: "judge parse failure", + execution_error: { stage: "judge", message: "judge parse failure" }, }); vi.mocked(generateAnswer).mockClear(); @@ -126,21 +163,7 @@ describe("LoCoMo checkpoint resume", () => { expect(retried.predictions[0]).toMatchObject({ status: "completed", attempt: 2, correct: false }); }); - it("ignores existing checkpoints when resume is disabled", async () => { - const checkpointDir = await createCheckpointDir(); - const sample = createSample(); - vi.mocked(evaluateLLMJudge).mockResolvedValueOnce(judgeResult(0)); - await createEvaluator(checkpointDir).evaluateQA(sample); - - vi.mocked(generateAnswer).mockClear(); - vi.mocked(evaluateLLMJudge).mockResolvedValueOnce(judgeResult(1)); - const fresh = await createEvaluator(checkpointDir, false).evaluateQA(sample); - - expect(generateAnswer).toHaveBeenCalledOnce(); - expect(fresh.predictions[0]).toMatchObject({ status: "completed", attempt: 1, correct: true }); - }); - - it("does not reuse a checkpoint from a different answerer or judge model", async () => { + it("does not reuse checkpoints from different model identities", async () => { const checkpointDir = await createCheckpointDir(); const sample = createSample(); vi.mocked(evaluateLLMJudge).mockResolvedValueOnce(judgeResult(0)); @@ -150,15 +173,13 @@ describe("LoCoMo checkpoint resume", () => { vi.mocked(generateAnswer).mockClear(); vi.mocked(evaluateLLMJudge).mockResolvedValueOnce(judgeResult(1)); const rerun = await createEvaluator(checkpointDir).evaluateQA(sample); - - expect(generateAnswer).toHaveBeenCalledOnce(); expect(rerun.predictions[0]).toMatchObject({ status: "completed", - attempt: 2, + attempt: 1, answerer_model: "anthropic-compatible:answerer-b", }); - const checkpointPath = join(checkpointDir, "fixture-sample.json"); + const checkpointPath = join(checkpointDir, "fixture-sample.observation.json"); const checkpoint = JSON.parse(await readFile(checkpointPath, "utf-8")) as Record< string, Record @@ -169,25 +190,23 @@ describe("LoCoMo checkpoint resume", () => { vi.mocked(generateAnswer).mockClear(); vi.mocked(evaluateLLMJudge).mockResolvedValueOnce(judgeResult(0)); const judgeRerun = await createEvaluator(checkpointDir).evaluateQA(sample); - expect(generateAnswer).toHaveBeenCalledOnce(); expect(judgeRerun.predictions[0]).toMatchObject({ status: "completed", - attempt: 3, - judge_model: JUDGE_MODEL, + attempt: 1, + judge_model: getJudgeModelIdentity(), }); }); - it("records an empty answer as an execution error without calling the judge", async () => { + it("records an empty answer as an answerer execution error", async () => { const checkpointDir = await createCheckpointDir(); - vi.mocked(generateAnswer).mockResolvedValueOnce({ text: "", token_usage: fixtureUsage }); + vi.mocked(generateAnswer).mockResolvedValueOnce({ text: "", token_usage: fixtureUsage, attempt: 1 }); const result = await createEvaluator(checkpointDir).evaluateQA(createSample()); - expect(evaluateLLMJudge).not.toHaveBeenCalled(); expect(result.predictions[0]).toMatchObject({ status: "execution_error", - error: "Answerer returned an empty response", + execution_error: { stage: "answerer", message: "Answerer returned an empty response" }, }); }); }); @@ -198,4 +217,85 @@ describe("LoCoMo judge parsing", () => { expect(parseLLMJudgeResponse("WRONG")).toBe(0); expect(() => parseLLMJudgeResponse("unknown")).toThrow("could not be parsed"); }); + + it("extracts a JSON label after an explanation without matching incorrectly as CORRECT", () => { + expect(parseLLMJudgeResponse('The generated answer is incorrect. {"label": "WRONG"}')).toBe(0); + }); +}); + +describe("LoCoMo raw-session mapping", () => { + it("preserves one complete dialog session as one RawMessage and retains turn ids", () => { + const sample = createSample(); + sample.conversation = { + speaker_a: "Alice", + speaker_b: "Bob", + session_1_date_time: "2024-01-02", + session_1: [ + { speaker: "Alice", text: "I adopted Luna.", dia_id: "D1:3" }, + { speaker: "Bob", text: "That is wonderful.", dia_id: "D1:4" }, + ], + }; + + const built = buildSampleMessages(sample, RetrievalMode.DIALOG); + + expect(built.messages).toHaveLength(1); + expect(built.messages[0]).toMatchObject({ + messageId: "locomo_fixture-sample_dialog_1", + timestamp: Date.UTC(2024, 0, 2), + metadata: { + sampleId: "fixture-sample", + sessionId: "1", + sessionDate: "2024-01-02", + evidenceIds: ["D1:3", "D1:4"], + }, + }); + expect(built.messages[0]?.content).toContain("[D1:3] [Alice] I adopted Luna."); + expect(built.sessions[0]).toMatchObject({ session_id: "1", ingest_status: "pending" }); + }); + + it("retains MiniCPM image descriptions in the source RawMessage", () => { + const sample = createSample(); + sample.conversation = { + session_1_date_time: "2024-01-02", + session_1: [ + { + speaker: "Alice", + text: "Here is the photo from my trip.", + dia_id: "D1:3", + minicpm_caption: "A red bicycle beside the Eiffel Tower.", + }, + ], + }; + + const built = buildSampleMessages(sample, RetrievalMode.DIALOG); + expect(built.messages[0]?.content).toContain( + "[Image description: A red bicycle beside the Eiffel Tower.]", + ); + }); + + it("parses native LoCoMo timestamps deterministically", () => { + const sample = createSample(); + sample.conversation = { + session_1_date_time: "1:56 pm on 8 May, 2023", + session_1: [{ speaker: "Alice", text: "Timestamped turn", dia_id: "D1:1" }], + }; + + const built = buildSampleMessages(sample, RetrievalMode.DIALOG); + expect(built.messages[0]?.timestamp).toBe(Date.UTC(2023, 4, 8, 13, 56)); + }); + + it("adds observation references and original turns without splitting a session", () => { + const sample = createSample(); + sample.conversation = { + session_2_date_time: "2024-02-03", + session_2: [{ speaker: "Alice", text: "Trip detail", dia_id: "D2:1" }], + }; + sample.observation = { session_2_observation: { Alice: [["Planned a trip", "D2:1"]] } }; + + const built = buildSampleMessages(sample, RetrievalMode.OBSERVATION); + expect(built.messages).toHaveLength(1); + expect(built.messages[0]?.content).toContain("Alice: Planned a trip [Ref: D2:1]"); + expect(built.messages[0]?.content).toContain("[D2:1] [Alice] Trip detail"); + expect(built.sessions[0]?.evidence_ids).toEqual(["D2:1"]); + }); }); diff --git a/benchmark/locomo/src/evaluator.ts b/benchmark/locomo/src/evaluator.ts index ba89d148..c68733cd 100644 --- a/benchmark/locomo/src/evaluator.ts +++ b/benchmark/locomo/src/evaluator.ts @@ -1,289 +1,375 @@ /** - * LoCoMo Evaluator for the OpenContext memory store. - * - * Flow (no agent, no filesystem — pure memory-store HTTP): - * 1. loadSample: convert the sample's sessions into raw messages and POST - * them to the OpenContext daemon (`POST /v1/raw-messages`, embedOnInsert). - * Granularity is unchanged: one memory message per session record - * (dialog / observation / session_summary, depending on retrieval mode). - * 2. evaluateQA: retrieve relevant memories (`POST /v1/search`), then ask - * the answerer LLM (see opencontext-client.ts) using only the retrieved - * excerpts. - * 3. Judge with the existing metrics.ts model and prompt (OpenRouter). + * LoCoMo evaluator. The benchmark maps one upstream session to one RawMessage; + * OpenContext owns chunking, indexing, retrieval, fusion, and reranking. */ import { mkdir, readFile, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; -import { sumTokenUsage, unavailableTokenUsage } from "../../run-support"; -import { JUDGE_MODEL, calculateMetrics, evaluateLLMJudge } from "./metrics"; +import { sumTokenUsage, type TokenUsage, unavailableTokenUsage } from "../../run-support"; import { + buildRetrievalErrorTrace, + buildRetrievalTrace, + deriveFailureStage, + extractEvidenceIds, + sha256Text, +} from "./diagnostics"; +import { calculateMetrics, evaluateLLMJudge, getJudgeModelIdentity } from "./metrics"; +import { + AnswererGenerationError, type BenchRawMessage, - type GeneratedAnswer, + INGEST_BATCH_SIZE, + type IngestBatchTrace, + IngestMessagesError, type MemorySearchHit, + type MemorySearchResponse, + checkOpencontextHealth, generateAnswer, getAnswererModelIdentity, getOpencontextBaseUrl, ingestMessages, searchMemory, } from "./opencontext-client"; -import { RetrievalMode } from "./types"; -import type { EvaluationResult, LoCoMoSample, Prediction, QAPair } from "./types"; - -/** How many retrieved memories are shown to the answerer. */ -export const RETRIEVAL_LIMIT = 8; - -function isReusableCheckpoint( - prediction: Prediction | undefined, - answererModel: string, - judgeModel: string, -): boolean { - return ( - prediction?.status === "completed" && - prediction.answerer_model === answererModel && - prediction.judge_model === judgeModel - ); +import { + LOCOMO_TRACE_SCHEMA_VERSION, + type EvaluationResult, + type LoCoMoAnswerTrace, + type LoCoMoJudgeTrace, + type LoCoMoRetrievalTrace, + type LoCoMoSample, + type LoCoMoSessionTrace, + type Prediction, + type QAPair, + RetrievalMode, +} from "./types"; + +const configuredRetrievalLimit = Number.parseInt(process.env.LOCOMO_TOP_K ?? "8", 10); +export const RETRIEVAL_LIMIT = + Number.isInteger(configuredRetrievalLimit) && configuredRetrievalLimit > 0 + ? Math.min(50, configuredRetrievalLimit) + : 8; + +interface DialogTurn { + speaker?: string; + dia_id?: string; + text?: string; + minicpm_caption?: string; } -/** - * Parse timestamp string to Unix ms. - */ -function parseTimestamp(ts: string): number | undefined { - if (!ts) return undefined; - try { - const date = new Date(ts); - if (!Number.isNaN(date.getTime())) { - return date.getTime(); +function unique(values: string[]): string[] { + return [...new Set(values)]; +} + +function parseTimestamp(timestamp: string): number | undefined { + if (!timestamp) return undefined; + const locomoDate = timestamp.match( + /^(\d{1,2}):(\d{2})\s*(am|pm)\s+on\s+(\d{1,2})\s+([A-Za-z]+),?\s+(\d{4})$/i, + ); + if (locomoDate) { + const [, hourText, minuteText, meridiem, dayText, monthText, yearText] = locomoDate; + const months = [ + "january", + "february", + "march", + "april", + "may", + "june", + "july", + "august", + "september", + "october", + "november", + "december", + ]; + const month = months.indexOf(monthText.toLowerCase()); + let hour = Number.parseInt(hourText, 10) % 12; + if (meridiem.toLowerCase() === "pm") hour += 12; + if (month >= 0) { + return Date.UTC( + Number.parseInt(yearText, 10), + month, + Number.parseInt(dayText, 10), + hour, + Number.parseInt(minuteText, 10), + ); } - const parsed = Date.parse(ts); - return Number.isNaN(parsed) ? undefined : parsed; - } catch { - return undefined; } + const parsed = Date.parse(timestamp); + return Number.isNaN(parsed) ? undefined : parsed; } -function toBenchMessage( - id: string, - sessionTimestamp: string, - content: string, - metadata: Record, - now: number, -): BenchRawMessage { - return { - messageId: `locomo_${id}`, - userId: "benchmark_user", - platform: "benchmark", - botId: "locomo", - timestamp: parseTimestamp(sessionTimestamp) ?? now, - content, - createdAt: now, - metadata, - }; +function normalizedSessionId(value: string): string { + return value.replace(/^session_/, ""); } -/** - * Format conversation data into raw memory messages (one per session). - */ -function createMessagesFromDialog(sample: LoCoMoSample, now: number): BenchRawMessage[] { - const messages: BenchRawMessage[] = []; - const speakerA = sample.conversation.speaker_a ?? "Speaker A"; - const speakerB = sample.conversation.speaker_b ?? "Speaker B"; +function sessionSortValue(value: string): [number, string] { + const match = value.match(/(\d+)/); + return [match ? Number.parseInt(match[1], 10) : Number.MAX_SAFE_INTEGER, value]; +} - for (const key of Object.keys(sample.conversation).sort()) { - if (!key.startsWith("session_") || key.endsWith("_date_time")) { - continue; - } +function sortSessionKeys(keys: string[]): string[] { + return keys.sort((left, right) => { + const [leftNumber, leftText] = sessionSortValue(left); + const [rightNumber, rightText] = sessionSortValue(right); + return leftNumber - rightNumber || leftText.localeCompare(rightText); + }); +} - const sessionNum = key.replace("session_", ""); - const datetimeKey = `session_${sessionNum}_date_time`; - const sessionTimestamp = String(sample.conversation[datetimeKey] ?? ""); - const session = sample.conversation[key] as unknown[]; +function conversationDate(sample: LoCoMoSample, sessionId: string): string { + return String(sample.conversation[`session_${sessionId}_date_time`] ?? ""); +} - const dialogParts: string[] = []; - dialogParts.push(`# Conversation Session ${sessionNum}`); - if (sessionTimestamp) { - dialogParts.push(`# Timestamp: ${sessionTimestamp}`); - } - dialogParts.push(`# Speakers: ${speakerA}, ${speakerB}`); - dialogParts.push(""); - - for (const turn of session) { - // Each turn is an object with speaker, dia_id, and text properties - const turnText = - typeof turn === "string" ? turn : (turn as { text?: string }).text || JSON.stringify(turn); - const speaker = typeof turn === "string" ? "" : `[${(turn as { speaker?: string }).speaker || ""}] `; - if (sessionTimestamp) { - dialogParts.push(`[${sessionTimestamp}] ${speaker}${turnText}`); - } else { - dialogParts.push(`${speaker}${turnText}`); - } - } +function conversationTurns(sample: LoCoMoSample, sessionId: string): unknown[] { + const value = sample.conversation[`session_${sessionId}`]; + return Array.isArray(value) ? value : []; +} - messages.push( - toBenchMessage( - `${sample.sample_id}_dialog_${sessionNum}`, - sessionTimestamp, - dialogParts.join("\n"), - { - sampleId: sample.sample_id, - sessionId: sessionNum, - contentType: "dialog", - }, - now, - ), - ); - } +function dialogTurn(value: unknown): DialogTurn | null { + return typeof value === "object" && value !== null ? (value as DialogTurn) : null; +} - return messages; +function formatDialogTurn(value: unknown, timestamp: string): string { + if (typeof value === "string") return timestamp ? `[${timestamp}] ${value}` : value; + const turn = dialogTurn(value); + const evidence = turn?.dia_id ? `[${turn.dia_id}] ` : ""; + const speaker = turn?.speaker ? `[${turn.speaker}] ` : ""; + const text = turn?.text ?? JSON.stringify(value); + const caption = turn?.minicpm_caption?.trim(); + const imageDescription = caption ? `\n[Image description: ${caption}]` : ""; + return `${timestamp ? `[${timestamp}] ` : ""}${evidence}${speaker}${text}${imageDescription}`; } -/** - * Format observation data into raw memory messages (one per session). - */ -function createMessagesFromObservation(sample: LoCoMoSample, now: number): BenchRawMessage[] { - const messages: BenchRawMessage[] = []; +function evidenceIdsFromTurns(turns: unknown[]): string[] { + return unique( + turns.flatMap((value) => { + const id = dialogTurn(value)?.dia_id; + return typeof id === "string" && id.length > 0 ? [id] : []; + }), + ); +} - for (const key of Object.keys(sample.observation).sort()) { - if (!key.endsWith("_observation")) { - continue; - } +function toMessageAndTrace(input: { + sample: LoCoMoSample; + retrievalMode: RetrievalMode; + sessionId: string; + sessionIndex: number; + timestamp: string; + content: string; + evidenceIds: string[]; + now: number; +}): { message: BenchRawMessage; trace: LoCoMoSessionTrace } { + const messageId = `locomo_${input.sample.sample_id}_${input.retrievalMode}_${input.sessionId}`; + const message: BenchRawMessage = { + messageId, + userId: "benchmark_user", + platform: "benchmark", + botId: "locomo", + timestamp: parseTimestamp(input.timestamp) ?? input.now, + content: input.content, + createdAt: input.now, + metadata: { + sampleId: input.sample.sample_id, + sessionId: input.sessionId, + contentType: input.retrievalMode, + sessionDate: input.timestamp || null, + evidenceIds: input.evidenceIds, + }, + }; + return { + message, + trace: { + schema_version: LOCOMO_TRACE_SCHEMA_VERSION, + sample_id: input.sample.sample_id, + retrieval_mode: input.retrievalMode, + message_id: messageId, + session_id: input.sessionId, + session_index: input.sessionIndex, + ingest_batch_index: Math.floor(input.sessionIndex / INGEST_BATCH_SIZE), + session_date: input.timestamp || null, + evidence_ids: input.evidenceIds, + content_sha256: sha256Text(input.content), + content_characters: input.content.length, + ingest_status: "pending", + ingest_latency_ms: null, + ingest_warnings: [], + }, + }; +} - const sessionNum = key.replace("_observation", ""); - const datetimeKey = `${sessionNum}_date_time`; - const sessionTimestamp = String(sample.conversation[datetimeKey] ?? ""); - const obsContent = sample.observation[key]; +function createMessagesFromDialog(sample: LoCoMoSample, now: number) { + const speakerA = sample.conversation.speaker_a ?? "Speaker A"; + const speakerB = sample.conversation.speaker_b ?? "Speaker B"; + const keys = sortSessionKeys( + Object.keys(sample.conversation).filter( + (key) => key.startsWith("session_") && !key.endsWith("_date_time"), + ), + ); + return keys.map((key, sessionIndex) => { + const sessionId = normalizedSessionId(key); + const timestamp = conversationDate(sample, sessionId); + const turns = conversationTurns(sample, sessionId); + const parts = [ + `# Conversation Session ${sessionId}`, + ...(timestamp ? [`# Timestamp: ${timestamp}`] : []), + `# Speakers: ${speakerA}, ${speakerB}`, + "", + ...turns.map((turn) => formatDialogTurn(turn, timestamp)), + ]; + return toMessageAndTrace({ + sample, + retrievalMode: RetrievalMode.DIALOG, + sessionId, + sessionIndex, + timestamp, + content: parts.join("\n"), + evidenceIds: evidenceIdsFromTurns(turns), + now, + }); + }); +} - const obsParts: string[] = []; - obsParts.push(`# Observation Summary ${sessionNum}`); - if (sessionTimestamp) { - obsParts.push(`# Session Date: ${sessionTimestamp}`); - } - obsParts.push(""); - - // Add observation summary with dialog references - if (typeof obsContent === "object" && obsContent !== null) { - for (const [speaker, utterances] of Object.entries(obsContent)) { - if (Array.isArray(utterances)) { - for (const item of utterances) { - if (Array.isArray(item) && item.length >= 2) { - const [text, diaId] = item; - // Include text and its dialog reference - obsParts.push(`${speaker}: ${text} [Ref: ${diaId}]`); - } else { - obsParts.push(`${speaker}: ${item}`); - } - } +function createMessagesFromObservation(sample: LoCoMoSample, now: number) { + const keys = sortSessionKeys(Object.keys(sample.observation).filter((key) => key.endsWith("_observation"))); + return keys.map((key, sessionIndex) => { + const sessionId = normalizedSessionId(key.replace(/_observation$/, "")); + const timestamp = conversationDate(sample, sessionId); + const observation = sample.observation[key]; + const parts = [ + `# Observation Summary ${sessionId}`, + ...(timestamp ? [`# Session Date: ${timestamp}`] : []), + "", + ]; + if (typeof observation === "object" && observation !== null) { + for (const [speaker, utterances] of Object.entries(observation)) { + if (!Array.isArray(utterances)) continue; + for (const item of utterances) { + if (Array.isArray(item) && item.length >= 2) parts.push(`${speaker}: ${item[0]} [Ref: ${item[1]}]`); + else parts.push(`${speaker}: ${String(item)}`); } } } else { - obsParts.push(String(obsContent)); + parts.push(String(observation)); } + const turns = conversationTurns(sample, sessionId); + if (turns.length > 0) { + parts.push("", "# Original Dialog (for date/time reasoning):"); + parts.push(...turns.map((turn) => formatDialogTurn(turn, ""))); + } + const content = parts.join("\n"); + return toMessageAndTrace({ + sample, + retrievalMode: RetrievalMode.OBSERVATION, + sessionId, + sessionIndex, + timestamp, + content, + evidenceIds: unique([...evidenceIdsFromTurns(turns), ...extractEvidenceIds(content)]), + now, + }); + }); +} - obsParts.push(""); - - // Add original dialog for this session to enable temporal reasoning - const sessionKey = `session_${sessionNum}`; - const dialogContent = sample.conversation[sessionKey]; - if (Array.isArray(dialogContent)) { - obsParts.push("# Original Dialog (for date/time reasoning):"); - for (const turn of dialogContent) { - if (typeof turn === "object" && turn !== null && "speaker" in turn && "text" in turn) { - obsParts.push(`${turn.speaker}: ${turn.text}`); - } else if (typeof turn === "string") { - obsParts.push(turn); - } - } +function createMessagesFromSummary(sample: LoCoMoSample, now: number) { + const keys = sortSessionKeys(Object.keys(sample.session_summary).filter((key) => key.endsWith("_summary"))); + return keys.map((key, sessionIndex) => { + const sessionId = normalizedSessionId(key.replace(/_summary$/, "")); + const timestamp = conversationDate(sample, sessionId); + const summary = sample.session_summary[key]; + const parts = [`# Session Summary ${sessionId}`, ...(timestamp ? [`# Timestamp: ${timestamp}`] : []), ""]; + if (typeof summary === "object" && summary !== null) { + for (const [speaker, text] of Object.entries(summary)) parts.push(`${speaker}: ${String(text)}`); + } else { + parts.push(String(summary)); } + const content = parts.join("\n"); + return toMessageAndTrace({ + sample, + retrievalMode: RetrievalMode.SESSION_SUMMARY, + sessionId, + sessionIndex, + timestamp, + content, + evidenceIds: extractEvidenceIds(content), + now, + }); + }); +} - messages.push( - toBenchMessage( - `${sample.sample_id}_observation_${sessionNum}`, - sessionTimestamp, - obsParts.join("\n"), - { - sampleId: sample.sample_id, - sessionId: sessionNum, - contentType: "observation", - }, - now, - ), - ); - } +export function buildSampleMessages( + sample: LoCoMoSample, + retrievalMode: RetrievalMode, +): { messages: BenchRawMessage[]; sessions: LoCoMoSessionTrace[] } { + const now = Date.now(); + const built = + retrievalMode === RetrievalMode.DIALOG + ? createMessagesFromDialog(sample, now) + : retrievalMode === RetrievalMode.OBSERVATION + ? createMessagesFromObservation(sample, now) + : createMessagesFromSummary(sample, now); + return { + messages: built.map((item) => item.message), + sessions: built.map((item) => item.trace), + }; +} - return messages; +export function fingerprintSample(sample: LoCoMoSample): string { + return sha256Text(JSON.stringify(sample)); } -/** - * Format session summary data into raw memory messages (one per session). - */ -function createMessagesFromSummary(sample: LoCoMoSample, now: number): BenchRawMessage[] { - const messages: BenchRawMessage[] = []; +export function fingerprintQuestion( + sample: LoCoMoSample, + qa: QAPair, + questionIndex: number, + retrievalMode: RetrievalMode, +): string { + return sha256Text( + JSON.stringify({ + sample_id: sample.sample_id, + question_index: questionIndex, + question: qa.question, + answer: qa.answer, + category: qa.category, + evidence: qa.evidence, + retrieval_mode: retrievalMode, + top_k: RETRIEVAL_LIMIT, + }), + ); +} - for (const key of Object.keys(sample.session_summary).sort()) { - if (!key.endsWith("_summary")) { +function applyIngestBatchTraces(sessions: LoCoMoSessionTrace[], batches: IngestBatchTrace[]): void { + const byMessageId = new Map( + batches.flatMap((batch) => batch.message_ids.map((messageId) => [messageId, batch] as const)), + ); + for (const session of sessions) { + const batch = byMessageId.get(session.message_id); + if (!batch) { + session.ingest_status = "not_attempted"; continue; } - - const sessionNum = key.replace("_summary", ""); - const datetimeKey = `${sessionNum}_date_time`; - const sessionTimestamp = String(sample.conversation[datetimeKey] ?? ""); - const summaryContent = sample.session_summary[key]; - - const summaryParts: string[] = []; - summaryParts.push(`# Session Summary ${sessionNum}`); - if (sessionTimestamp) { - summaryParts.push(`# Timestamp: ${sessionTimestamp}`); - } - summaryParts.push(""); - - if (typeof summaryContent === "object" && summaryContent !== null) { - for (const [speaker, text] of Object.entries(summaryContent)) { - summaryParts.push(`${speaker}: ${text}`); - } - } else { - summaryParts.push(String(summaryContent)); - } - - messages.push( - toBenchMessage( - `${sample.sample_id}_summary_${sessionNum}`, - sessionTimestamp, - summaryParts.join("\n"), - { - sampleId: sample.sample_id, - sessionId: sessionNum, - contentType: "session_summary", - }, - now, - ), - ); + session.ingest_status = batch.status; + session.ingest_latency_ms = batch.latency_ms; + session.ingest_warnings = batch.warnings; + if (batch.error) session.error = batch.error; } - - return messages; } -/** - * Build the answer prompt from retrieved memory excerpts. - */ function buildAnswerPrompt(qa: QAPair, sample: LoCoMoSample, hits: MemorySearchHit[]): string { const speakerA = sample.conversation.speaker_a ?? "Speaker A"; const speakerB = sample.conversation.speaker_b ?? "Speaker B"; - const excerpts = hits .map( - (h, i) => - `--- Memory excerpt ${i + 1} (id=${h.id}, score=${h.similarity.toFixed(3)}) ---\n${h.content}`, + (hit, index) => + `--- Memory excerpt ${index + 1} (id=${hit.id}, score=${hit.similarity.toFixed(3)}) ---\n${hit.content}`, ) .join("\n\n"); - const categoryGuidance = qa.category === 5 - ? `- This is an ADVERSARIAL question. If the excerpts do not contain the relevant information, say you don't know — do not guess or hallucinate.` + ? "- This is an ADVERSARIAL question. If the excerpts do not contain the relevant information, say you don't know — do not guess or hallucinate." : qa.category === 2 ? "- This is a TEMPORAL question. Use the timestamps in the excerpts as the authoritative dates. Convert every relative time reference into a specific date, month, or year." - : qa.category === 3 + : qa.category === 1 ? "- This is a MULTI-HOP question. The answer requires combining information from multiple excerpts — cite each fact you use." - : qa.category === 4 + : qa.category === 3 ? "- This is an OPEN-DOMAIN question. Ground your answer in the excerpts; do not invent facts that are not there." : "- This is a SINGLE-HOP question. Pull the specific fact from the excerpts."; @@ -318,9 +404,17 @@ Question: ${qa.question} Answer based only on the retrieved memory excerpts above:`; } -/** - * Evaluator for the LoCoMo benchmark against the OpenContext memory store. - */ +export { checkOpencontextHealth, getOpencontextBaseUrl }; + +function locomoUserId(sample: LoCoMoSample): string { + return `locomo_${sample.sample_id}`; +} + +export function getLoCoMoCheckpointDir(): string { + const configured = process.env.LOCOMO_CHECKPOINT_DIR?.trim(); + return configured ? resolve(configured) : join(import.meta.dirname, "..", "checkpoints", "locomo"); +} + export class LoCoMoEvaluator { private retrievalMode: RetrievalMode; private baseUrl: string; @@ -328,189 +422,369 @@ export class LoCoMoEvaluator { private checkpointDir: string; private resume: boolean; private ingestedCount = 0; + private sessionTraces = new Map(); + private sampleFingerprints = new Map(); constructor( - retrievalMode: RetrievalMode | string = RetrievalMode.OBSERVATION, + retrievalMode: RetrievalMode | string = RetrievalMode.DIALOG, baseUrl?: string, quickLimit?: number, resume = true, ) { - // Convert string to enum if needed - if (typeof retrievalMode === "string") { - const modeMap: Record = { - dialog: RetrievalMode.DIALOG, - observation: RetrievalMode.OBSERVATION, - session_summary: RetrievalMode.SESSION_SUMMARY, - }; - this.retrievalMode = modeMap[retrievalMode] || RetrievalMode.OBSERVATION; - } else { - this.retrievalMode = retrievalMode; - } + this.retrievalMode = Object.values(RetrievalMode).includes(retrievalMode as RetrievalMode) + ? (retrievalMode as RetrievalMode) + : RetrievalMode.DIALOG; this.baseUrl = baseUrl ?? getOpencontextBaseUrl(); this.quickLimit = quickLimit; this.resume = resume; - this.checkpointDir = join(import.meta.dirname, "..", "checkpoints", "locomo"); + this.checkpointDir = getLoCoMoCheckpointDir(); } - /** - * Get checkpoint file path for a sample - */ private getCheckpointPath(sampleId: string): string { - return join(this.checkpointDir, `${sampleId}.json`); + return join(this.checkpointDir, `${sampleId}.${this.retrievalMode}.json`); } - /** - * Load checkpoint for a sample if it exists - */ private async loadCheckpoint(sampleId: string): Promise | null> { if (!this.resume) return null; try { - const path = this.getCheckpointPath(sampleId); - const data = await readFile(path, "utf-8"); - const parsed = JSON.parse(data); - // Return predictions keyed by question index - return parsed as Record; + return JSON.parse(await readFile(this.getCheckpointPath(sampleId), "utf-8")) as Record< + number, + Prediction + >; } catch { return null; } } - /** - * Save checkpoint for a sample after each question is evaluated - */ private async saveCheckpoint(sampleId: string, predictions: Record): Promise { try { await mkdir(this.checkpointDir, { recursive: true }); - const path = this.getCheckpointPath(sampleId); - await writeFile(path, JSON.stringify(predictions, null, 2), "utf-8"); + await writeFile(this.getCheckpointPath(sampleId), JSON.stringify(predictions, null, 2), "utf-8"); } catch (error) { - console.error(`Failed to save checkpoint: ${error}`); + process.stderr.write(`Failed to save checkpoint: ${error}\n`); } } - /** - * Ingest a LoCoMo sample into the OpenContext memory store. - * Returns the number of ingested memory messages. - */ - async loadSample(sample: LoCoMoSample): Promise { - const now = Date.now(); + private selectedQuestions(sample: LoCoMoSample): QAPair[] { + return this.quickLimit ? sample.qa_pairs.slice(0, this.quickLimit) : sample.qa_pairs; + } - // Build memory messages based on retrieval mode (one per session) - let messages: BenchRawMessage[]; + private checkpointMatches( + prediction: Prediction | undefined, + sample: LoCoMoSample, + qa: QAPair, + questionIndex: number, + ): boolean { + return ( + prediction?.trace_schema_version === LOCOMO_TRACE_SCHEMA_VERSION && + prediction.sample_sha256 === + (this.sampleFingerprints.get(sample.sample_id) ?? fingerprintSample(sample)) && + prediction.question_sha256 === fingerprintQuestion(sample, qa, questionIndex, this.retrievalMode) && + prediction.retrieval_mode === this.retrievalMode && + prediction.answerer_model === getAnswererModelIdentity() && + prediction.judge_model === getJudgeModelIdentity() + ); + } - if (this.retrievalMode === RetrievalMode.DIALOG) { - messages = createMessagesFromDialog(sample, now); - } else if (this.retrievalMode === RetrievalMode.OBSERVATION) { - messages = createMessagesFromObservation(sample, now); - } else if (this.retrievalMode === RetrievalMode.SESSION_SUMMARY) { - messages = createMessagesFromSummary(sample, now); - } else { - messages = []; - } + getSessionTraces(): LoCoMoSessionTrace[] { + return [...this.sessionTraces.values()]; + } - const inserted = await ingestMessages(messages, this.baseUrl, `locomo_${sample.sample_id}`); + async reuseCompletedSample(sample: LoCoMoSample): Promise { + if (!this.resume) return false; + this.sampleFingerprints.set(sample.sample_id, fingerprintSample(sample)); + const checkpoint = await this.loadCheckpoint(sample.sample_id); + const questions = this.selectedQuestions(sample); + if ( + !checkpoint || + questions.length === 0 || + !questions.every( + (qa, questionIndex) => + checkpoint[questionIndex]?.status === "completed" && + this.checkpointMatches(checkpoint[questionIndex], sample, qa, questionIndex), + ) + ) { + return false; + } + const { messages, sessions } = buildSampleMessages(sample, this.retrievalMode); + for (const session of sessions) { + session.ingest_status = "completed"; + session.ingest_warnings = [ + "Restored from context-matched completed checkpoints; original ingest latency was not persisted.", + ]; + this.sessionTraces.set(session.message_id, session); + } this.ingestedCount = messages.length; + return true; + } - console.log( - `[LoCoMo] Ingested ${inserted} memory messages for ${sample.sample_id} (mode: ${this.retrievalMode}) → ${this.baseUrl}`, + async loadSample(sample: LoCoMoSample): Promise { + const { messages, sessions } = buildSampleMessages(sample, this.retrievalMode); + if (messages.length === 0) + throw new Error(`No ${this.retrievalMode} sessions found in ${sample.sample_id}`); + this.sampleFingerprints.set(sample.sample_id, fingerprintSample(sample)); + for (const session of sessions) this.sessionTraces.set(session.message_id, session); + const startedAt = performance.now(); + try { + const result = await ingestMessages(messages, this.baseUrl, locomoUserId(sample)); + applyIngestBatchTraces(sessions, result.batches); + this.ingestedCount = messages.length; + process.stdout.write( + `[LoCoMo] Ingested ${result.inserted} raw sessions for ${sample.sample_id} (mode: ${this.retrievalMode}) → ${this.baseUrl}\n`, + ); + return messages.length; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (error instanceof IngestMessagesError) applyIngestBatchTraces(sessions, error.result.batches); + else { + const latencyMs = Math.round(performance.now() - startedAt); + for (const session of sessions) { + session.ingest_status = "execution_error"; + session.ingest_latency_ms = latencyMs; + session.error = message; + } + } + throw error; + } + } + + createExecutionErrorPrediction( + sample: LoCoMoSample, + qa: QAPair, + questionIndex: number, + error: unknown, + stage: "ingest" | "retrieval" | "answerer" | "judge" | "provider", + attempt = 1, + trace: { + retrieval?: LoCoMoRetrievalTrace | null; + answerer?: LoCoMoAnswerTrace | null; + judge?: LoCoMoJudgeTrace | null; + } = {}, + tokenUsage: TokenUsage = unavailableTokenUsage(), + ): Prediction { + const errorMessage = error instanceof Error ? error.message : String(error); + return { + trace_schema_version: LOCOMO_TRACE_SCHEMA_VERSION, + sample_sha256: this.sampleFingerprints.get(sample.sample_id) ?? fingerprintSample(sample), + question_sha256: fingerprintQuestion(sample, qa, questionIndex, this.retrievalMode), + status: "execution_error", + attempt, + answerer_model: getAnswererModelIdentity(), + judge_model: getJudgeModelIdentity(), + execution_error: { stage, message: errorMessage }, + token_usage: tokenUsage, + sample_id: sample.sample_id, + question_index: questionIndex, + retrieval_mode: this.retrievalMode, + question: qa.question, + answer: qa.answer, + response: `Error: ${errorMessage}`, + prediction: `Error: ${errorMessage}`, + ground_truth: qa.answer, + category: String(qa.category), + llm_score: 0, + correct: false, + f1_score: 0, + bleu_score: 0, + bleu1: 0, + bleu2: 0, + bleu3: 0, + bleu4: 0, + evidence: qa.evidence, + trace: { + retrieval: trace.retrieval ?? null, + answerer: trace.answerer ?? null, + judge: trace.judge ?? null, + }, + failure_stage: deriveFailureStage({ + qa, + retrieval: trace.retrieval ?? null, + correct: false, + executionStage: stage, + }), + }; + } + + createIngestErrorResult(sample: LoCoMoSample, error: unknown): EvaluationResult { + const predictions = this.selectedQuestions(sample).map((qa, questionIndex) => + this.createExecutionErrorPrediction(sample, qa, questionIndex, error, "ingest"), ); - return messages.length; + return { + sample_id: sample.sample_id, + retrieval_mode: this.retrievalMode, + total_questions: predictions.length, + correct_answers: 0, + accuracy: 0, + token_usage: sumTokenUsage(predictions.map((prediction) => prediction.token_usage)), + predictions, + error: error instanceof Error ? error.message : String(error), + }; } - /** - * Evaluate question answering on a LoCoMo sample. - */ async evaluateQA(sample: LoCoMoSample): Promise { - if (this.ingestedCount === 0) { - return { - sample_id: sample.sample_id, - retrieval_mode: this.retrievalMode, - total_questions: sample.qa_pairs.length, - correct_answers: 0, - accuracy: 0, - token_usage: unavailableTokenUsage(), - predictions: [], - error: "No records in storage", - }; - } - - const checkpoint = (await this.loadCheckpoint(sample.sample_id)) || {}; - const answererModel = getAnswererModelIdentity(); - const judgeModel = JUDGE_MODEL; - const reusableIndices = new Set(); - let executionErrorsToRetry = 0; - let incompatibleCheckpoints = 0; - - for (const [idx, pred] of Object.entries(checkpoint)) { - const i = Number(idx); - if (isReusableCheckpoint(pred, answererModel, judgeModel)) { - reusableIndices.add(i); - } else if (pred?.status === "execution_error") { - executionErrorsToRetry++; - } else { - incompatibleCheckpoints++; - } - } + if (this.ingestedCount === 0) return this.createIngestErrorResult(sample, "No records in storage"); + const checkpoint = (await this.loadCheckpoint(sample.sample_id)) ?? {}; + const questions = this.selectedQuestions(sample); const predictions: Prediction[] = []; let correct = 0; - + let reused = 0; + let retrying = 0; + let incompatible = 0; + + for (let questionIndex = 0; questionIndex < questions.length; questionIndex++) { + const qa = questions[questionIndex]; + const existing = checkpoint[questionIndex]; + const matches = this.checkpointMatches(existing, sample, qa, questionIndex); + if (existing?.status === "completed" && matches) reused++; + else if (existing?.status === "execution_error" && matches) retrying++; + else if (existing) incompatible++; + } if (Object.keys(checkpoint).length > 0) { - console.log( - `[LoCoMo] Resume: ${reusableIndices.size} completed result(s) reused, ${executionErrorsToRetry} execution error(s) retrying, ${incompatibleCheckpoints} legacy/model-mismatched result(s) re-running`, + process.stdout.write( + `[LoCoMo] Resume: ${reused} completed result(s) reused, ${retrying} execution error(s) retrying, ${incompatible} trace/data/model-mismatched result(s) re-running\n`, ); } - // Limit questions if quick mode is enabled - const questionsToEvaluate = this.quickLimit ? sample.qa_pairs.slice(0, this.quickLimit) : sample.qa_pairs; - - console.log( - `[LoCoMo] Evaluating ${questionsToEvaluate.length} questions (quick limit: ${this.quickLimit || "none"})`, - ); - - for (let i = 0; i < questionsToEvaluate.length; i++) { - const qa = questionsToEvaluate[i]; - - const existing = checkpoint[i]; - if (reusableIndices.has(i)) { + for (let questionIndex = 0; questionIndex < questions.length; questionIndex++) { + const qa = questions[questionIndex]; + const existing = checkpoint[questionIndex]; + const checkpointMatches = this.checkpointMatches(existing, sample, qa, questionIndex); + if (existing?.status === "completed" && checkpointMatches) { predictions.push(existing); if (existing.correct) correct++; continue; } - const attempt = (existing?.attempt ?? 0) + 1; + const attempt = + existing?.status === "execution_error" && checkpointMatches ? (existing.attempt ?? 0) + 1 : 1; let answerUsage = unavailableTokenUsage(); let judgeUsage = unavailableTokenUsage(); + let retrievalTrace: LoCoMoRetrievalTrace | null = null; + let answerTrace: LoCoMoAnswerTrace | null = null; + let judgeTrace: LoCoMoJudgeTrace | null = null; + let executionStage: "retrieval" | "answerer" | "judge" = "retrieval"; try { - // Retrieve relevant memories, then answer using only those excerpts - const answerResult = await this.answerQuestion(qa, sample); - const response = answerResult.text; - answerUsage = answerResult.token_usage; - - // Evaluate answer correctness using LLM judge - const judgeResult = await evaluateLLMJudge(qa.question, qa.answer, response); - judgeUsage = judgeResult.token_usage; - const isCorrect = judgeResult.score === 1; - console.log( - `[Q${i + 1}] ${isCorrect ? "✓" : "✗"} Q: "${qa.question.substring(0, 60)}..." GT: "${qa.answer}"`, + const sampleSessions = this.getSessionTraces().filter( + (session) => + session.sample_id === sample.sample_id && session.retrieval_mode === this.retrievalMode, ); - if (!isCorrect) { - console.log(` Agent response: "${response.substring(0, 300)}..."`); + const availableEvidenceIds = unique(sampleSessions.flatMap((session) => session.evidence_ids)); + const searchStartedAt = performance.now(); + let searchResponse: MemorySearchResponse; + try { + searchResponse = await searchMemory( + qa.question, + RETRIEVAL_LIMIT, + this.baseUrl, + locomoUserId(sample), + { includeRetrievalDiagnostics: true }, + ); + } catch (error) { + retrievalTrace = buildRetrievalErrorTrace({ + qa, + availableEvidenceIds, + userId: locomoUserId(sample), + topK: RETRIEVAL_LIMIT, + latencyMs: Math.round(performance.now() - searchStartedAt), + error: error instanceof Error ? error.message : String(error), + }); + throw error; + } + retrievalTrace = buildRetrievalTrace({ + qa, + response: searchResponse, + sessionsByMessageId: new Map( + sampleSessions.map((session) => [session.message_id, session] as const), + ), + availableEvidenceIds, + userId: locomoUserId(sample), + topK: RETRIEVAL_LIMIT, + latencyMs: Math.round(performance.now() - searchStartedAt), + }); + + const hits = searchResponse.results; + const prompt = buildAnswerPrompt(qa, sample, hits); + executionStage = "answerer"; + const answerStartedAt = performance.now(); + let response: string; + let answerAttempt = 1; + try { + const answerResult = await generateAnswer(prompt); + if (!answerResult.text.trim()) throw new Error("Answerer returned an empty response"); + answerUsage = answerResult.token_usage; + answerAttempt = answerResult.attempt; + response = answerResult.text; + answerTrace = { + model: getAnswererModelIdentity(), + status: "completed", + attempt: answerAttempt, + latency_ms: Math.round(performance.now() - answerStartedAt), + prompt_version: "locomo-answer-v1", + prompt_sha256: sha256Text(prompt), + prompt_characters: prompt.length, + system_prompt: null, + prompt, + token_usage: answerUsage, + included_hit_ids: hits.map((hit) => hit.id), + included_context_characters: hits.reduce((sum, hit) => sum + hit.content.length, 0), + }; + } catch (error) { + answerAttempt = error instanceof AnswererGenerationError ? error.attempts : answerAttempt; + answerTrace = { + model: getAnswererModelIdentity(), + status: "execution_error", + attempt: answerAttempt, + latency_ms: Math.round(performance.now() - answerStartedAt), + prompt_version: "locomo-answer-v1", + prompt_sha256: sha256Text(prompt), + prompt_characters: prompt.length, + system_prompt: null, + prompt, + token_usage: answerUsage, + included_hit_ids: hits.map((hit) => hit.id), + included_context_characters: hits.reduce((sum, hit) => sum + hit.content.length, 0), + error: error instanceof Error ? error.message : String(error), + }; + throw error; } - if (isCorrect) { - correct++; + executionStage = "judge"; + const judgeResult = await evaluateLLMJudge(qa.question, qa.answer, response); + judgeUsage = judgeResult.token_usage; + judgeTrace = { + model: getJudgeModelIdentity(), + status: judgeResult.status, + attempt: judgeResult.attempt, + latency_ms: judgeResult.latency_ms, + prompt_version: judgeResult.prompt_version, + prompt_sha256: judgeResult.prompt_sha256, + prompt_characters: judgeResult.prompt_characters, + system_prompt: judgeResult.system_prompt, + prompt: judgeResult.prompt, + token_usage: judgeResult.token_usage, + raw_response: judgeResult.raw_response, + parse_status: judgeResult.parse_status, + ...(judgeResult.error ? { error: judgeResult.error } : {}), + }; + if (judgeResult.status === "execution_error") { + throw new Error(judgeResult.error ?? "Judge failed without returning a result"); } - // Calculate additional metrics + const isCorrect = judgeResult.score === 1; const metrics = calculateMetrics(response, qa.answer); - - const pred: Prediction = { + const prediction: Prediction = { + trace_schema_version: LOCOMO_TRACE_SCHEMA_VERSION, + sample_sha256: this.sampleFingerprints.get(sample.sample_id) ?? fingerprintSample(sample), + question_sha256: fingerprintQuestion(sample, qa, questionIndex, this.retrievalMode), status: "completed", attempt, - answerer_model: answererModel, - judge_model: judgeModel, + answerer_model: getAnswererModelIdentity(), + judge_model: getJudgeModelIdentity(), token_usage: sumTokenUsage([answerUsage, judgeUsage]), + sample_id: sample.sample_id, + question_index: questionIndex, + retrieval_mode: this.retrievalMode, question: qa.question, answer: qa.answer, response, @@ -526,75 +800,38 @@ export class LoCoMoEvaluator { bleu3: metrics.bleu3, bleu4: metrics.bleu4, evidence: qa.evidence, + trace: { retrieval: retrievalTrace, answerer: answerTrace, judge: judgeTrace }, + failure_stage: deriveFailureStage({ qa, retrieval: retrievalTrace, correct: isCorrect }), }; - - predictions.push(pred); - - // Save checkpoint after each question - checkpoint[i] = pred; + predictions.push(prediction); + checkpoint[questionIndex] = prediction; await this.saveCheckpoint(sample.sample_id, checkpoint); + if (isCorrect) correct++; } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - const errorCause = error instanceof Error && error.cause ? String(error.cause) : ""; - console.error( - `Error evaluating question: ${errorMessage}${errorCause ? ` (cause: ${errorCause})` : ""}`, - ); - - const pred: Prediction = { - status: "execution_error", + const prediction = this.createExecutionErrorPrediction( + sample, + qa, + questionIndex, + error, + executionStage, attempt, - answerer_model: answererModel, - judge_model: judgeModel, - error: errorMessage, - token_usage: sumTokenUsage([answerUsage, judgeUsage]), - question: qa.question, - answer: qa.answer, - response: `Error: ${errorMessage}`, - prediction: `Error: ${errorMessage}`, - ground_truth: qa.answer, - category: String(qa.category), - llm_score: 0, - correct: false, - f1_score: 0.0, - bleu_score: 0.0, - bleu1: 0.0, - bleu2: 0.0, - bleu3: 0.0, - bleu4: 0.0, - evidence: qa.evidence, - }; - - predictions.push(pred); - - // Save checkpoint after each question - checkpoint[i] = pred; + { retrieval: retrievalTrace, answerer: answerTrace, judge: judgeTrace }, + sumTokenUsage([answerUsage, judgeUsage]), + ); + predictions.push(prediction); + checkpoint[questionIndex] = prediction; await this.saveCheckpoint(sample.sample_id, checkpoint); } } - const total = sample.qa_pairs.length; - return { sample_id: sample.sample_id, retrieval_mode: this.retrievalMode, - total_questions: total, + total_questions: questions.length, correct_answers: correct, - accuracy: total > 0 ? correct / total : 0, + accuracy: questions.length > 0 ? correct / questions.length : 0, token_usage: sumTokenUsage(predictions.map((prediction) => prediction.token_usage)), predictions, }; } - - /** - * Answer a question using retrieved memory excerpts. - */ - private async answerQuestion(qa: QAPair, sample: LoCoMoSample): Promise { - const hits = await searchMemory(qa.question, RETRIEVAL_LIMIT, this.baseUrl, `locomo_${sample.sample_id}`); - const prompt = buildAnswerPrompt(qa, sample, hits); - const response = await generateAnswer(prompt); - if (!response.text.trim()) { - throw new Error("Answerer returned an empty response"); - } - return response; - } } diff --git a/benchmark/locomo/src/index.ts b/benchmark/locomo/src/index.ts index ae2899de..6221338a 100644 --- a/benchmark/locomo/src/index.ts +++ b/benchmark/locomo/src/index.ts @@ -1,30 +1,29 @@ -/** - * LoCoMo Benchmark CLI - * - * Run via: pnpm benchmark -- --dataset dataset/locomo_v2.json --mode observation --quick - */ +/** LoCoMo benchmark CLI. */ import "dotenv/config"; import { mkdir, writeFile } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; -import { - getManifestPath, - runPreflight, - sumTokenUsage, - unavailableTokenUsage, - writeRunManifest, -} from "../../run-support"; + +import { getManifestPath, runPreflight, sumTokenUsage, writeRunManifest } from "../../run-support"; import { loadLoCoMoDatasetFromJson } from "./dataset"; -import { LoCoMoEvaluator, RETRIEVAL_LIMIT } from "./evaluator"; -import { JUDGE_MODEL, calculateCategoryMetrics } from "./metrics"; +import { calculateDiagnosticSummary } from "./diagnostics"; import { + LoCoMoEvaluator, + RETRIEVAL_LIMIT, checkOpencontextHealth, - getAnswererModelIdentity, + getLoCoMoCheckpointDir, getOpencontextBaseUrl, -} from "./opencontext-client"; -import { CATEGORY_NAMES } from "./scorer"; -import { RetrievalMode } from "./types"; -import type { EvaluationResult, Prediction } from "./types"; +} from "./evaluator"; +import { calculateCategoryMetrics, getJudgeModelIdentity } from "./metrics"; +import { getAnswererModelIdentity } from "./opencontext-client"; +import { getCategoryName } from "./scorer"; +import { + LOCOMO_TRACE_SCHEMA_VERSION, + type EvaluationResult, + type LoCoMoSessionTrace, + type Prediction, + RetrievalMode, +} from "./types"; interface CliArgs { dataset: string; @@ -34,67 +33,54 @@ interface CliArgs { output?: string; port?: number; resume: boolean; + preflightOnly: boolean; } function parseCliArgs(): CliArgs { - // Simple manual argument parsing for flexibility const args = process.argv.slice(2); - const values: Record = { - mode: "observation", + const values: Record = { + mode: RetrievalMode.DIALOG, quick: false, resume: true, + preflightOnly: false, }; - - for (let i = 0; i < args.length; i++) { - const arg = args[i]; - if (arg === "--dataset" || arg === "-d") { - values.dataset = args[++i]; - } else if (arg === "--mode" || arg === "-m") { - values.mode = args[++i]; - } else if (arg === "--samples" || arg === "-s") { - values.samples = args[++i]; - } else if (arg === "--quick" || arg === "-q") { - values.quick = true; - } else if (arg === "--output" || arg === "-o") { - values.output = args[++i]; - } else if (arg === "--port" || arg === "-p") { - values.port = Number.parseInt(args[++i], 10); - } else if (arg === "--resume") { - values.resume = true; - } else if (arg === "--no-resume") { - values.resume = false; - } else if (arg === "--help" || arg === "-h") { + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg === "--dataset" || arg === "-d") values.dataset = args[++index]; + else if (arg === "--mode" || arg === "-m") values.mode = args[++index]; + else if (arg === "--samples" || arg === "-s") values.samples = args[++index]; + else if (arg === "--quick" || arg === "-q") values.quick = true; + else if (arg === "--output" || arg === "-o") values.output = args[++index]; + else if (arg === "--port" || arg === "-p") values.port = Number.parseInt(args[++index], 10); + else if (arg === "--resume") values.resume = true; + else if (arg === "--no-resume") values.resume = false; + else if (arg === "--preflight-only") values.preflightOnly = true; + else if (arg === "--help" || arg === "-h") { printHelp(); process.exit(0); } } - if (!values.dataset) { - console.error("Error: --dataset is required"); printHelp(); process.exit(1); } - - const mode = values.mode as RetrievalMode; - - let samples: string[] | undefined; - if (values.samples) { - samples = (values.samples as string).split(",").map((s: string) => s.trim()); - } - return { dataset: values.dataset as string, - mode, - samples, - quick: values.quick as boolean, + mode: values.mode as RetrievalMode, + samples: + typeof values.samples === "string" + ? values.samples.split(",").map((sample) => sample.trim()) + : undefined, + quick: values.quick === true, output: values.output as string | undefined, port: values.port as number | undefined, resume: values.resume !== false, + preflightOnly: values.preflightOnly === true, }; } function printHelp(): void { - console.log(`LoCoMo Benchmark CLI + process.stdout.write(`LoCoMo Benchmark CLI Usage: pnpm benchmark -- --dataset [options] @@ -104,81 +90,51 @@ Required: Filter: -m, --mode Retrieval mode: dialog, observation, - session_summary (default: observation) + session_summary (default: dialog) -s, --samples Filter to specific sample IDs (csv) - -q, --quick First 5 questions per sample (smoke test) + -q, --quick First 5 questions per sample Mode: - --resume / --no-resume Reuse cached judge results (default: resume) + --resume / --no-resume Reuse context-matched checkpoints (default: resume) + --preflight-only Validate readiness without ingest/model calls API: - -p, --port OpenContext memory daemon port (default: 7421, + -p, --port OpenContext daemon port (default: 7421, env: OPENCONTEXT_PORT / OPENCONTEXT_URL) Output: -o, --output Write results JSON to this path - -Examples: - # Smoke test (locomo_v2.json ships with the repo) - pnpm benchmark -- --dataset dataset/locomo_v2.json --quick - - # Full run, dialog mode - pnpm benchmark -- --dataset dataset/locomo_v2.json --mode dialog \\ - --output results/locomo_$(date +%Y%m%d_%H%M%S).json `); } -async function printEvaluationSummary(resultsByCategory: Record): Promise { - console.log("=".repeat(80)); - console.log("LoCoMo Evaluation Results Summary"); - console.log("=".repeat(80)); +function diagnosticArtifactPath(outputPath: string, suffix: "trace" | "sessions"): string { + const resolved = resolve(outputPath); + return `${resolved.replace(/\.json$/i, "")}.${suffix}.jsonl`; +} - // Calculate overall metrics - const allResults: Prediction[] = []; - for (const [category, results] of Object.entries(resultsByCategory)) { - // Skip category 5 (adversarial questions) - if (category === "5") { - continue; - } - allResults.push(...results); - } +async function writeJsonl(path: string, rows: unknown[]): Promise { + await mkdir(dirname(path), { recursive: true }); + const content = rows.map((row) => JSON.stringify(row)).join("\n"); + await writeFile(path, content.length > 0 ? `${content}\n` : "", "utf-8"); +} - const overallMetrics = calculateCategoryMetrics(allResults); +function withoutDiagnosticTrace(prediction: Prediction): Omit { + const { trace: _trace, ...result } = prediction; + return result; +} - console.log("\n📊 Overall Results:"); - console.log(` Total Questions: ${overallMetrics.count}`); - console.log( - ` LLM Judge Accuracy: ${overallMetrics.llm_judge_accuracy.toFixed(4)} (${overallMetrics.llm_judge_correct}/${overallMetrics.count})`, +function printEvaluationSummary(resultsByCategory: Record): void { + const predictions = Object.values(resultsByCategory).flat(); + const overall = calculateCategoryMetrics(predictions); + process.stdout.write( + `LoCoMo summary: questions=${overall.count}, correct=${overall.llm_judge_correct}, accuracy=${overall.llm_judge_accuracy.toFixed(4)}\n`, ); - console.log(` F1 Score (Mean): ${overallMetrics.f1_mean.toFixed(4)}`); - console.log(` BLEU-1 (Mean): ${overallMetrics.bleu1_mean.toFixed(4)}`); - console.log(` BLEU-4 (Mean): ${overallMetrics.bleu4_mean.toFixed(4)}`); - - console.log(`\n${"=".repeat(80)}`); - console.log("Results by Category"); - console.log("=".repeat(80)); - for (const category of Object.keys(resultsByCategory).sort()) { - // Skip category 5 - if (category === "5") { - continue; - } - - const results = resultsByCategory[category]; - const metrics = calculateCategoryMetrics(results); - - const categoryName = CATEGORY_NAMES[category] || `category_${category}`; - console.log(`\nCategory ${category} (${categoryName}):`); - console.log(` Count: ${metrics.count}`); - console.log( - ` LLM Judge Accuracy: ${metrics.llm_judge_accuracy.toFixed(4)} (${metrics.llm_judge_correct}/${metrics.count})`, + const metrics = calculateCategoryMetrics(resultsByCategory[category]); + process.stdout.write( + ` ${category} (${getCategoryName(Number(category))}): ${metrics.llm_judge_correct}/${metrics.count} (${metrics.llm_judge_accuracy.toFixed(4)})\n`, ); - console.log(` F1 Score: ${metrics.f1_mean.toFixed(4)}`); - console.log(` BLEU-1: ${metrics.bleu1_mean.toFixed(4)}`); - console.log(` BLEU-4: ${metrics.bleu4_mean.toFixed(4)}`); } - - console.log(`\n${"=".repeat(80)}`); } async function main() { @@ -187,23 +143,28 @@ async function main() { const baseUrl = args.port ? `http://127.0.0.1:${args.port}` : getOpencontextBaseUrl(); const benchmarkDir = join(import.meta.dirname, ".."); const manifestPath = getManifestPath(args.output, benchmarkDir, startedAt); + const tracePath = args.output ? diagnosticArtifactPath(args.output, "trace") : undefined; + const sessionsPath = args.output ? diagnosticArtifactPath(args.output, "sessions") : undefined; let filteredSamples: Awaited> = []; const parameterErrors: string[] = []; if (!Object.values(RetrievalMode).includes(args.mode)) { - parameterErrors.push( - `--mode must be one of: ${Object.values(RetrievalMode).join(", ")} (received: ${args.mode})`, - ); + parameterErrors.push(`--mode must be one of: ${Object.values(RetrievalMode).join(", ")}`); } if (args.port !== undefined && (!Number.isInteger(args.port) || args.port < 1 || args.port > 65_535)) { parameterErrors.push("--port must be an integer between 1 and 65535"); } + if (!process.env.OPENROUTER_JUDGE_MODEL?.trim()) { + parameterErrors.push("judge model missing: set OPENROUTER_JUDGE_MODEL"); + } await runPreflight({ datasetPath: args.dataset, writablePaths: [ manifestPath, - join(benchmarkDir, "checkpoints", "locomo", ".preflight"), + join(getLoCoMoCheckpointDir(), ".preflight"), ...(args.output ? [args.output] : []), + ...(tracePath ? [tracePath] : []), + ...(sessionsPath ? [sessionsPath] : []), ], parameterErrors, validateDataset: async () => { @@ -212,126 +173,164 @@ async function main() { args.samples && args.samples.length > 0 ? samples.filter((sample) => args.samples?.includes(sample.sample_id)) : samples; - if (filteredSamples.length === 0) { - throw new Error("no samples remain after applying --samples"); - } + if (filteredSamples.length === 0) throw new Error("no samples remain after applying --samples"); }, checkDaemon: () => checkOpencontextHealth(baseUrl), }); - console.log(`🔌 OpenContext memory daemon: ${baseUrl}`); - console.log(`\n📁 Loaded dataset from: ${args.dataset}`); - if (args.samples && args.samples.length > 0) { - console.log(`🔍 Filtered to ${filteredSamples.length} samples by ID`); - } - // Apply quick mode (first 5 questions only) - if (args.quick) { - console.log("⚡ Quick mode: limiting to first 5 questions per sample"); + if (args.preflightOnly) { + const questionCount = filteredSamples.reduce( + (sum, sample) => sum + (args.quick ? Math.min(5, sample.qa_pairs.length) : sample.qa_pairs.length), + 0, + ); + process.stdout.write( + `LoCoMo preflight passed: samples=${filteredSamples.length}, questions=${questionCount}, mode=${args.mode}, daemon=${baseUrl}, answerer=${getAnswererModelIdentity()}, judge=${getJudgeModelIdentity()}, top_k=${RETRIEVAL_LIMIT}\n`, + ); + return; } - console.log(`📊 Loaded ${filteredSamples.length} LoCoMo samples for evaluation`); - console.log(`🔧 Retrieval mode: ${args.mode}\n`); - - // Run evaluation const resultsBySample: EvaluationResult[] = []; - const allPredictionsByCategory: Record = {}; - + const resultsByCategory: Record = {}; + const sessionTraces: LoCoMoSessionTrace[] = []; for (const sample of filteredSamples) { const evaluator = new LoCoMoEvaluator(args.mode, baseUrl, args.quick ? 5 : undefined, args.resume); - + let result: EvaluationResult; try { - // Ingest sample into the memory store - await evaluator.loadSample(sample); - - // Evaluate QA - const result = await evaluator.evaluateQA(sample); - resultsBySample.push(result); - - // Organize predictions by category - for (const pred of result.predictions) { - const category = pred.category; - if (!allPredictionsByCategory[category]) { - allPredictionsByCategory[category] = []; - } - allPredictionsByCategory[category].push(pred); - } - - console.log( - `Sample ${sample.sample_id}: ${result.correct_answers}/${result.total_questions} correct (${(result.accuracy * 100).toFixed(2)}%)`, - ); + const reused = await evaluator.reuseCompletedSample(sample); + if (!reused) await evaluator.loadSample(sample); + result = await evaluator.evaluateQA(sample); } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - console.error(`Error evaluating sample ${sample.sample_id}: ${errorMessage}`); - - resultsBySample.push({ - sample_id: sample.sample_id, - retrieval_mode: args.mode, - total_questions: sample.qa_pairs.length, - correct_answers: 0, - accuracy: 0, - token_usage: unavailableTokenUsage(), - predictions: [], - error: errorMessage, - }); + result = evaluator.createIngestErrorResult(sample, error); + } + resultsBySample.push(result); + sessionTraces.push(...evaluator.getSessionTraces()); + for (const prediction of result.predictions) { + if (!resultsByCategory[prediction.category]) resultsByCategory[prediction.category] = []; + resultsByCategory[prediction.category].push(prediction); } } - // Aggregate results - const totalQuestions = resultsBySample.reduce((sum, r) => sum + (r.total_questions || 0), 0); - const totalCorrect = resultsBySample.reduce((sum, r) => sum + (r.correct_answers || 0), 0); - const overallAccuracy = totalQuestions > 0 ? totalCorrect / totalQuestions : 0; - - const runTokenUsage = sumTokenUsage(resultsBySample.map((result) => result.token_usage)); - - // Print summary - await printEvaluationSummary(allPredictionsByCategory); - - // Prepare output + printEvaluationSummary(resultsByCategory); + const predictions = resultsBySample.flatMap((result) => result.predictions); + const completedPredictions = predictions.filter((prediction) => prediction.status === "completed"); + const totalQuestions = predictions.length; + const totalCorrect = predictions.filter((prediction) => prediction.correct).length; + const runTokenUsage = sumTokenUsage(predictions.map((prediction) => prediction.token_usage)); + const diagnosticSummary = calculateDiagnosticSummary(predictions); const finishedAt = new Date().toISOString(); const runManifest = await writeRunManifest(manifestPath, { benchmark: "locomo", datasetPath: args.dataset, answerer_model: getAnswererModelIdentity(), - judge_model: JUDGE_MODEL, - retrieval: { strategy: args.mode, top_k: RETRIEVAL_LIMIT }, + judge_model: getJudgeModelIdentity(), + retrieval: { strategy: "daemon-default", mode: args.mode, top_k: RETRIEVAL_LIMIT, diagnostics: true }, resume: args.resume, started_at: startedAt, finished_at: finishedAt, token_usage: runTokenUsage, - parameters: { samples: args.samples ?? null, quick: args.quick ?? false }, + parameters: { + samples: args.samples ?? null, + quick: args.quick ?? false, + trace_schema_version: LOCOMO_TRACE_SCHEMA_VERSION, + trace_artifact: tracePath ?? null, + sessions_artifact: sessionsPath ?? null, + }, }); + const output = { retrieval_mode: args.mode, num_samples: resultsBySample.length, total_questions: totalQuestions, total_correct: totalCorrect, - overall_accuracy: overallAccuracy, + overall_accuracy: totalQuestions > 0 ? totalCorrect / totalQuestions : 0, token_usage: runTokenUsage, - total_tokens: runTokenUsage.total_tokens, + summary: { + all_records: calculateCategoryMetrics(predictions), + completed_only: calculateCategoryMetrics(completedPredictions), + execution_error_rate: + predictions.length === 0 + ? 0 + : (predictions.length - completedPredictions.length) / predictions.length, + }, + diagnostics: diagnosticSummary, + diagnostic_artifacts: { + trace_schema_version: LOCOMO_TRACE_SCHEMA_VERSION, + trace: tracePath ?? null, + sessions: sessionsPath ?? null, + }, run_manifest: runManifest, - results_by_sample: resultsBySample.map((r) => ({ - sample_id: r.sample_id, - accuracy: r.accuracy, - correct: r.correct_answers, - total: r.total_questions, - token_usage: r.token_usage, - error: r.error, + results_by_sample: resultsBySample.map((result) => ({ + sample_id: result.sample_id, + accuracy: result.accuracy, + correct: result.correct_answers, + total: result.total_questions, + token_usage: result.token_usage, + error: result.error, })), - results_by_category: allPredictionsByCategory, + results_by_category: Object.fromEntries( + Object.entries(resultsByCategory).map(([category, categoryPredictions]) => { + const metrics = calculateCategoryMetrics(categoryPredictions); + return [ + category, + { + name: getCategoryName(Number(category)), + count: metrics.count, + accuracy: metrics.llm_judge_accuracy, + f1_mean: metrics.f1_mean, + bleu1_mean: metrics.bleu1_mean, + bleu4_mean: metrics.bleu4_mean, + completed_only: calculateCategoryMetrics( + categoryPredictions.filter((prediction) => prediction.status === "completed"), + ), + }, + ]; + }), + ), + predictions, }; - // Save output if requested if (args.output) { await mkdir(dirname(resolve(args.output)), { recursive: true }); - await writeFile(args.output, JSON.stringify(output, null, 2), "utf-8"); - console.log(`\n💾 Results saved to: ${args.output}`); + await writeFile( + args.output, + JSON.stringify({ ...output, predictions: output.predictions.map(withoutDiagnosticTrace) }, null, 2), + "utf-8", + ); + await writeJsonl( + tracePath as string, + predictions.map((prediction) => ({ + schema_version: prediction.trace_schema_version, + sample_id: prediction.sample_id, + question_index: prediction.question_index, + sample_sha256: prediction.sample_sha256, + question_sha256: prediction.question_sha256, + retrieval_mode: prediction.retrieval_mode, + question: prediction.question, + ground_truth: prediction.ground_truth, + evidence: prediction.evidence, + status: prediction.status, + attempt: prediction.attempt, + execution_error: prediction.execution_error ?? null, + failure_stage: prediction.failure_stage, + answerer_model: prediction.answerer_model, + judge_model: prediction.judge_model, + response: prediction.response, + llm_score: prediction.llm_score, + correct: prediction.correct, + token_usage: prediction.token_usage, + trace: prediction.trace, + })), + ); + await writeJsonl(sessionsPath as string, sessionTraces); + process.stdout.write(`Results saved to: ${args.output}\n`); + process.stdout.write(`Question traces saved to: ${tracePath}\n`); + process.stdout.write(`Session ingest traces saved to: ${sessionsPath}\n`); } - console.log(`🧾 Run manifest saved to: ${manifestPath}`); - + process.stdout.write(`Run manifest saved to: ${manifestPath}\n`); return output; } main().catch((error) => { - console.error(error instanceof Error ? error.message : String(error)); + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); process.exitCode = 1; }); diff --git a/benchmark/locomo/src/metrics.ts b/benchmark/locomo/src/metrics.ts index 17100b4a..caab5a67 100644 --- a/benchmark/locomo/src/metrics.ts +++ b/benchmark/locomo/src/metrics.ts @@ -7,7 +7,7 @@ import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; import { generateText } from "ai"; -import { tokenUsage, type TokenUsage } from "../../run-support"; +import { tokenUsage, type TokenUsage, unavailableTokenUsage } from "../../run-support"; const openrouter = createOpenAICompatible({ baseURL: "https://openrouter.ai/api/v1", @@ -15,6 +15,18 @@ const openrouter = createOpenAICompatible({ name: "openrouter", }); import { LLM_JUDGE_PROMPT } from "./prompts"; +import { sha256Text } from "./diagnostics"; + +const configuredModelRequestTimeoutMs = Number.parseInt( + process.env.LOCOMO_MODEL_REQUEST_TIMEOUT_MS ?? "90000", + 10, +); +const MODEL_REQUEST_TIMEOUT_MS = + Number.isInteger(configuredModelRequestTimeoutMs) && configuredModelRequestTimeoutMs >= 10_000 + ? configuredModelRequestTimeoutMs + : 90_000; +const JUDGE_SYSTEM_PROMPT = + "You are an impartial judge evaluating answers to questions. Always respond with valid JSON."; /** * Calculate F1 score between prediction and ground truth. @@ -143,11 +155,30 @@ interface LLMJudgeResult { reasoning?: string; } -export const JUDGE_MODEL = "qwen/qwen3.7-max"; +export function getJudgeModel(): string { + const model = process.env.OPENROUTER_JUDGE_MODEL?.trim(); + if (!model) throw new Error("Judge model missing: set OPENROUTER_JUDGE_MODEL"); + return model; +} + +export function getJudgeModelIdentity(): string { + return `openrouter:${getJudgeModel()}`; +} export interface LLMJudgeEvaluation { score: number; token_usage: TokenUsage; + status: "completed" | "execution_error"; + attempt: number; + latency_ms: number; + prompt_version: string; + prompt_sha256: string; + prompt_characters: number; + system_prompt: string; + prompt: string; + raw_response: string | null; + parse_status: "parsed" | "failed"; + error?: string; } export function parseLLMJudgeResponse(text: string): number { @@ -164,9 +195,12 @@ export function parseLLMJudgeResponse(text: string): number { throw new Error("Judge JSON response is missing a valid CORRECT/WRONG label"); } catch (error) { if (error instanceof SyntaxError) { + const embeddedLabel = normalized.match(/"label"\s*:\s*"(CORRECT|WRONG)"/i)?.[1]; + if (embeddedLabel?.toUpperCase() === "CORRECT") return 1; + if (embeddedLabel?.toUpperCase() === "WRONG") return 0; const upper = normalized.toUpperCase(); - const hasCorrect = upper.includes("CORRECT"); - const hasWrong = upper.includes("WRONG"); + const hasCorrect = /\bCORRECT\b/.test(upper); + const hasWrong = /\bWRONG\b/.test(upper); if (hasCorrect && !hasWrong) return 1; if (hasWrong && !hasCorrect) return 0; throw new Error("Judge response could not be parsed as CORRECT or WRONG", { cause: error }); @@ -185,37 +219,65 @@ export async function evaluateLLMJudge( question: string, goldAnswer: string, generatedAnswer: string, - maxRetries = 3, + maxRetries = 5, ): Promise { const prompt = LLM_JUDGE_PROMPT.replace("{question}", question) .replace("{gold_answer}", goldAnswer) .replace("{generated_answer}", generatedAnswer); let lastError: Error | undefined; + let lastRawResponse: string | null = null; + let lastUsage = unavailableTokenUsage(); + const startedAt = performance.now(); for (let attempt = 1; attempt <= maxRetries; attempt++) { try { const { text, usage } = await generateText({ - model: openrouter(JUDGE_MODEL), - system: "You are an impartial judge evaluating answers to questions. Always respond with valid JSON.", + model: openrouter(getJudgeModel()), + system: JUDGE_SYSTEM_PROMPT, prompt, + maxRetries: 0, + abortSignal: AbortSignal.timeout(MODEL_REQUEST_TIMEOUT_MS), }); + lastRawResponse = text; + lastUsage = tokenUsage(usage.inputTokens, usage.outputTokens, usage.totalTokens); return { score: parseLLMJudgeResponse(text), - token_usage: tokenUsage(usage.inputTokens, usage.outputTokens, usage.totalTokens), + token_usage: lastUsage, + status: "completed", + attempt, + latency_ms: Math.round(performance.now() - startedAt), + prompt_version: "locomo-judge-v1", + prompt_sha256: sha256Text(prompt), + prompt_characters: prompt.length, + system_prompt: JUDGE_SYSTEM_PROMPT, + prompt, + raw_response: text, + parse_status: "parsed", }; } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)); - console.log(`[Judge] Attempt ${attempt}/${maxRetries} failed: ${lastError.message.substring(0, 80)}`); if (attempt < maxRetries) { - // Wait before retry (exponential backoff) await new Promise((resolve) => setTimeout(resolve, 1000 * attempt)); } } } - - throw lastError ?? new Error("Judge failed without returning a result"); + return { + score: 0, + token_usage: lastUsage, + status: "execution_error", + attempt: maxRetries, + latency_ms: Math.round(performance.now() - startedAt), + prompt_version: "locomo-judge-v1", + prompt_sha256: sha256Text(prompt), + prompt_characters: prompt.length, + system_prompt: JUDGE_SYSTEM_PROMPT, + prompt, + raw_response: lastRawResponse, + parse_status: "failed", + error: lastError?.message ?? "Judge failed without returning a result", + }; } /** diff --git a/benchmark/locomo/src/opencontext-client.test.ts b/benchmark/locomo/src/opencontext-client.test.ts new file mode 100644 index 00000000..65957c4a --- /dev/null +++ b/benchmark/locomo/src/opencontext-client.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it, vi } from "vitest"; + +import { AnswererGenerationError, retryAnswererOperation } from "./opencontext-client"; + +describe("retryAnswererOperation", () => { + it("retries transient failures and reports the successful attempt", async () => { + const operation = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new Error("timeout")) + .mockRejectedValueOnce(new Error("provider error")) + .mockResolvedValueOnce("answer"); + const wait = vi.fn(async () => undefined); + + await expect(retryAnswererOperation(operation, 3, wait)).resolves.toEqual({ + result: "answer", + attempt: 3, + }); + expect(operation).toHaveBeenCalledTimes(3); + expect(wait).toHaveBeenNthCalledWith(1, 1000); + expect(wait).toHaveBeenNthCalledWith(2, 2000); + }); + + it("reports the attempt count after exhausting retries", async () => { + const operation = vi.fn<() => Promise>().mockRejectedValue(new Error("timeout")); + const error = await retryAnswererOperation(operation, 3, async () => undefined).catch((caught) => caught); + + expect(error).toBeInstanceOf(AnswererGenerationError); + expect(error).toMatchObject({ attempts: 3 }); + expect(error.message).toContain("Last error: timeout"); + }); +}); diff --git a/benchmark/locomo/src/opencontext-client.ts b/benchmark/locomo/src/opencontext-client.ts index 92ed7656..aa9eaae2 100644 --- a/benchmark/locomo/src/opencontext-client.ts +++ b/benchmark/locomo/src/opencontext-client.ts @@ -1,20 +1,16 @@ -/** - * Client for the OpenContext memory-store HTTP daemon plus the answerer LLM. - * - * The daemon (default http://127.0.0.1:7421, no auth) exposes: - * GET /health - * POST /v1/raw-messages { userId, messages[], embedOnInsert } - * POST /v1/search { userId, query, limit, sources } - * - * The answerer LLM defaults to the Anthropic-compatible endpoint configured - * via ANTHROPIC_AUTH_TOKEN / ANTHROPIC_BASE_URL / ANSWER_MODEL (e.g. MiniMax). - * When ANTHROPIC_AUTH_TOKEN is absent it falls back to OpenRouter - * (OPENROUTER_API_KEY, OPENROUTER_ANSWER_MODEL). - */ +/** Client for the OpenContext daemon and the answerer LLM. */ import { tokenUsage, type TokenUsage } from "../../run-support"; export const DEFAULT_OPENCONTEXT_PORT = 7421; +const configuredModelRequestTimeoutMs = Number.parseInt( + process.env.LOCOMO_MODEL_REQUEST_TIMEOUT_MS ?? "90000", + 10, +); +const MODEL_REQUEST_TIMEOUT_MS = + Number.isInteger(configuredModelRequestTimeoutMs) && configuredModelRequestTimeoutMs >= 10_000 + ? configuredModelRequestTimeoutMs + : 90_000; export function getOpencontextBaseUrl(): string { if (process.env.OPENCONTEXT_URL) return process.env.OPENCONTEXT_URL; @@ -34,20 +30,53 @@ export function getAnswererModelIdentity(): string { export interface GeneratedAnswer { text: string; token_usage: TokenUsage; + attempt: number; +} + +export class AnswererGenerationError extends Error { + constructor( + message: string, + readonly attempts: number, + options?: ErrorOptions, + ) { + super(message, options); + this.name = "AnswererGenerationError"; + } +} + +export async function retryAnswererOperation( + operation: () => Promise, + maxAttempts = 5, + wait: (delayMs: number) => Promise = (delayMs) => + new Promise((resolve) => setTimeout(resolve, delayMs)), +): Promise<{ result: T; attempt: number }> { + let lastError: Error | undefined; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return { result: await operation(), attempt }; + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + if (attempt < maxAttempts) { + process.stderr.write(`[Answerer] Attempt ${attempt}/${maxAttempts} failed: ${lastError.message}\n`); + await wait(1000 * attempt); + } + } + } + throw new AnswererGenerationError( + `Answerer failed after ${maxAttempts} attempts. Last error: ${lastError?.message ?? "unknown error"}`, + maxAttempts, + { cause: lastError }, + ); } export async function checkOpencontextHealth(baseUrl = getOpencontextBaseUrl()): Promise { let res: Response; try { - res = await fetch(`${baseUrl}/health`, { - signal: AbortSignal.timeout(5_000), - }); + res = await fetch(`${baseUrl}/health`, { signal: AbortSignal.timeout(5_000) }); } catch (error) { throw new Error(`OpenContext daemon not reachable at ${baseUrl}: ${(error as Error).message}`); } - if (!res.ok) { - throw new Error(`OpenContext /health returned ${res.status}`); - } + if (!res.ok) throw new Error(`OpenContext /health returned ${res.status}`); } export interface BenchRawMessage { @@ -61,41 +90,93 @@ export interface BenchRawMessage { metadata?: Record; } -const INGEST_BATCH_SIZE = 25; +export const INGEST_BATCH_SIZE = 25; + +export interface IngestMessagesResult { + inserted: number; + warnings: unknown[]; + batches: IngestBatchTrace[]; +} + +export interface IngestBatchTrace { + batch_index: number; + message_ids: string[]; + requested: number; + inserted: number; + status: "completed" | "partial" | "execution_error"; + latency_ms: number; + warnings: unknown[]; + error?: string; +} + +export class IngestMessagesError extends Error { + constructor( + message: string, + readonly result: IngestMessagesResult, + ) { + super(message); + this.name = "IngestMessagesError"; + } +} -/** - * Ingest messages into the OpenContext memory store. `embedOnInsert: true` - * lets the daemon fill embeddings server-side (requires it to be started - * with an --embedding-provider). - */ export async function ingestMessages( messages: BenchRawMessage[], baseUrl = getOpencontextBaseUrl(), userId = BENCH_USER_ID, -): Promise { +): Promise { let inserted = 0; + const warnings: unknown[] = []; + const batches: IngestBatchTrace[] = []; for (let i = 0; i < messages.length; i += INGEST_BATCH_SIZE) { - const batch = messages.slice(i, i + INGEST_BATCH_SIZE).map((m) => ({ - ...m, - userId, - })); - const res = await fetch(`${baseUrl}/v1/raw-messages`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - userId, - messages: batch, - embedOnInsert: true, - }), - signal: AbortSignal.timeout(600_000), - }); - if (!res.ok) { - throw new Error(`ingest /v1/raw-messages failed: ${res.status} ${await res.text()}`); + const batch = messages.slice(i, i + INGEST_BATCH_SIZE).map((message) => ({ ...message, userId })); + const batchIndex = i / INGEST_BATCH_SIZE; + const startedAt = performance.now(); + try { + const res = await fetch(`${baseUrl}/v1/raw-messages`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ userId, messages: batch, embedOnInsert: true }), + signal: AbortSignal.timeout(600_000), + }); + if (!res.ok) throw new Error(`ingest /v1/raw-messages failed: ${res.status} ${await res.text()}`); + const data = (await res.json()) as { count?: number; warnings?: unknown[] }; + const batchInserted = data.count ?? batch.length; + const batchWarnings = data.warnings ?? []; + inserted += batchInserted; + warnings.push(...batchWarnings); + const status = batchInserted === batch.length ? "completed" : "partial"; + batches.push({ + batch_index: batchIndex, + message_ids: batch.map((message) => message.messageId), + requested: batch.length, + inserted: batchInserted, + status, + latency_ms: Math.round(performance.now() - startedAt), + warnings: batchWarnings, + }); + if (status === "partial") { + throw new IngestMessagesError( + `ingest inserted ${batchInserted} of ${batch.length} messages in batch ${batchIndex}`, + { inserted, warnings, batches }, + ); + } + } catch (error) { + if (error instanceof IngestMessagesError) throw error; + const message = error instanceof Error ? error.message : String(error); + batches.push({ + batch_index: batchIndex, + message_ids: batch.map((item) => item.messageId), + requested: batch.length, + inserted: 0, + status: "execution_error", + latency_ms: Math.round(performance.now() - startedAt), + warnings: [], + error: message, + }); + throw new IngestMessagesError(message, { inserted, warnings, batches }); } - const data = (await res.json()) as { count?: number }; - inserted += data.count ?? batch.length; } - return inserted; + return { inserted, warnings, batches }; } export interface MemorySearchHit { @@ -103,15 +184,71 @@ export interface MemorySearchHit { content: string; similarity: number; metadata: Record; + signals?: Record; +} + +export interface MemorySearchEvidence { + id: string; + snippet: string; + score: number; + originalCharacters?: number; + startCharacter?: number; + endCharacter?: number; + truncated?: boolean; +} + +export interface MemorySearchResponse { + query: string; + sources: string[]; + results: MemorySearchHit[]; + evidence?: MemorySearchEvidence[]; + count: number; + warnings: unknown[]; + reasoning?: unknown; + retrievalDiagnostics?: { + mergeStrategy: "rrf" | "similarity"; + candidateLimit: number; + backend?: string; + semanticDegradedReason?: string; + candidateCounts?: { + semantic: number; + lexical: number; + hybrid: number; + entity: number; + fused: number; + final: number; + }; + channels: { + semantic: MemorySearchHit[]; + lexical: MemorySearchHit[]; + hybrid?: MemorySearchHit[]; + entity?: MemorySearchHit[]; + }; + fusedBeforeRerank: MemorySearchHit[]; + reranker?: { + enabled: boolean; + provider?: string; + model?: string; + inputCount: number; + outputCount: number; + latencyMs: number; + orderChanged: boolean; + }; + final?: MemorySearchHit[]; + }; +} + +export interface MemorySearchOptions { + includeRetrievalDiagnostics?: boolean; } -/** Retrieve relevant memories for a question. */ export async function searchMemory( query: string, limit = 8, baseUrl = getOpencontextBaseUrl(), userId = BENCH_USER_ID, -): Promise { + options: MemorySearchOptions = {}, +): Promise { const res = await fetch(`${baseUrl}/v1/search`, { method: "POST", headers: { "Content-Type": "application/json" }, @@ -119,38 +256,53 @@ export async function searchMemory( userId, query, limit, + ...(options.includeRetrievalDiagnostics === undefined + ? {} + : { includeRetrievalDiagnostics: options.includeRetrievalDiagnostics }), sources: ["memory"], }), signal: AbortSignal.timeout(120_000), }); - if (!res.ok) { - throw new Error(`search /v1/search failed: ${res.status} ${await res.text()}`); - } + if (!res.ok) throw new Error(`search /v1/search failed: ${res.status} ${await res.text()}`); const data = (await res.json()) as { + query?: string; + sources?: string[]; + count?: number; + warnings?: unknown[]; + reasoning?: unknown; + retrievalDiagnostics?: MemorySearchResponse["retrievalDiagnostics"]; + evidence?: MemorySearchEvidence[]; results?: Array<{ id: string; content: string; similarity: number; metadata?: Record; + signals?: Record; }>; }; - return (data.results ?? []).map((r) => ({ - id: r.id, - content: r.content, - similarity: r.similarity, - metadata: r.metadata ?? {}, + const results = (data.results ?? []).map((result) => ({ + id: result.id, + content: result.content, + similarity: result.similarity, + metadata: result.metadata ?? {}, + ...(result.signals ? { signals: result.signals } : {}), })); + return { + query: data.query ?? query, + sources: data.sources ?? [], + results, + evidence: data.evidence ?? [], + count: data.count ?? results.length, + warnings: data.warnings ?? [], + ...(data.reasoning === undefined ? {} : { reasoning: data.reasoning }), + ...(data.retrievalDiagnostics === undefined ? {} : { retrievalDiagnostics: data.retrievalDiagnostics }), + }; } -/** - * Generate an answer from the answerer LLM. - * - * Primary: Anthropic-compatible Messages API (MiniMax by default): - * ANTHROPIC_AUTH_TOKEN / ANTHROPIC_BASE_URL / ANSWER_MODEL - * Fallback: OpenRouter chat completions: - * OPENROUTER_API_KEY / OPENROUTER_ANSWER_MODEL (default deepseek/deepseek-chat) - */ -export async function generateAnswer(prompt: string, system?: string): Promise { +async function generateAnswerOnce( + prompt: string, + system?: string, +): Promise> { const token = process.env.ANTHROPIC_AUTH_TOKEN; if (token) { const base = (process.env.ANTHROPIC_BASE_URL ?? "https://api.minimaxi.com/anthropic").replace(/\/+$/, ""); @@ -170,16 +322,14 @@ export async function generateAnswer(prompt: string, system?: string): Promise; usage?: { input_tokens?: number; output_tokens?: number }; }; const text = (data.content ?? []) - .filter((b) => b.type === "text" && typeof b.text === "string") - .map((b) => b.text as string) + .filter((block) => block.type === "text" && typeof block.text === "string") + .map((block) => block.text as string) .join(""); return { text, @@ -202,9 +352,20 @@ export async function generateAnswer(prompt: string, system?: string): Promise { + const { result, attempt } = await retryAnswererOperation(async () => { + const generated = await generateAnswerOnce(prompt, system); + if (!generated.text.trim()) throw new Error("Answerer returned an empty response"); + return generated; + }); + return { ...result, attempt }; +} diff --git a/benchmark/locomo/src/scorer.test.ts b/benchmark/locomo/src/scorer.test.ts new file mode 100644 index 00000000..283a86e8 --- /dev/null +++ b/benchmark/locomo/src/scorer.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from "vitest"; + +import { CATEGORY_NAMES, getCategoryName } from "./scorer"; + +describe("LoCoMo category mapping", () => { + it("uses the dataset's published category ids", () => { + expect(CATEGORY_NAMES).toEqual({ + "1": "multi_hop", + "2": "temporal", + "3": "open_domain", + "4": "single_hop", + "5": "adversarial", + }); + expect(getCategoryName(9)).toBe("category_9"); + }); +}); diff --git a/benchmark/locomo/src/scorer.ts b/benchmark/locomo/src/scorer.ts index 3a3ca130..b6f9fb77 100644 --- a/benchmark/locomo/src/scorer.ts +++ b/benchmark/locomo/src/scorer.ts @@ -3,11 +3,15 @@ */ export const CATEGORY_NAMES: Record = { - "1": "single_hop", + "1": "multi_hop", "2": "temporal", - "3": "multi_hop", - "4": "open_domain", + "3": "open_domain", + "4": "single_hop", "5": "adversarial", // Usually excluded from overall stats }; -export const CATEGORIES = ["single_hop", "temporal", "multi_hop", "open_domain"]; +export const CATEGORIES = ["multi_hop", "temporal", "open_domain", "single_hop"]; + +export function getCategoryName(category: number): string { + return CATEGORY_NAMES[String(category)] ?? `category_${category}`; +} diff --git a/benchmark/locomo/src/types.ts b/benchmark/locomo/src/types.ts index 1fdb43be..6c73d662 100644 --- a/benchmark/locomo/src/types.ts +++ b/benchmark/locomo/src/types.ts @@ -1,6 +1,4 @@ -/** - * Types for LoCoMo benchmark evaluation. - */ +/** Types for LoCoMo benchmark evaluation. */ import type { TokenUsage } from "../../run-support"; @@ -26,6 +24,154 @@ export interface LoCoMoSample { qa_pairs: QAPair[]; } +export const LOCOMO_TRACE_SCHEMA_VERSION = "1.0"; + +export type LoCoMoExecutionStatus = "completed" | "execution_error"; + +export type LoCoMoFailureStage = + | "none" + | "gold_evidence_unavailable" + | "dataset_reference_missing" + | "dataset_reference_partial" + | "ingest_or_index_error" + | "retrieval_error" + | "retrieval_miss" + | "retrieval_partial" + | "answerer_error" + | "context_present_answer_failed" + | "judge_error" + | "provider_error"; + +export interface LoCoMoSessionTrace { + schema_version: string; + sample_id: string; + retrieval_mode: RetrievalMode; + message_id: string; + session_id: string; + session_index: number; + ingest_batch_index: number; + session_date: string | null; + evidence_ids: string[]; + content_sha256: string; + content_characters: number; + ingest_status: "pending" | "not_attempted" | "completed" | "partial" | "execution_error"; + ingest_latency_ms: number | null; + ingest_warnings: unknown[]; + error?: string; +} + +export interface LoCoMoRetrievalHitTrace { + rank: number; + id: string; + similarity: number; + signals: Record | null; + metadata: Record; + content_sha256: string; + content_characters: number; + content_excerpt: string; + /** Full text is retained for final Top-K hits; candidate channels retain hash/excerpt only. */ + content?: string; + session_ids: string[]; + evidence_ids: string[]; + matched_evidence_ids: string[]; + relevant: boolean | null; +} + +export interface LoCoMoRetrievalTrace { + status: "completed" | "execution_error"; + query: string; + user_id: string; + top_k: number; + candidate_k: number | null; + strategy: "daemon-default"; + merge_strategy: "rrf" | "similarity" | null; + threshold: null; + backend: string | null; + semantic_degraded_reason: string | null; + candidate_counts: { + semantic: number; + lexical: number; + hybrid: number; + entity: number; + fused: number; + final: number; + } | null; + latency_ms: number; + response_query: string; + response_sources: string[]; + response_count: number; + response_warnings: unknown[]; + response_reasoning: unknown | null; + premerge_diagnostics_available: boolean; + candidate_channels: { + semantic: LoCoMoRetrievalHitTrace[]; + lexical: LoCoMoRetrievalHitTrace[]; + hybrid: LoCoMoRetrievalHitTrace[]; + entity: LoCoMoRetrievalHitTrace[]; + }; + fused_before_rerank: LoCoMoRetrievalHitTrace[]; + reranker: { + enabled: boolean; + provider: string | null; + model: string | null; + input_count: number; + output_count: number; + latency_ms: number; + order_changed: boolean; + } | null; + hits: LoCoMoRetrievalHitTrace[]; + retrieval_applicable: boolean; + evidence_granularity: "dialog_turn" | "unavailable"; + required_evidence_ids: string[]; + available_evidence_ids: string[]; + missing_evidence_ids: string[]; + retrieved_evidence_ids: string[]; + missed_evidence_ids: string[]; + dataset_evidence_coverage: number | null; + evidence_recall_at_k: number | null; + retrievable_evidence_recall_at_k: number | null; + all_evidence_retrieved: boolean | null; + hit_at_k: number | null; + first_relevant_rank: number | null; + mrr: number | null; + precision_at_k: number | null; + relevant_hit_precision: number | null; + semantic_evidence_recall_at_candidate_k: number | null; + lexical_evidence_recall_at_candidate_k: number | null; + hybrid_evidence_recall_at_candidate_k: number | null; + error?: string; +} + +export interface LoCoMoModelCallTrace { + model: string; + status: "completed" | "skipped" | "execution_error"; + attempt: number; + latency_ms: number; + prompt_version: string; + prompt_sha256: string | null; + prompt_characters: number; + system_prompt: string | null; + prompt: string | null; + token_usage: TokenUsage; + error?: string; +} + +export interface LoCoMoAnswerTrace extends LoCoMoModelCallTrace { + included_hit_ids: string[]; + included_context_characters: number; +} + +export interface LoCoMoJudgeTrace extends LoCoMoModelCallTrace { + raw_response: string | null; + parse_status: "parsed" | "skipped" | "failed"; +} + +export interface LoCoMoQuestionTrace { + retrieval: LoCoMoRetrievalTrace | null; + answerer: LoCoMoAnswerTrace | null; + judge: LoCoMoJudgeTrace | null; +} + export interface EvaluationResult { sample_id: string; retrieval_mode: RetrievalMode; @@ -38,12 +184,18 @@ export interface EvaluationResult { } export interface Prediction { - status: "completed" | "execution_error"; + trace_schema_version: string; + sample_sha256: string; + question_sha256: string; + status: LoCoMoExecutionStatus; attempt: number; answerer_model: string; judge_model: string; - error?: string; + execution_error?: { stage: string; message: string }; token_usage: TokenUsage; + sample_id: string; + question_index: number; + retrieval_mode: RetrievalMode; question: string; answer: string; response: string; @@ -59,6 +211,8 @@ export interface Prediction { bleu3: number; bleu4: number; evidence: string[]; + trace: LoCoMoQuestionTrace; + failure_stage: LoCoMoFailureStage; } export interface Chunk { diff --git a/benchmark/locomo/start-daemon.ps1 b/benchmark/locomo/start-daemon.ps1 new file mode 100644 index 00000000..db0d1ada --- /dev/null +++ b/benchmark/locomo/start-daemon.ps1 @@ -0,0 +1,120 @@ +param( + [ValidateRange(1, 65535)] + [int]$Port = 7421, + + [string]$DatabasePath +) + +$ErrorActionPreference = "Stop" +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path +$runtimeRoot = Join-Path $PSScriptRoot "runtime" +$launchTag = Get-Date -Format "yyyyMMdd-HHmmss" +if ($DatabasePath) { + if (-not (Test-Path -LiteralPath $DatabasePath -PathType Leaf)) { + throw "Existing database not found: $DatabasePath" + } + $dbPath = (Resolve-Path -LiteralPath $DatabasePath).Path + $runtimeDir = Split-Path -Parent $dbPath + $stdoutPath = Join-Path $runtimeDir "daemon-resume-$launchTag.stdout.log" + $stderrPath = Join-Path $runtimeDir "daemon-resume-$launchTag.stderr.log" +} else { + $runtimeDir = Join-Path $runtimeRoot $launchTag + $dbPath = Join-Path $runtimeDir "store.db" + $stdoutPath = Join-Path $runtimeDir "daemon.stdout.log" + $stderrPath = Join-Path $runtimeDir "daemon.stderr.log" +} +$cliPath = Join-Path $repoRoot "packages\opencontext\dist\cli\opencontext.js" + +if (-not (Test-Path -LiteralPath $cliPath)) { + throw "OpenContext CLI is not built: $cliPath" +} +if (Get-NetTCPConnection -State Listen -LocalPort $Port -ErrorAction SilentlyContinue) { + throw "Port $Port already has a listener; refusing to replace it." +} + +New-Item -ItemType Directory -Force -Path $runtimeDir | Out-Null + +$previousDbPath = $env:MEMORY_STORE_DB_PATH +$previousRawStoreBackend = $env:OPENCONTEXT_MEMORY_STORE_BACKEND +$previousRerankerDtype = $env:LOCAL_RERANKER_DTYPE +$previousRerankerLocalOnly = $env:LOCAL_RERANKER_LOCAL_ONLY +try { + $env:MEMORY_STORE_DB_PATH = $dbPath + $env:OPENCONTEXT_MEMORY_STORE_BACKEND = "sqlite" + $env:LOCAL_RERANKER_DTYPE = "q8" + $env:LOCAL_RERANKER_LOCAL_ONLY = "true" + + $arguments = @( + $cliPath, + "http", + "--host", "127.0.0.1", + "--port", [string]$Port, + "--embedding-provider", "local", + "--embedding-model", "Xenova/all-MiniLM-L6-v2", + "--embedding-cache-dir", (Join-Path $env:USERPROFILE ".cache\opencontext\local-embeddings"), + "--memory-backend", "sqlite-vec", + "--reranker-provider", "local", + "--reranker-model", "Xenova/ms-marco-MiniLM-L-6-v2", + "--reranker-cache-dir", (Join-Path $env:USERPROFILE ".cache\opencontext\local-reranker"), + "--reranker-batch-size", "8", + "--reranker-max-tokens", "512", + "--insights-backend", "none", + "--knowledge-backend", "none" + ) + $process = Start-Process ` + -FilePath "node" ` + -ArgumentList $arguments ` + -WorkingDirectory $repoRoot ` + -RedirectStandardOutput $stdoutPath ` + -RedirectStandardError $stderrPath ` + -WindowStyle Hidden ` + -PassThru +} finally { + if ($null -eq $previousDbPath) { Remove-Item Env:MEMORY_STORE_DB_PATH -ErrorAction SilentlyContinue } + else { $env:MEMORY_STORE_DB_PATH = $previousDbPath } + if ($null -eq $previousRawStoreBackend) { Remove-Item Env:OPENCONTEXT_MEMORY_STORE_BACKEND -ErrorAction SilentlyContinue } + else { $env:OPENCONTEXT_MEMORY_STORE_BACKEND = $previousRawStoreBackend } + if ($null -eq $previousRerankerDtype) { Remove-Item Env:LOCAL_RERANKER_DTYPE -ErrorAction SilentlyContinue } + else { $env:LOCAL_RERANKER_DTYPE = $previousRerankerDtype } + if ($null -eq $previousRerankerLocalOnly) { Remove-Item Env:LOCAL_RERANKER_LOCAL_ONLY -ErrorAction SilentlyContinue } + else { $env:LOCAL_RERANKER_LOCAL_ONLY = $previousRerankerLocalOnly } +} + +$healthy = $false +for ($attempt = 0; $attempt -lt 60; $attempt++) { + try { + Invoke-RestMethod -Uri "http://127.0.0.1:$Port/health" -TimeoutSec 2 | Out-Null + $healthy = $true + break + } catch { + if ($process.HasExited) { break } + Start-Sleep -Seconds 1 + } +} + +if (-not $healthy) { + if (-not $process.HasExited) { Stop-Process -Id $process.Id } + $details = if (Test-Path -LiteralPath $stderrPath) { + (Get-Content -LiteralPath $stderrPath -Tail 80) -join [Environment]::NewLine + } else { + "No stderr log was created." + } + throw "Daemon failed health check. $details" +} + +$metadata = [ordered]@{ + pid = $process.Id + port = $Port + database = $dbPath + stdout = $stdoutPath + stderr = $stderrPath + started_at = (Get-Date).ToString("o") + embedding_provider = "local" + embedding_model = "Xenova/all-MiniLM-L6-v2" + memory_backend = "sqlite-vec" + reranker_provider = "local" + reranker_model = "Xenova/ms-marco-MiniLM-L-6-v2" + resumed_database = [bool]$DatabasePath +} +$metadata | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $runtimeDir "daemon.json") -Encoding utf8 +$metadata | ConvertTo-Json