Fix/benchmark runnable baseline

.github/workflows/ci.yml

@@ -54,6 +54,63 @@ jobs:
       - run: python -m py_compile benchmark/aml-local/serve.py benchmark/aml-local/test_contract.py
       - run: python benchmark/aml-local/test_contract.py
 
+  benchmark-smoke:
+    # Offline-only coverage for the supported benchmark entrypoints. This job
+    # must not receive provider credentials, contact paid models, or download
+    # full benchmark datasets.
+    name: Benchmark smoke (offline)
+    runs-on: ubuntu-latest
+    env:
+      ANTHROPIC_AUTH_TOKEN: ""
+      OPENROUTER_API_KEY: ""
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: pnpm/action-setup@v4
+        with:
+          version: 10.14.0
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+          cache: pnpm
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pnpm install --frozen-lockfile
+      - name: Typecheck benchmark CLIs
+        run: |
+          pnpm --filter @melandlabs/benchmark-locomo typecheck
+          pnpm --filter @melandlabs/benchmark-longmemeval typecheck
+          pnpm --filter @melandlabs/benchmark-beam typecheck
+      - name: Lint benchmark CLIs
+        run: |
+          pnpm exec biome lint benchmark/run-support.ts
+          pnpm --filter @melandlabs/benchmark-locomo lint
+          pnpm --filter @melandlabs/benchmark-longmemeval lint
+          pnpm --filter @melandlabs/benchmark-beam lint
+      - name: Check CLI help without preflight
+        run: |
+          pnpm --filter @melandlabs/benchmark-locomo benchmark -- --help
+          pnpm --filter @melandlabs/benchmark-longmemeval benchmark -- --help
+          pnpm --filter @melandlabs/benchmark-beam benchmark -- --help
+          python benchmark/beam/dataset/convert.py --help
+          python benchmark/aml-local/retrieve.py --help
+      - name: Run checkpoint, preflight, mapping, and AML fixture tests
+        run: |
+          pnpm --filter @melandlabs/benchmark-locomo test
+          pnpm --filter @melandlabs/benchmark-longmemeval test
+          (cd benchmark/beam/dataset && python -m unittest test_convert.py)
+          (cd benchmark/aml-local && python -m unittest test_retrieve.py)
+      - name: Convert bundled BEAM sample without Hugging Face dependencies
+        run: python benchmark/beam/dataset/convert.py --scale sample --out-dir "$RUNNER_TEMP/beam-sample"
+      - name: Confirm smoke tests did not change the checkout
+        run: |
+          if [[ -n "$(git status --porcelain --untracked-files=all)" ]]; then
+            git status --short --untracked-files=all
+            exit 1
+          fi
+
   lint:
     name: Lint (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
@@ -238,4 +295,4 @@ jobs:
 
       - name: Run smoke tests
         working-directory: examples
-        run: pnpm test
+        run: pnpm test

.gitignore

@@ -193,8 +193,7 @@ benchmark/beam/dataset/beam_10m.json
 benchmark/scriptmem/dataset/raw/
 benchmark/scriptmem/dataset/scripts/
 
-# AML local experiment harness (local-only probes/builders, not for upstream)
-benchmark/aml-local/retrieve.py
+# AML local experiment helpers (local-only probes/builders, not for upstream)
 benchmark/aml-local/_probe_openrouter.py
 benchmark/aml-local/_lme_sample100_ids.txt
 benchmark/aml-local/build_union.py

benchmark/aml-local/README.md

@@ -12,6 +12,15 @@ dataset -> retrieve.py -> OpenContext (POST /v1/raw-messages ingest, per-sample
         -> judged.jsonl + aggregate score
 ```
 
+This is the **AML-compatible local evaluation** path. It uses public/local
+datasets with the vendored AML answer and scoring pipelines, so it is distinct
+from both:
+
+- the repository's internal TypeScript evaluations under `benchmark/locomo`,
+  `benchmark/longmemeval`, and `benchmark/beam`; and
+- the hosted AML leaderboard evaluation, which uses the platform's controlled
+  datasets and orchestration through the Add/Search adapter described below.
+
 ## Prerequisites
 
 - The OpenContext daemon is running (`curl http://127.0.0.1:7421/health`):
@@ -54,6 +63,12 @@ Common flags: `-Limit N` (number of samples; for personamem = number of personas
 already-ingested memories, re-retrieve only), `-Dataset beam_1m.json` (switch BEAM
 dataset), `-Mode mcq|generative` (personamem only, default mcq).
 
+Before retrieval, ingest, or model calls, the runner checks the dataset and
+sample filter, daemon, `OPENROUTER_API_KEY`, AML Python interpreter and `httpx`,
+pipeline files, output path, and arguments. All detected failures are reported
+together without printing secret values. `python retrieve.py --help` does not run
+preflight.
+
 PersonaMem-v2 reads `benchmark/personamem-v2/dataset/benchmark.csv` plus the
 per-persona 32k chat histories — fetch them with
 `python benchmark/personamem-v2/dataset/download.py --max-personas 5`. Each
@@ -72,7 +87,10 @@ The evaluate step scores all 457 gold questions; unanswered ones count as wrong,
 so use `-MaxQuestions` only for smoke tests and compare on full runs.
 
 Artifacts land in `outputs/<bench>/`: `input.jsonl` (retrieval results),
-`answers.jsonl` (generated answers), `judged.jsonl` (scoring details).
+`answers.jsonl` (generated answers), `judged.jsonl` (scoring details), and
+`run-manifest.json` (Git commit, dataset identity, models, retrieval settings,
+parameters, and wall-clock time). The vendored AML pipelines do not expose
+provider token usage, so the manifest records those fields as `null`.
 
 ## Local smoke baselines (2026-08-18, daemon: sqlite-vec + local embeddings)
 

benchmark/aml-local/fixtures/retrieve/clbench.jsonl

@@ -0,0 +1 @@
+{"metadata":{"task_id":"clbench-fixture-1"},"messages":[{"role":"user","content":"I use a standing desk."},{"role":"assistant","content":"Understood."},{"role":"user","content":"What kind of desk do I use?"}],"rubrics":[{"rubric_id":"r1","rubric_content":"Mentions a standing desk."}]}

benchmark/aml-local/fixtures/retrieve/locomo.json

@@ -0,0 +1,19 @@
+[
+	{
+		"sample_id": "locomo-fixture-1",
+		"conversation": {
+			"session_1": [
+				{ "speaker": "Caroline", "text": "My favorite color is green." },
+				{ "speaker": "Melanie", "text": "I will remember that." }
+			],
+			"session_1_date_time": "2024-01-02T09:00:00Z"
+		},
+		"qa": [
+			{
+				"question": "What is Caroline's favorite color?",
+				"answer": "Green",
+				"category": 1
+			}
+		]
+	}
+]

benchmark/aml-local/fixtures/retrieve/longmemeval.json

@@ -0,0 +1,16 @@
+[
+	{
+		"question_id": "lme-fixture-1",
+		"question": "Where does the user live?",
+		"question_type": "single-session-user",
+		"answer": "Berlin",
+		"haystack_session_ids": ["session-1"],
+		"haystack_dates": ["2024-05-01T10:00:00Z"],
+		"haystack_sessions": [
+			[
+				{ "role": "user", "content": "I moved to Berlin." },
+				{ "role": "assistant", "content": "I hope the move went well." }
+			]
+		]
+	}
+]

benchmark/aml-local/fixtures/retrieve/personamem/benchmark.csv

@@ -0,0 +1,2 @@
+persona_id,chat_history_32k_link,user_query,correct_answer,incorrect_answers,preference
+persona-1,history.json,"{'role': 'user', 'content': 'Which pet would suit me?'}",A cat,"['A dog', 'A bird']",Prefers cats

benchmark/aml-local/fixtures/retrieve/personamem/history.json

@@ -0,0 +1,6 @@
+{
+	"chat_history": [
+		{ "role": "user", "content": "Cats are my favorite pets." },
+		{ "role": "assistant", "content": "I will keep that preference in mind." }
+	]
+}

benchmark/aml-local/fixtures/retrieve/scriptmem/angry.json

@@ -0,0 +1,11 @@
+[
+	{
+		"sample_id": "fixture-1",
+		"conversation": {
+			"speakers": ["Juror 1", "Juror 2"],
+			"session_1": [{ "speaker": "Juror 1", "text": "The vote is not unanimous." }],
+			"session_1_date_time": "1957-01-01T00:00:00Z"
+		},
+		"qa": [{ "question": "Was the vote unanimous?", "qa_type": "single", "answer": "B" }]
+	}
+]

benchmark/aml-local/fixtures/retrieve/scriptmem/enemy.json

@@ -0,0 +1,11 @@
+[
+	{
+		"sample_id": "fixture-1",
+		"conversation": {
+			"speakers": ["Thomas", "Peter"],
+			"session_1": [{ "speaker": "Thomas", "text": "The baths are contaminated." }],
+			"session_1_date_time": "1882-01-01T00:00:00Z"
+		},
+		"qa": [{ "question": "What is wrong with the baths?", "qa_type": "single", "answer": "A" }]
+	}
+]

benchmark/aml-local/fixtures/retrieve/scriptmem/friends.json

@@ -0,0 +1,11 @@
+[
+	{
+		"sample_id": "fixture-1",
+		"conversation": {
+			"speakers": ["Rachel", "Monica"],
+			"session_1": [{ "speaker": "Rachel", "text": "I left the wedding." }],
+			"session_1_date_time": "1994-09-22T00:00:00Z"
+		},
+		"qa": [{ "question": "What did Rachel leave?", "qa_type": "single", "answer": "C" }]
+	}
+]

benchmark/aml-local/fixtures/retrieve/scriptmem/man_earth.json

@@ -0,0 +1,11 @@
+[
+	{
+		"sample_id": "fixture-1",
+		"conversation": {
+			"speakers": ["John", "Dan"],
+			"session_1": [{ "speaker": "John", "text": "I am moving away." }],
+			"session_1_date_time": "2007-01-01T00:00:00Z"
+		},
+		"qa": [{ "question": "What is John planning to do?", "qa_type": "single", "answer": "D" }]
+	}
+]