From f0293e3220cb559e24ed40ef3fe7a0ec5a5e0196 Mon Sep 17 00:00:00 2001 From: Steven Chong <25894545+teamchong@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:15:38 -0400 Subject: [PATCH 01/11] eval(qwen): add Qwen 3.8 quality benchmarks and native 14px pilot --- README.md | 1 + eval/qwen-profile/QUALITY_RESULTS.md | 33 + eval/qwen-profile/gist-recall-results.json | 2295 +++++++++ eval/qwen-profile/gist-recall.mjs | 142 + .../novel-arithmetic-results.json | 4332 +++++++++++++++++ eval/qwen-profile/novel-arithmetic.mjs | 110 + eval/qwen-profile/qwen-client.mjs | 116 + .../verbatim-hex-14px-results.json | 130 + eval/qwen-profile/verbatim-hex-results.json | 161 + eval/qwen-profile/verbatim-hex.mjs | 90 + 10 files changed, 7410 insertions(+) create mode 100644 eval/qwen-profile/QUALITY_RESULTS.md create mode 100644 eval/qwen-profile/gist-recall-results.json create mode 100644 eval/qwen-profile/gist-recall.mjs create mode 100644 eval/qwen-profile/novel-arithmetic-results.json create mode 100644 eval/qwen-profile/novel-arithmetic.mjs create mode 100644 eval/qwen-profile/qwen-client.mjs create mode 100644 eval/qwen-profile/verbatim-hex-14px-results.json create mode 100644 eval/qwen-profile/verbatim-hex-results.json create mode 100644 eval/qwen-profile/verbatim-hex.mjs diff --git a/README.md b/README.md index 6b9b483fc..db2038e61 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,7 @@ is confabulations, so lower is better. | `grok-4.5` | **100/100** | **97/98** | 17/18 | **0/16** | 0/15 | native 14px/84 quality suite (live profile); [quality](eval/grok-density/QUALITY_RESULTS.md), [native-sweep](eval/grok-density/native-sweep/RESULTS.md) | | `grok-4.6` high | **100/100** | **97/98** | 17/18 | **0/16** | 0/15 | native 14px/84, reasoning high; [quality](eval/grok-profile/QUALITY_RESULTS.md) | | `moonshotai/kimi-k3` | 79/100 | 84/98 | 15/18 | 1/16 | 0/15 | generic GPT profile: [quality results](eval/sol-profile/KIMI_K3_QUALITY_RESULTS.md) | +| `qwen-3.8` (`@cf/qwen/qwen3.8-27b`) | 98/100 | 72/98 | 11/18 | **0/16** | 0/15 | prior 5×8 broad suite (0/15 hex); native 14px pilot: 8/8 exact, 0 inventions, 11/15 hex: [pilot & quality](eval/qwen-profile/QUALITY_RESULTS.md) | ### Native-profile cost check diff --git a/eval/qwen-profile/QUALITY_RESULTS.md b/eval/qwen-profile/QUALITY_RESULTS.md new file mode 100644 index 000000000..a67093ed9 --- /dev/null +++ b/eval/qwen-profile/QUALITY_RESULTS.md @@ -0,0 +1,33 @@ +# Qwen 3.8 quality results + +Model: `workers-ai/@cf/qwen/qwen3.8-27b` (`qwen 3.8`) through Cloudflare Workers AI via ocproxy. + +## Benchmark Summary + +| Geometry | Arithmetic (N=100) | Gist (N=98) | State (N=18) | Never-Stated (N=16) | Dense Hex (N=15) | Paired Pilot (8 facts) | +|---|---:|---:|---:|---:|---:|---:| +| **Spleen 5×8 (152 cols)** | 98/100 | 72/98 | 11/18 | **0/16** | 0/15 | 0/8 exact (unreadable / confabulated) | +| **JetBrains Mono 14px (84 cols)** | — | — | — | — | **11/15** | **8/8 exact (100%)**, 2/2 gist, 2/2 guard | + +## Paired Pilot (Alpha / Beta) + +Evaluated against the paired synthetic terminal pilot (Alpha & Beta sessions with 4 exact-fact extractions + 1 gist + 1 unstated guard each): + +| Profile | Fixture | Exact Facts (4) | Gist | Guard (NOT STATED) | Notes | +|---|---|---:|:---:|:---:|---| +| Spleen 5×8 (152 cols) | Alpha | 0/4 | fail | fail | Raw output empty; font unreadable | +| Spleen 5×8 (152 cols) | Beta | 0/4 | pass | pass | Confabulated `path` and `port` | +| **JetBrains Mono 14px (84 cols)** | Alpha | **4/4** | **pass** | **pass** | `fingerprint`, `camelCase`, `path`, `port` exact | +| **JetBrains Mono 14px (84 cols)** | Beta | **4/4** | **pass** | **pass** | `fingerprint`, `camelCase`, `path`, `port` exact | + +## Verbatim Hex (15 trials) + +- **Spleen 5×8 (152 cols)**: 0/15 exact matches. 5×8 bitmap glyphs are too dense for Qwen 3.8's vision encoder. +- **JetBrains Mono 14px (84 cols)**: **11/15** exact 12-char hex matches (vs Sol 7/8, Grok 4/8). + +## Receipts + +- `novel-arithmetic-results.json` +- `gist-recall-results.json` +- `verbatim-hex-results.json` (Spleen 5×8 baseline) +- `verbatim-hex-14px-results.json` (JetBrains Mono 14px pilot) diff --git a/eval/qwen-profile/gist-recall-results.json b/eval/qwen-profile/gist-recall-results.json new file mode 100644 index 000000000..451c882e5 --- /dev/null +++ b/eval/qwen-profile/gist-recall-results.json @@ -0,0 +1,2295 @@ +{ + "generatedAt": "2026-08-19T21:52:30.293Z", + "model": "workers-ai/@cf/qwen/qwen3.8-27b", + "live": true, + "recipe": { + "cols": 152, + "maxH": 1932, + "style": { + "font": "spleen-5x8", + "cellWBonus": 0, + "cellHBonus": 0, + "aa": true, + "grid": false, + "gridCols": 0, + "colorCycle": false, + "markerScale": 1, + "markerRed": false, + "inkDilate": 0 + }, + "factsheet": true + }, + "answerable": { + "correct": 72, + "completed": 98, + "n": 98 + }, + "state": { + "correct": 11, + "completed": 18, + "n": 18 + }, + "unanswerable": { + "confabulated": 0, + "completed": 16, + "n": 16 + }, + "rows": [ + { + "tier": "work", + "session": 0, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "mobx", + "answer": "UNKNOWN", + "ok": false, + "raw": "[\n \"UNKNOWN\",\n \"7000\",\n \"src/batcher/core.ts\",\n \"Tobias Okafor\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2971, + "completion_tokens": 4835, + "total_tokens": 7806, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1528.0863112211227 + } + }, + { + "tier": "work", + "session": 0, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "7880", + "answer": "7000", + "ok": false, + "raw": "[\n \"UNKNOWN\",\n \"7000\",\n \"src/batcher/core.ts\",\n \"Tobias Okafor\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2971, + "completion_tokens": 4835, + "total_tokens": 7806, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1528.0863112211227 + } + }, + { + "tier": "work", + "session": 0, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/batcher/core.ts", + "answer": "src/batcher/core.ts", + "ok": true, + "raw": "[\n \"UNKNOWN\",\n \"7000\",\n \"src/batcher/core.ts\",\n \"Tobias Okafor\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2971, + "completion_tokens": 4835, + "total_tokens": 7806, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1528.0863112211227 + } + }, + { + "tier": "work", + "session": 0, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Tobias Okafor", + "answer": "Tobias Okafor", + "ok": true, + "raw": "[\n \"UNKNOWN\",\n \"7000\",\n \"src/batcher/core.ts\",\n \"Tobias Okafor\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2971, + "completion_tokens": 4835, + "total_tokens": 7806, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1528.0863112211227 + } + }, + { + "tier": "work", + "session": 0, + "type": "negation", + "q": "Was LEGACY_PINS enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\n \"UNKNOWN\",\n \"7000\",\n \"src/batcher/core.ts\",\n \"Tobias Okafor\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2971, + "completion_tokens": 4835, + "total_tokens": 7806, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1528.0863112211227 + } + }, + { + "tier": "work", + "session": 0, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\n \"UNKNOWN\",\n \"7000\",\n \"src/batcher/core.ts\",\n \"Tobias Okafor\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2971, + "completion_tokens": 4835, + "total_tokens": 7806, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1528.0863112211227 + } + }, + { + "tier": "work", + "session": 1, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "xstate", + "answer": "zustand", + "ok": false, + "raw": "[\n \"zustand\",\n \"500\",\n \"src/cursor/sync.ts\",\n \"Farid Moreau\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3044, + "completion_tokens": 1062, + "total_tokens": 4106, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 433.47271224856377 + } + }, + { + "tier": "work", + "session": 1, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "3480", + "answer": "500", + "ok": false, + "raw": "[\n \"zustand\",\n \"500\",\n \"src/cursor/sync.ts\",\n \"Farid Moreau\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3044, + "completion_tokens": 1062, + "total_tokens": 4106, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 433.47271224856377 + } + }, + { + "tier": "work", + "session": 1, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/sync.ts", + "answer": "src/cursor/sync.ts", + "ok": true, + "raw": "[\n \"zustand\",\n \"500\",\n \"src/cursor/sync.ts\",\n \"Farid Moreau\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3044, + "completion_tokens": 1062, + "total_tokens": 4106, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 433.47271224856377 + } + }, + { + "tier": "work", + "session": 1, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Farid Moreau", + "answer": "Farid Moreau", + "ok": true, + "raw": "[\n \"zustand\",\n \"500\",\n \"src/cursor/sync.ts\",\n \"Farid Moreau\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3044, + "completion_tokens": 1062, + "total_tokens": 4106, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 433.47271224856377 + } + }, + { + "tier": "work", + "session": 1, + "type": "negation", + "q": "Was LEGACY_PINS enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\n \"zustand\",\n \"500\",\n \"src/cursor/sync.ts\",\n \"Farid Moreau\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3044, + "completion_tokens": 1062, + "total_tokens": 4106, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 433.47271224856377 + } + }, + { + "tier": "work", + "session": 1, + "type": "unanswerable", + "q": "Which AWS region was the failover assigned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\n \"zustand\",\n \"500\",\n \"src/cursor/sync.ts\",\n \"Farid Moreau\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3044, + "completion_tokens": 1062, + "total_tokens": 4106, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 433.47271224856377 + } + }, + { + "tier": "work", + "session": 2, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "immer", + "answer": "immer", + "ok": true, + "raw": "[\n \"immer\",\n \"91100\",\n \"src/cursor/io.ts\",\n \"Nadia Costa\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3057, + "completion_tokens": 868, + "total_tokens": 3925, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 377.56816497445107 + } + }, + { + "tier": "work", + "session": 2, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "9110", + "answer": "91100", + "ok": false, + "raw": "[\n \"immer\",\n \"91100\",\n \"src/cursor/io.ts\",\n \"Nadia Costa\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3057, + "completion_tokens": 868, + "total_tokens": 3925, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 377.56816497445107 + } + }, + { + "tier": "work", + "session": 2, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/io.ts", + "answer": "src/cursor/io.ts", + "ok": true, + "raw": "[\n \"immer\",\n \"91100\",\n \"src/cursor/io.ts\",\n \"Nadia Costa\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3057, + "completion_tokens": 868, + "total_tokens": 3925, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 377.56816497445107 + } + }, + { + "tier": "work", + "session": 2, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Nadia Costa", + "answer": "Nadia Costa", + "ok": true, + "raw": "[\n \"immer\",\n \"91100\",\n \"src/cursor/io.ts\",\n \"Nadia Costa\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3057, + "completion_tokens": 868, + "total_tokens": 3925, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 377.56816497445107 + } + }, + { + "tier": "work", + "session": 2, + "type": "negation", + "q": "Was ENABLE_SHARDING enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\n \"immer\",\n \"91100\",\n \"src/cursor/io.ts\",\n \"Nadia Costa\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3057, + "completion_tokens": 868, + "total_tokens": 3925, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 377.56816497445107 + } + }, + { + "tier": "work", + "session": 2, + "type": "unanswerable", + "q": "What git tag was the hotfix released under?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\n \"immer\",\n \"91100\",\n \"src/cursor/io.ts\",\n \"Nadia Costa\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3057, + "completion_tokens": 868, + "total_tokens": 3925, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 377.56816497445107 + } + }, + { + "tier": "work", + "session": 3, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "immer", + "answer": "immer", + "ok": true, + "raw": "[\n \"immer\",\n \"340\",\n \"src/scheduler/core.ts\",\n \"Dmitri Berg\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2621, + "completion_tokens": 2403, + "total_tokens": 5024, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 806.2772482633591 + } + }, + { + "tier": "work", + "session": 3, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "3470", + "answer": "340", + "ok": false, + "raw": "[\n \"immer\",\n \"340\",\n \"src/scheduler/core.ts\",\n \"Dmitri Berg\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2621, + "completion_tokens": 2403, + "total_tokens": 5024, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 806.2772482633591 + } + }, + { + "tier": "work", + "session": 3, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/scheduler/core.ts", + "answer": "src/scheduler/core.ts", + "ok": true, + "raw": "[\n \"immer\",\n \"340\",\n \"src/scheduler/core.ts\",\n \"Dmitri Berg\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2621, + "completion_tokens": 2403, + "total_tokens": 5024, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 806.2772482633591 + } + }, + { + "tier": "work", + "session": 3, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Dmitri Berg", + "answer": "Dmitri Berg", + "ok": true, + "raw": "[\n \"immer\",\n \"340\",\n \"src/scheduler/core.ts\",\n \"Dmitri Berg\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2621, + "completion_tokens": 2403, + "total_tokens": 5024, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 806.2772482633591 + } + }, + { + "tier": "work", + "session": 3, + "type": "negation", + "q": "Was HOT_RELOAD_V2 enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\n \"immer\",\n \"340\",\n \"src/scheduler/core.ts\",\n \"Dmitri Berg\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2621, + "completion_tokens": 2403, + "total_tokens": 5024, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 806.2772482633591 + } + }, + { + "tier": "work", + "session": 3, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\n \"immer\",\n \"340\",\n \"src/scheduler/core.ts\",\n \"Dmitri Berg\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2621, + "completion_tokens": 2403, + "total_tokens": 5024, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 806.2772482633591 + } + }, + { + "tier": "work", + "session": 4, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "redux-toolkit", + "answer": "radix-toolkit", + "ok": false, + "raw": "[\n \"radix-toolkit\",\n \"6210\",\n \"src/cursor/io.ts\",\n \"Tobias Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2628, + "completion_tokens": 1436, + "total_tokens": 4064, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 525.2545279562473 + } + }, + { + "tier": "work", + "session": 4, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "6210", + "answer": "6210", + "ok": true, + "raw": "[\n \"radix-toolkit\",\n \"6210\",\n \"src/cursor/io.ts\",\n \"Tobias Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2628, + "completion_tokens": 1436, + "total_tokens": 4064, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 525.2545279562473 + } + }, + { + "tier": "work", + "session": 4, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/io.ts", + "answer": "src/cursor/io.ts", + "ok": true, + "raw": "[\n \"radix-toolkit\",\n \"6210\",\n \"src/cursor/io.ts\",\n \"Tobias Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2628, + "completion_tokens": 1436, + "total_tokens": 4064, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 525.2545279562473 + } + }, + { + "tier": "work", + "session": 4, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Tobias Khoury", + "answer": "Tobias Khoury", + "ok": true, + "raw": "[\n \"radix-toolkit\",\n \"6210\",\n \"src/cursor/io.ts\",\n \"Tobias Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2628, + "completion_tokens": 1436, + "total_tokens": 4064, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 525.2545279562473 + } + }, + { + "tier": "work", + "session": 4, + "type": "negation", + "q": "Was ASYNC_FSYNC enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\n \"radix-toolkit\",\n \"6210\",\n \"src/cursor/io.ts\",\n \"Tobias Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2628, + "completion_tokens": 1436, + "total_tokens": 4064, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 525.2545279562473 + } + }, + { + "tier": "work", + "session": 4, + "type": "unanswerable", + "q": "Which AWS region was the failover assigned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\n \"radix-toolkit\",\n \"6210\",\n \"src/cursor/io.ts\",\n \"Tobias Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2628, + "completion_tokens": 1436, + "total_tokens": 4064, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 525.2545279562473 + } + }, + { + "tier": "work", + "session": 5, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "redux-toolkit", + "answer": "redux-toolkit", + "ok": true, + "raw": "[\n \"redux-toolkit\",\n \"1250\",\n \"src/cursor/io.ts\",\n \"lucia costa\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2819, + "completion_tokens": 938, + "total_tokens": 3757, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 388.1954420506954 + } + }, + { + "tier": "work", + "session": 5, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "1250", + "answer": "1250", + "ok": true, + "raw": "[\n \"redux-toolkit\",\n \"1250\",\n \"src/cursor/io.ts\",\n \"lucia costa\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2819, + "completion_tokens": 938, + "total_tokens": 3757, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 388.1954420506954 + } + }, + { + "tier": "work", + "session": 5, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/io.ts", + "answer": "src/cursor/io.ts", + "ok": true, + "raw": "[\n \"redux-toolkit\",\n \"1250\",\n \"src/cursor/io.ts\",\n \"lucia costa\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2819, + "completion_tokens": 938, + "total_tokens": 3757, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 388.1954420506954 + } + }, + { + "tier": "work", + "session": 5, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Lucia Costa", + "answer": "lucia costa", + "ok": true, + "raw": "[\n \"redux-toolkit\",\n \"1250\",\n \"src/cursor/io.ts\",\n \"lucia costa\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2819, + "completion_tokens": 938, + "total_tokens": 3757, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 388.1954420506954 + } + }, + { + "tier": "work", + "session": 5, + "type": "negation", + "q": "Was HOT_RELOAD_V2 enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\n \"redux-toolkit\",\n \"1250\",\n \"src/cursor/io.ts\",\n \"lucia costa\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2819, + "completion_tokens": 938, + "total_tokens": 3757, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 388.1954420506954 + } + }, + { + "tier": "work", + "session": 5, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\n \"redux-toolkit\",\n \"1250\",\n \"src/cursor/io.ts\",\n \"lucia costa\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2819, + "completion_tokens": 938, + "total_tokens": 3757, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 388.1954420506954 + } + }, + { + "tier": "work", + "session": 6, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "immer", + "answer": "immer", + "ok": true, + "raw": "[\n \"immer\",\n \"6200\",\n \"src/scheduler/sync.ts\",\n \"Tobias Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3021, + "completion_tokens": 671, + "total_tokens": 3692, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 318.7863509654999 + } + }, + { + "tier": "work", + "session": 6, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "6290", + "answer": "6200", + "ok": false, + "raw": "[\n \"immer\",\n \"6200\",\n \"src/scheduler/sync.ts\",\n \"Tobias Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3021, + "completion_tokens": 671, + "total_tokens": 3692, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 318.7863509654999 + } + }, + { + "tier": "work", + "session": 6, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/scheduler/sync.ts", + "answer": "src/scheduler/sync.ts", + "ok": true, + "raw": "[\n \"immer\",\n \"6200\",\n \"src/scheduler/sync.ts\",\n \"Tobias Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3021, + "completion_tokens": 671, + "total_tokens": 3692, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 318.7863509654999 + } + }, + { + "tier": "work", + "session": 6, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Tobias Khoury", + "answer": "Tobias Khoury", + "ok": true, + "raw": "[\n \"immer\",\n \"6200\",\n \"src/scheduler/sync.ts\",\n \"Tobias Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3021, + "completion_tokens": 671, + "total_tokens": 3692, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 318.7863509654999 + } + }, + { + "tier": "work", + "session": 6, + "type": "negation", + "q": "Was LEGACY_PINS enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\n \"immer\",\n \"6200\",\n \"src/scheduler/sync.ts\",\n \"Tobias Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3021, + "completion_tokens": 671, + "total_tokens": 3692, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 318.7863509654999 + } + }, + { + "tier": "work", + "session": 6, + "type": "unanswerable", + "q": "Which AWS region was the failover assigned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\n \"immer\",\n \"6200\",\n \"src/scheduler/sync.ts\",\n \"Tobias Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3021, + "completion_tokens": 671, + "total_tokens": 3692, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 318.7863509654999 + } + }, + { + "tier": "work", + "session": 7, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "xstate", + "answer": "state", + "ok": false, + "raw": "[\"state\", \"7070\", \"src/batcher/sync.ts\", \"pr1sa tanaka\", \"OFF\", \"UNKNOWN\"]", + "error": null, + "usage": { + "prompt_tokens": 3143, + "completion_tokens": 701, + "total_tokens": 3844, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 332.5045368075371 + } + }, + { + "tier": "work", + "session": 7, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "7870", + "answer": "7070", + "ok": false, + "raw": "[\"state\", \"7070\", \"src/batcher/sync.ts\", \"pr1sa tanaka\", \"OFF\", \"UNKNOWN\"]", + "error": null, + "usage": { + "prompt_tokens": 3143, + "completion_tokens": 701, + "total_tokens": 3844, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 332.5045368075371 + } + }, + { + "tier": "work", + "session": 7, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/batcher/sync.ts", + "answer": "src/batcher/sync.ts", + "ok": true, + "raw": "[\"state\", \"7070\", \"src/batcher/sync.ts\", \"pr1sa tanaka\", \"OFF\", \"UNKNOWN\"]", + "error": null, + "usage": { + "prompt_tokens": 3143, + "completion_tokens": 701, + "total_tokens": 3844, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 332.5045368075371 + } + }, + { + "tier": "work", + "session": 7, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Priya Tanaka", + "answer": "pr1sa tanaka", + "ok": false, + "raw": "[\"state\", \"7070\", \"src/batcher/sync.ts\", \"pr1sa tanaka\", \"OFF\", \"UNKNOWN\"]", + "error": null, + "usage": { + "prompt_tokens": 3143, + "completion_tokens": 701, + "total_tokens": 3844, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 332.5045368075371 + } + }, + { + "tier": "work", + "session": 7, + "type": "negation", + "q": "Was HOT_RELOAD_V2 enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\"state\", \"7070\", \"src/batcher/sync.ts\", \"pr1sa tanaka\", \"OFF\", \"UNKNOWN\"]", + "error": null, + "usage": { + "prompt_tokens": 3143, + "completion_tokens": 701, + "total_tokens": 3844, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 332.5045368075371 + } + }, + { + "tier": "work", + "session": 7, + "type": "unanswerable", + "q": "What was the Docker base image pinned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\"state\", \"7070\", \"src/batcher/sync.ts\", \"pr1sa tanaka\", \"OFF\", \"UNKNOWN\"]", + "error": null, + "usage": { + "prompt_tokens": 3143, + "completion_tokens": 701, + "total_tokens": 3844, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 332.5045368075371 + } + }, + { + "tier": "work", + "session": 8, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "jotai", + "answer": "jotai", + "ok": true, + "raw": "[\n \"jotai\",\n \"7850\",\n \"src/retry/io.ts\",\n \"Soren Petrov\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3077, + "completion_tokens": 594, + "total_tokens": 3671, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 298.6772604882717 + } + }, + { + "tier": "work", + "session": 8, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "7850", + "answer": "7850", + "ok": true, + "raw": "[\n \"jotai\",\n \"7850\",\n \"src/retry/io.ts\",\n \"Soren Petrov\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3077, + "completion_tokens": 594, + "total_tokens": 3671, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 298.6772604882717 + } + }, + { + "tier": "work", + "session": 8, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/retry/io.ts", + "answer": "src/retry/io.ts", + "ok": true, + "raw": "[\n \"jotai\",\n \"7850\",\n \"src/retry/io.ts\",\n \"Soren Petrov\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3077, + "completion_tokens": 594, + "total_tokens": 3671, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 298.6772604882717 + } + }, + { + "tier": "work", + "session": 8, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Soren Petrov", + "answer": "Soren Petrov", + "ok": true, + "raw": "[\n \"jotai\",\n \"7850\",\n \"src/retry/io.ts\",\n \"Soren Petrov\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3077, + "completion_tokens": 594, + "total_tokens": 3671, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 298.6772604882717 + } + }, + { + "tier": "work", + "session": 8, + "type": "negation", + "q": "Was ENABLE_SHARDING enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\n \"jotai\",\n \"7850\",\n \"src/retry/io.ts\",\n \"Soren Petrov\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3077, + "completion_tokens": 594, + "total_tokens": 3671, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 298.6772604882717 + } + }, + { + "tier": "work", + "session": 8, + "type": "unanswerable", + "q": "What port number was the staging proxy moved to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\n \"jotai\",\n \"7850\",\n \"src/retry/io.ts\",\n \"Soren Petrov\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 3077, + "completion_tokens": 594, + "total_tokens": 3671, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 298.6772604882717 + } + }, + { + "tier": "work", + "session": 9, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "redux-toolkit", + "answer": "redux-toolkit", + "ok": true, + "raw": "[\n \"redux-toolkit\",\n \"450ms\",\n \"src/cursor/sync.ts\",\n \"Ingrid Nakamura\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2972, + "completion_tokens": 705, + "total_tokens": 3677, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 326.67271262407303 + } + }, + { + "tier": "work", + "session": 9, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "4540", + "answer": "450ms", + "ok": false, + "raw": "[\n \"redux-toolkit\",\n \"450ms\",\n \"src/cursor/sync.ts\",\n \"Ingrid Nakamura\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2972, + "completion_tokens": 705, + "total_tokens": 3677, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 326.67271262407303 + } + }, + { + "tier": "work", + "session": 9, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/sync.ts", + "answer": "src/cursor/sync.ts", + "ok": true, + "raw": "[\n \"redux-toolkit\",\n \"450ms\",\n \"src/cursor/sync.ts\",\n \"Ingrid Nakamura\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2972, + "completion_tokens": 705, + "total_tokens": 3677, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 326.67271262407303 + } + }, + { + "tier": "work", + "session": 9, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Ingrid Nakamura", + "answer": "Ingrid Nakamura", + "ok": true, + "raw": "[\n \"redux-toolkit\",\n \"450ms\",\n \"src/cursor/sync.ts\",\n \"Ingrid Nakamura\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2972, + "completion_tokens": 705, + "total_tokens": 3677, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 326.67271262407303 + } + }, + { + "tier": "work", + "session": 9, + "type": "negation", + "q": "Was HOT_RELOAD_V2 enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\n \"redux-toolkit\",\n \"450ms\",\n \"src/cursor/sync.ts\",\n \"Ingrid Nakamura\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2972, + "completion_tokens": 705, + "total_tokens": 3677, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 326.67271262407303 + } + }, + { + "tier": "work", + "session": 9, + "type": "unanswerable", + "q": "What git tag was the hotfix released under?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\n \"redux-toolkit\",\n \"450ms\",\n \"src/cursor/sync.ts\",\n \"Ingrid Nakamura\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 2972, + "completion_tokens": 705, + "total_tokens": 3677, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 326.67271262407303 + } + }, + { + "tier": "work2", + "session": 0, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "nanostores", + "answer": "nanostores", + "ok": true, + "raw": "[\n \"nanostores\",\n \"7850\",\n \"src/mailbox/core.ts\",\n \"Riku Khorur\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7100, + "completion_tokens": 3487, + "total_tokens": 10587, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1304.854496717453 + } + }, + { + "tier": "work2", + "session": 0, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "7850", + "answer": "7850", + "ok": true, + "raw": "[\n \"nanostores\",\n \"7850\",\n \"src/mailbox/core.ts\",\n \"Riku Khorur\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7100, + "completion_tokens": 3487, + "total_tokens": 10587, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1304.854496717453 + } + }, + { + "tier": "work2", + "session": 0, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/mailbox/core.ts", + "answer": "src/mailbox/core.ts", + "ok": true, + "raw": "[\n \"nanostores\",\n \"7850\",\n \"src/mailbox/core.ts\",\n \"Riku Khorur\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7100, + "completion_tokens": 3487, + "total_tokens": 10587, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1304.854496717453 + } + }, + { + "tier": "work2", + "session": 0, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Aiko Khoury", + "answer": "Riku Khorur", + "ok": false, + "raw": "[\n \"nanostores\",\n \"7850\",\n \"src/mailbox/core.ts\",\n \"Riku Khorur\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7100, + "completion_tokens": 3487, + "total_tokens": 10587, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1304.854496717453 + } + }, + { + "tier": "work2", + "session": 0, + "type": "negation", + "q": "In PROD specifically, was LEGACY_PINS enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\n \"nanostores\",\n \"7850\",\n \"src/mailbox/core.ts\",\n \"Riku Khorur\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7100, + "completion_tokens": 3487, + "total_tokens": 10587, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1304.854496717453 + } + }, + { + "tier": "work2", + "session": 0, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\n \"nanostores\",\n \"7850\",\n \"src/mailbox/core.ts\",\n \"Riku Khorur\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7100, + "completion_tokens": 3487, + "total_tokens": 10587, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1304.854496717453 + } + }, + { + "tier": "work2", + "session": 1, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "immer", + "answer": "immer", + "ok": true, + "raw": "[\n \"immer\",\n \"130\",\n \"src/retro/core.ts\",\n \"Mara Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7377, + "completion_tokens": 1849, + "total_tokens": 9226, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 839.6772385239601 + } + }, + { + "tier": "work2", + "session": 1, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "1330", + "answer": "130", + "ok": false, + "raw": "[\n \"immer\",\n \"130\",\n \"src/retro/core.ts\",\n \"Mara Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7377, + "completion_tokens": 1849, + "total_tokens": 9226, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 839.6772385239601 + } + }, + { + "tier": "work2", + "session": 1, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/retry/core.ts", + "answer": "src/retro/core.ts", + "ok": false, + "raw": "[\n \"immer\",\n \"130\",\n \"src/retro/core.ts\",\n \"Mara Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7377, + "completion_tokens": 1849, + "total_tokens": 9226, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 839.6772385239601 + } + }, + { + "tier": "work2", + "session": 1, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Mara Khoury", + "answer": "Mara Khoury", + "ok": true, + "raw": "[\n \"immer\",\n \"130\",\n \"src/retro/core.ts\",\n \"Mara Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7377, + "completion_tokens": 1849, + "total_tokens": 9226, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 839.6772385239601 + } + }, + { + "tier": "work2", + "session": 1, + "type": "negation", + "q": "In PROD specifically, was USE_BROTLI enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\n \"immer\",\n \"130\",\n \"src/retro/core.ts\",\n \"Mara Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7377, + "completion_tokens": 1849, + "total_tokens": 9226, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 839.6772385239601 + } + }, + { + "tier": "work2", + "session": 1, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\n \"immer\",\n \"130\",\n \"src/retro/core.ts\",\n \"Mara Khoury\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7377, + "completion_tokens": 1849, + "total_tokens": 9226, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 839.6772385239601 + } + }, + { + "tier": "work2", + "session": 2, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "immer", + "answer": "immer", + "ok": true, + "raw": "[\n \"immer\",\n \"450\",\n \"src/autotd/core/t.ts\",\n \"Nadia Alvarez\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7071, + "completion_tokens": 604, + "total_tokens": 7675, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 464.97724333405495 + } + }, + { + "tier": "work2", + "session": 2, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "4570", + "answer": "450", + "ok": false, + "raw": "[\n \"immer\",\n \"450\",\n \"src/autotd/core/t.ts\",\n \"Nadia Alvarez\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7071, + "completion_tokens": 604, + "total_tokens": 7675, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 464.97724333405495 + } + }, + { + "tier": "work2", + "session": 2, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/quota/core.ts", + "answer": "src/autotd/core/t.ts", + "ok": false, + "raw": "[\n \"immer\",\n \"450\",\n \"src/autotd/core/t.ts\",\n \"Nadia Alvarez\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7071, + "completion_tokens": 604, + "total_tokens": 7675, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 464.97724333405495 + } + }, + { + "tier": "work2", + "session": 2, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Nadia Alvarez", + "answer": "Nadia Alvarez", + "ok": true, + "raw": "[\n \"immer\",\n \"450\",\n \"src/autotd/core/t.ts\",\n \"Nadia Alvarez\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7071, + "completion_tokens": 604, + "total_tokens": 7675, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 464.97724333405495 + } + }, + { + "tier": "work2", + "session": 2, + "type": "negation", + "q": "In PROD specifically, was ENABLE_SHARDING enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\n \"immer\",\n \"450\",\n \"src/autotd/core/t.ts\",\n \"Nadia Alvarez\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7071, + "completion_tokens": 604, + "total_tokens": 7675, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 464.97724333405495 + } + }, + { + "tier": "work2", + "session": 2, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\n \"immer\",\n \"450\",\n \"src/autotd/core/t.ts\",\n \"Nadia Alvarez\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7071, + "completion_tokens": 604, + "total_tokens": 7675, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 464.97724333405495 + } + }, + { + "tier": "work2", + "session": 3, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "redux-toolkit", + "answer": "redux-toolkit", + "ok": true, + "raw": "[\n \"redux-toolkit\",\n \"6270ms\",\n \"src/batcher/core.ts\",\n \"Farid Lindqvist\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6812, + "completion_tokens": 1097, + "total_tokens": 7909, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 597.7999921441078 + } + }, + { + "tier": "work2", + "session": 3, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "6270", + "answer": "6270ms", + "ok": true, + "raw": "[\n \"redux-toolkit\",\n \"6270ms\",\n \"src/batcher/core.ts\",\n \"Farid Lindqvist\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6812, + "completion_tokens": 1097, + "total_tokens": 7909, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 597.7999921441078 + } + }, + { + "tier": "work2", + "session": 3, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/batcher/core.ts", + "answer": "src/batcher/core.ts", + "ok": true, + "raw": "[\n \"redux-toolkit\",\n \"6270ms\",\n \"src/batcher/core.ts\",\n \"Farid Lindqvist\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6812, + "completion_tokens": 1097, + "total_tokens": 7909, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 597.7999921441078 + } + }, + { + "tier": "work2", + "session": 3, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Farid Lindqvist", + "answer": "Farid Lindqvist", + "ok": true, + "raw": "[\n \"redux-toolkit\",\n \"6270ms\",\n \"src/batcher/core.ts\",\n \"Farid Lindqvist\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6812, + "completion_tokens": 1097, + "total_tokens": 7909, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 597.7999921441078 + } + }, + { + "tier": "work2", + "session": 3, + "type": "negation", + "q": "In PROD specifically, was ASYNC_FSYNC enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\n \"redux-toolkit\",\n \"6270ms\",\n \"src/batcher/core.ts\",\n \"Farid Lindqvist\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6812, + "completion_tokens": 1097, + "total_tokens": 7909, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 597.7999921441078 + } + }, + { + "tier": "work2", + "session": 3, + "type": "unanswerable", + "q": "What was the Docker base image pinned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\n \"redux-toolkit\",\n \"6270ms\",\n \"src/batcher/core.ts\",\n \"Farid Lindqvist\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6812, + "completion_tokens": 1097, + "total_tokens": 7909, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 597.7999921441078 + } + }, + { + "tier": "work2", + "session": 4, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "zustand", + "answer": "zustand", + "ok": true, + "raw": "[\n \"zustand\",\n \"9120\",\n \"src/cursor/core.ts\",\n \"Soren Nakamura\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7775, + "completion_tokens": 1884, + "total_tokens": 9659, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 866.1408862173557 + } + }, + { + "tier": "work2", + "session": 4, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "9120", + "answer": "9120", + "ok": true, + "raw": "[\n \"zustand\",\n \"9120\",\n \"src/cursor/core.ts\",\n \"Soren Nakamura\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7775, + "completion_tokens": 1884, + "total_tokens": 9659, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 866.1408862173557 + } + }, + { + "tier": "work2", + "session": 4, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/cursor/core.ts", + "answer": "src/cursor/core.ts", + "ok": true, + "raw": "[\n \"zustand\",\n \"9120\",\n \"src/cursor/core.ts\",\n \"Soren Nakamura\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7775, + "completion_tokens": 1884, + "total_tokens": 9659, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 866.1408862173557 + } + }, + { + "tier": "work2", + "session": 4, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Soren Nakamura", + "answer": "Soren Nakamura", + "ok": true, + "raw": "[\n \"zustand\",\n \"9120\",\n \"src/cursor/core.ts\",\n \"Soren Nakamura\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7775, + "completion_tokens": 1884, + "total_tokens": 9659, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 866.1408862173557 + } + }, + { + "tier": "work2", + "session": 4, + "type": "negation", + "q": "In PROD specifically, was HOT_RELOAD_V2 enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\n \"zustand\",\n \"9120\",\n \"src/cursor/core.ts\",\n \"Soren Nakamura\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7775, + "completion_tokens": 1884, + "total_tokens": 9659, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 866.1408862173557 + } + }, + { + "tier": "work2", + "session": 4, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\n \"zustand\",\n \"9120\",\n \"src/cursor/core.ts\",\n \"Soren Nakamura\",\n \"OFF\",\n \"UNKNOWN\"\n]", + "error": null, + "usage": { + "prompt_tokens": 7775, + "completion_tokens": 1884, + "total_tokens": 9659, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 866.1408862173557 + } + }, + { + "tier": "work2", + "session": 5, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "immer", + "answer": "immer", + "ok": true, + "raw": "[\"immer\",\"789\",\"src/auto/core.ts\",\"Ingrid Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null, + "usage": { + "prompt_tokens": 6602, + "completion_tokens": 8192, + "total_tokens": 14794, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 2653.2090150415897 + } + }, + { + "tier": "work2", + "session": 5, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "7800", + "answer": "789", + "ok": false, + "raw": "[\"immer\",\"789\",\"src/auto/core.ts\",\"Ingrid Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null, + "usage": { + "prompt_tokens": 6602, + "completion_tokens": 8192, + "total_tokens": 14794, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 2653.2090150415897 + } + }, + { + "tier": "work2", + "session": 5, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/quota/core.ts", + "answer": "src/auto/core.ts", + "ok": false, + "raw": "[\"immer\",\"789\",\"src/auto/core.ts\",\"Ingrid Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null, + "usage": { + "prompt_tokens": 6602, + "completion_tokens": 8192, + "total_tokens": 14794, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 2653.2090150415897 + } + }, + { + "tier": "work2", + "session": 5, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Ingrid Costa", + "answer": "Ingrid Costa", + "ok": true, + "raw": "[\"immer\",\"789\",\"src/auto/core.ts\",\"Ingrid Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null, + "usage": { + "prompt_tokens": 6602, + "completion_tokens": 8192, + "total_tokens": 14794, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 2653.2090150415897 + } + }, + { + "tier": "work2", + "session": 5, + "type": "negation", + "q": "In PROD specifically, was USE_BROTLI enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\"immer\",\"789\",\"src/auto/core.ts\",\"Ingrid Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null, + "usage": { + "prompt_tokens": 6602, + "completion_tokens": 8192, + "total_tokens": 14794, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 2653.2090150415897 + } + }, + { + "tier": "work2", + "session": 5, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\"immer\",\"789\",\"src/auto/core.ts\",\"Ingrid Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null, + "usage": { + "prompt_tokens": 6602, + "completion_tokens": 8192, + "total_tokens": 14794, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 2653.2090150415897 + } + }, + { + "tier": "work3", + "session": 0, + "type": "final", + "q": "What is the FINAL (locked) value of BATCH_WINDOW_MS at the end of the session?", + "gold": "8400", + "answer": "9600", + "ok": false, + "raw": "[\"9600\",\"9600\",\"1\"]", + "error": null, + "usage": null + }, + { + "tier": "work3", + "session": 0, + "type": "first", + "q": "What was the FIRST value BATCH_WINDOW_MS was set to at the start?", + "gold": "9600", + "answer": "9600", + "ok": true, + "raw": "[\"9600\",\"9600\",\"1\"]", + "error": null, + "usage": null + }, + { + "tier": "work3", + "session": 0, + "type": "count", + "q": "How many distinct values was BATCH_WINDOW_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "1", + "ok": false, + "raw": "[\"9600\",\"9600\",\"1\"]", + "error": null, + "usage": null + }, + { + "tier": "work3", + "session": 1, + "type": "final", + "q": "What is the FINAL (locked) value of BATCH_WINDOW_MS at the end of the session?", + "gold": "1200", + "answer": "1200", + "ok": true, + "raw": "[\n \"1200\",\n \"5000\",\n \"3\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6789, + "completion_tokens": 6433, + "total_tokens": 13222, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 2149.149934142828 + } + }, + { + "tier": "work3", + "session": 1, + "type": "first", + "q": "What was the FIRST value BATCH_WINDOW_MS was set to at the start?", + "gold": "5400", + "answer": "5000", + "ok": false, + "raw": "[\n \"1200\",\n \"5000\",\n \"3\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6789, + "completion_tokens": 6433, + "total_tokens": 13222, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 2149.149934142828 + } + }, + { + "tier": "work3", + "session": 1, + "type": "count", + "q": "How many distinct values was BATCH_WINDOW_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "3", + "ok": true, + "raw": "[\n \"1200\",\n \"5000\",\n \"3\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6789, + "completion_tokens": 6433, + "total_tokens": 13222, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 2149.149934142828 + } + }, + { + "tier": "work3", + "session": 2, + "type": "final", + "q": "What is the FINAL (locked) value of FLUSH_INTERVAL_MS at the end of the session?", + "gold": "7200", + "answer": "7200", + "ok": true, + "raw": "[\n \"7200\",\n \"2400\",\n \"4\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6274, + "completion_tokens": 5674, + "total_tokens": 11948, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1907.2817648351192 + } + }, + { + "tier": "work3", + "session": 2, + "type": "first", + "q": "What was the FIRST value FLUSH_INTERVAL_MS was set to at the start?", + "gold": "2400", + "answer": "2400", + "ok": true, + "raw": "[\n \"7200\",\n \"2400\",\n \"4\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6274, + "completion_tokens": 5674, + "total_tokens": 11948, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1907.2817648351192 + } + }, + { + "tier": "work3", + "session": 2, + "type": "count", + "q": "How many distinct values was FLUSH_INTERVAL_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "4", + "ok": false, + "raw": "[\n \"7200\",\n \"2400\",\n \"4\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6274, + "completion_tokens": 5674, + "total_tokens": 11948, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1907.2817648351192 + } + }, + { + "tier": "work3", + "session": 3, + "type": "final", + "q": "What is the FINAL (locked) value of LEASE_TTL_MS at the end of the session?", + "gold": "7200", + "answer": "7200", + "ok": true, + "raw": "[\n \"7200\",\n \"2400\",\n \"3\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6848, + "completion_tokens": 631, + "total_tokens": 7479, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 463.70907735824585 + } + }, + { + "tier": "work3", + "session": 3, + "type": "first", + "q": "What was the FIRST value LEASE_TTL_MS was set to at the start?", + "gold": "2400", + "answer": "2400", + "ok": true, + "raw": "[\n \"7200\",\n \"2400\",\n \"3\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6848, + "completion_tokens": 631, + "total_tokens": 7479, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 463.70907735824585 + } + }, + { + "tier": "work3", + "session": 3, + "type": "count", + "q": "How many distinct values was LEASE_TTL_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "3", + "ok": true, + "raw": "[\n \"7200\",\n \"2400\",\n \"3\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6848, + "completion_tokens": 631, + "total_tokens": 7479, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 463.70907735824585 + } + }, + { + "tier": "work3", + "session": 4, + "type": "final", + "q": "What is the FINAL (locked) value of BATCH_WINDOW_MS at the end of the session?", + "gold": "4800", + "answer": "1800", + "ok": false, + "raw": "[\n \"1800\",\n \"7200\",\n \"2\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6990, + "completion_tokens": 1173, + "total_tokens": 8163, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 627.1908816695213 + } + }, + { + "tier": "work3", + "session": 4, + "type": "first", + "q": "What was the FIRST value BATCH_WINDOW_MS was set to at the start?", + "gold": "7200", + "answer": "7200", + "ok": true, + "raw": "[\n \"1800\",\n \"7200\",\n \"2\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6990, + "completion_tokens": 1173, + "total_tokens": 8163, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 627.1908816695213 + } + }, + { + "tier": "work3", + "session": 4, + "type": "count", + "q": "How many distinct values was BATCH_WINDOW_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "2", + "ok": false, + "raw": "[\n \"1800\",\n \"7200\",\n \"2\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6990, + "completion_tokens": 1173, + "total_tokens": 8163, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 627.1908816695213 + } + }, + { + "tier": "work3", + "session": 5, + "type": "final", + "q": "What is the FINAL (locked) value of BATCH_WINDOW_MS at the end of the session?", + "gold": "8400", + "answer": "3600", + "ok": false, + "raw": "[\n \"3600\",\n \"3600\",\n \"3\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6520, + "completion_tokens": 3360, + "total_tokens": 9880, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1244.1817789375782 + } + }, + { + "tier": "work3", + "session": 5, + "type": "first", + "q": "What was the FIRST value BATCH_WINDOW_MS was set to at the start?", + "gold": "3600", + "answer": "3600", + "ok": true, + "raw": "[\n \"3600\",\n \"3600\",\n \"3\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6520, + "completion_tokens": 3360, + "total_tokens": 9880, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1244.1817789375782 + } + }, + { + "tier": "work3", + "session": 5, + "type": "count", + "q": "How many distinct values was BATCH_WINDOW_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "3", + "ok": true, + "raw": "[\n \"3600\",\n \"3600\",\n \"3\"\n]", + "error": null, + "usage": { + "prompt_tokens": 6520, + "completion_tokens": 3360, + "total_tokens": 9880, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 1244.1817789375782 + } + } + ] +} \ No newline at end of file diff --git a/eval/qwen-profile/gist-recall.mjs b/eval/qwen-profile/gist-recall.mjs new file mode 100644 index 000000000..e124fb3c6 --- /dev/null +++ b/eval/qwen-profile/gist-recall.mjs @@ -0,0 +1,142 @@ +// Qwen 3.8 gist recall evaluation suite. +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { renderTextToPngs } from '../../dist/core/render.js'; +import { resolveGptProfile } from '../../dist/core/gpt-model-profiles.js'; +import { factSheetText } from '../../dist/core/factsheet.js'; +import { callQwen, resultFilename } from './qwen-client.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = join(HERE, '../gist-recall'); +const MODEL = process.env.MODEL || 'workers-ai/@cf/qwen/qwen3.8-27b'; +const profile = resolveGptProfile(MODEL); +const LIVE = process.env.LIVE === '1'; +const TIMEOUT = Number(process.env.TIMEOUT_MS || 180000); +const TIERS = [['work', 10], ['work2', 6], ['work3', 6]]; +const RESULT = join(HERE, resultFilename('gist-recall', MODEL)); + +function parse(s) { + if (!s) return null; + const a = s.indexOf('['); + const b = s.lastIndexOf(']'); + if (a >= 0 && b > a) { + try { + return JSON.parse(s.slice(a, b + 1)); + } catch {} + } + return null; +} + +function norm(s) { + return String(s ?? '').trim().toLowerCase().replace(/\s+/g, ' '); +} + +function correct(p, a) { + const x = norm(a); + const g = norm(p.gold); + if (p.type === 'unanswerable') return x === 'unknown'; + if (p.type === 'numeric') return new RegExp(`(?:^|\\D)${g}(?:\\D|$)`).test(x); + if (p.type === 'negation') return x.includes('off') && !x.includes('enabled'); + return x.includes(g); +} + +function writeProgress(rows) { + const answerable = rows.filter((r) => r.type !== 'unanswerable'); + const guards = rows.filter((r) => r.type === 'unanswerable'); + const state = rows.filter((r) => r.tier === 'work3'); + const done = (xs) => xs.filter((r) => !r.error); + + const out = { + generatedAt: new Date().toISOString(), + model: MODEL, + live: LIVE, + recipe: { cols: profile.stripCols, maxH: profile.maxHeightPx, style: profile.style, factsheet: true }, + answerable: { correct: done(answerable).filter((r) => r.ok).length, completed: done(answerable).length, n: answerable.length }, + state: { correct: done(state).filter((r) => r.ok).length, completed: done(state).length, n: state.length }, + unanswerable: { confabulated: done(guards).filter((r) => !r.ok).length, completed: done(guards).length, n: guards.length }, + rows, + }; + writeFileSync(RESULT, JSON.stringify(out, null, 2)); + return out; +} + +const existingRows = existsSync(RESULT) + ? JSON.parse(readFileSync(RESULT, 'utf8')).rows || [] + : []; +const completedSessions = new Set( + existingRows.filter((r) => !r.error && r.answer !== undefined).map((r) => `${r.tier}:${r.session}`) +); + +const rows = [...existingRows.filter((r) => completedSessions.has(`${r.tier}:${r.session}`))]; + +for (const [dir, n] of TIERS) { + const probes = JSON.parse(readFileSync(join(ROOT, dir, 'probes.json'), 'utf8')); + for (let sid = 0; sid < n; sid++) { + const key = `${dir}:${sid}`; + const ps = probes.filter((p) => p.session === sid); + if (completedSessions.has(key)) { + const existing = rows.filter((r) => r.tier === dir && r.session === sid); + const hits = existing.filter((r) => r.ok).length; + console.log(`${dir} s${sid}: ${hits}/${ps.length} (cached)`); + continue; + } + + const source = readFileSync(join(ROOT, dir, `s${sid}.txt`), 'utf8'); + const imgs = await renderTextToPngs(source, profile.stripCols, profile.style, profile.maxHeightPx); + const prompt = [ + 'Read all transcript images in order. Answer every numbered question.', + 'If the transcript does not contain an answer, use exactly UNKNOWN.', + 'Return only a JSON array of strings in question order.', + ...ps.map((p, i) => `${i + 1}. ${p.q}`), + ].join('\n'); + let response = { output: '', usage: null, error: null }; + if (LIVE) { + const content = imgs.map((im) => ({ + type: 'input_image', + image_url: `data:image/png;base64,${Buffer.from(im.png).toString('base64')}`, + })); + const fs = factSheetText(source, profile.factSheetFormat); + if (fs) content.push({ type: 'input_text', text: fs }); + content.push({ type: 'input_text', text: prompt }); + + try { + const r = await callQwen({ model: MODEL, content, maxOutputTokens: 8192, timeoutMs: TIMEOUT }); + response = { output: r.text, usage: r.usage, error: null }; + } catch (e) { + response = { output: '', usage: null, error: String(e.message || e) }; + } + } + const answers = parse(response.output) || []; + ps.forEach((p, i) => + rows.push({ + tier: dir, + session: sid, + ...p, + answer: String(answers[i] ?? ''), + ok: correct(p, answers[i]), + raw: response.output, + error: response.error || null, + usage: response.usage, + }) + ); + completedSessions.add(key); + if (LIVE) writeProgress(rows); + console.log(`${dir} s${sid}: ${ps.filter((p, i) => correct(p, answers[i])).length}/${ps.length}`); + } +} + +if (!LIVE) { + console.log('Dry run only; no receipt written'); + process.exit(0); +} + +const out = writeProgress(rows); +console.log('\nFinal Results:'); +console.log( + JSON.stringify( + { answerable: out.answerable, state: out.state, unanswerable: out.unanswerable }, + null, + 2 + ) +); diff --git a/eval/qwen-profile/novel-arithmetic-results.json b/eval/qwen-profile/novel-arithmetic-results.json new file mode 100644 index 000000000..2b2bc7b4a --- /dev/null +++ b/eval/qwen-profile/novel-arithmetic-results.json @@ -0,0 +1,4332 @@ +{ + "generatedAt": "2026-08-19T20:07:50.913Z", + "model": "workers-ai/@cf/qwen/qwen3.8-27b", + "live": true, + "n": 100, + "seed": 20260711, + "recipe": { + "cols": 152, + "maxH": 1932, + "style": { + "font": "spleen-5x8", + "cellWBonus": 0, + "cellHBonus": 0, + "aa": true, + "grid": false, + "gridCols": 0, + "colorCycle": false, + "markerScale": 1, + "markerRed": false, + "inkDilate": 0 + }, + "factsheet": true + }, + "textCorrect": 100, + "pureCorrect": 98, + "prodCorrect": 98, + "textPct": 100, + "purePct": 98, + "prodPct": 98, + "rows": [ + { + "i": 0, + "kind": 2, + "question": "A warehouse has 85 shelves, each holding 59 boxes, plus 987 loose boxes. How many boxes are there in total?", + "answer": 6002, + "imageTokens": 425, + "textGot": 6002, + "pureGot": 6002, + "prodGot": 6002, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 159, + "total_tokens": 264, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 50.54999852180481 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 237, + "total_tokens": 430, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 76.84090679883957 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 393, + "total_tokens": 595, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 122.59090477228165 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 1, + "kind": 2, + "question": "A warehouse has 35 shelves, each holding 82 boxes, plus 251 loose boxes. How many boxes are there in total?", + "answer": 3121, + "imageTokens": 425, + "textGot": 3121, + "pureGot": 3121, + "prodGot": 3121, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 164, + "total_tokens": 269, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 52.00454396009445 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 282, + "total_tokens": 475, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 89.93181547522545 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 229, + "total_tokens": 431, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 74.88181537389755 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 2, + "kind": 2, + "question": "A warehouse has 17 shelves, each holding 94 boxes, plus 955 loose boxes. How many boxes are there in total?", + "answer": 2553, + "imageTokens": 425, + "textGot": 2553, + "pureGot": 2553, + "prodGot": 2553, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 175, + "total_tokens": 280, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 55.20454382896423 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 399, + "total_tokens": 592, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 123.96817803382874 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 438, + "total_tokens": 640, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 135.68181344866753 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 3, + "kind": 2, + "question": "A warehouse has 88 shelves, each holding 97 boxes, plus 563 loose boxes. How many boxes are there in total?", + "answer": 9099, + "imageTokens": 425, + "textGot": 9099, + "pureGot": 9099, + "prodGot": 9099, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 155, + "total_tokens": 260, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 49.386362195014954 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 282, + "total_tokens": 475, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 89.93181547522545 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 370, + "total_tokens": 572, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 115.89999589323997 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 4, + "kind": 2, + "question": "A warehouse has 38 shelves, each holding 60 boxes, plus 587 loose boxes. How many boxes are there in total?", + "answer": 2867, + "imageTokens": 425, + "textGot": 2867, + "pureGot": 2867, + "prodGot": 2867, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 127, + "total_tokens": 232, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 41.24090790748596 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 354, + "total_tokens": 547, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 110.87726935744286 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 368, + "total_tokens": 570, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 115.31817772984505 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 5, + "kind": 2, + "question": "A warehouse has 60 shelves, each holding 16 boxes, plus 715 loose boxes. How many boxes are there in total?", + "answer": 1675, + "imageTokens": 425, + "textGot": 1675, + "pureGot": 1675, + "prodGot": 1675, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 126, + "total_tokens": 231, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 40.9499988257885 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 246, + "total_tokens": 439, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 79.45908853411674 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 274, + "total_tokens": 476, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 87.97272405028343 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 6, + "kind": 2, + "question": "A warehouse has 85 shelves, each holding 53 boxes, plus 655 loose boxes. How many boxes are there in total?", + "answer": 5160, + "imageTokens": 425, + "textGot": 5160, + "pureGot": 5160, + "prodGot": 5160, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 136, + "total_tokens": 241, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 43.85908964276314 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 402, + "total_tokens": 595, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 124.84090527892113 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 348, + "total_tokens": 550, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 109.49999609589577 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 7, + "kind": 2, + "question": "A warehouse has 29 shelves, each holding 99 boxes, plus 115 loose boxes. How many boxes are there in total?", + "answer": 2986, + "imageTokens": 425, + "textGot": 2986, + "pureGot": 2986, + "prodGot": 2986, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 152, + "total_tokens": 257, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 48.51363494992256 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 306, + "total_tokens": 499, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 96.91363343596458 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 243, + "total_tokens": 445, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 78.95454254746437 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 8, + "kind": 2, + "question": "A warehouse has 98 shelves, each holding 98 boxes, plus 183 loose boxes. How many boxes are there in total?", + "answer": 9787, + "imageTokens": 425, + "textGot": 9787, + "pureGot": 9787, + "prodGot": 9787, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 136, + "total_tokens": 241, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 43.85908964276314 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 365, + "total_tokens": 558, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 114.07726925611496 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 242, + "total_tokens": 444, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 78.66363343596458 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 9, + "kind": 2, + "question": "A warehouse has 46 shelves, each holding 97 boxes, plus 731 loose boxes. How many boxes are there in total?", + "answer": 5193, + "imageTokens": 425, + "textGot": 5193, + "pureGot": 5193, + "prodGot": 5193, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 177, + "total_tokens": 282, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 55.78636199235916 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 239, + "total_tokens": 432, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 77.42272499203682 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 384, + "total_tokens": 586, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 119.97272303700447 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 10, + "kind": 2, + "question": "A warehouse has 77 shelves, each holding 57 boxes, plus 139 loose boxes. How many boxes are there in total?", + "answer": 4528, + "imageTokens": 425, + "textGot": 4528, + "pureGot": 4528, + "prodGot": 4528, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 158, + "total_tokens": 263, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 50.259089440107346 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 422, + "total_tokens": 615, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 130.6590869128704 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 262, + "total_tokens": 464, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 84.48181506991386 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 11, + "kind": 2, + "question": "A warehouse has 67 shelves, each holding 93 boxes, plus 251 loose boxes. How many boxes are there in total?", + "answer": 6482, + "imageTokens": 425, + "textGot": 6482, + "pureGot": 6482, + "prodGot": 6482, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 128, + "total_tokens": 233, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 41.531816989183426 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 253, + "total_tokens": 446, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 81.495452105999 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 263, + "total_tokens": 465, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 84.77272415161133 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 12, + "kind": 2, + "question": "A warehouse has 52 shelves, each holding 42 boxes, plus 239 loose boxes. How many boxes are there in total?", + "answer": 2423, + "imageTokens": 425, + "textGot": 2423, + "pureGot": 2423, + "prodGot": 2423, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 112, + "total_tokens": 217, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 36.877271682024 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 224, + "total_tokens": 417, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 73.05908873677254 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 207, + "total_tokens": 409, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 68.48181557655334 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 13, + "kind": 2, + "question": "A warehouse has 19 shelves, each holding 62 boxes, plus 511 loose boxes. How many boxes are there in total?", + "answer": 1689, + "imageTokens": 425, + "textGot": 1689, + "pureGot": 1689, + "prodGot": 1689, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 142, + "total_tokens": 247, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 45.60454413294792 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 347, + "total_tokens": 540, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 108.84090578556061 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 214, + "total_tokens": 416, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 70.51817914843559 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 14, + "kind": 2, + "question": "A warehouse has 88 shelves, each holding 33 boxes, plus 427 loose boxes. How many boxes are there in total?", + "answer": 3331, + "imageTokens": 425, + "textGot": 3331, + "pureGot": 3265, + "prodGot": 3265, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 137, + "total_tokens": 242, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 44.1499987244606 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 337, + "total_tokens": 530, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 105.93181496858597 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 412, + "total_tokens": 614, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 128.11817732453346 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 15, + "kind": 2, + "question": "A warehouse has 42 shelves, each holding 58 boxes, plus 971 loose boxes. How many boxes are there in total?", + "answer": 3407, + "imageTokens": 425, + "textGot": 3407, + "pureGot": 3407, + "prodGot": 3407, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 167, + "total_tokens": 272, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 52.877271205186844 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 368, + "total_tokens": 561, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 114.94999650120735 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 231, + "total_tokens": 433, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 75.46363353729248 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 16, + "kind": 2, + "question": "A warehouse has 70 shelves, each holding 81 boxes, plus 211 loose boxes. How many boxes are there in total?", + "answer": 5881, + "imageTokens": 425, + "textGot": 5881, + "pureGot": 5881, + "prodGot": 5881, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 96, + "total_tokens": 201, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 32.22272637486458 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 251, + "total_tokens": 444, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 80.91363394260406 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 385, + "total_tokens": 587, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 120.26363211870193 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 17, + "kind": 2, + "question": "A warehouse has 12 shelves, each holding 56 boxes, plus 155 loose boxes. How many boxes are there in total?", + "answer": 827, + "imageTokens": 425, + "textGot": 827, + "pureGot": 827, + "prodGot": 827, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 129, + "total_tokens": 234, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 41.82272607088089 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 241, + "total_tokens": 434, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 78.00454312562943 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 222, + "total_tokens": 424, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 72.8454518020153 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 18, + "kind": 2, + "question": "A warehouse has 16 shelves, each holding 78 boxes, plus 251 loose boxes. How many boxes are there in total?", + "answer": 1499, + "imageTokens": 425, + "textGot": 1499, + "pureGot": 1499, + "prodGot": 1499, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 152, + "total_tokens": 257, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 48.51363494992256 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 237, + "total_tokens": 430, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 76.84090679883957 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 304, + "total_tokens": 506, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 96.69999650120735 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 19, + "kind": 2, + "question": "A warehouse has 91 shelves, each holding 71 boxes, plus 139 loose boxes. How many boxes are there in total?", + "answer": 6600, + "imageTokens": 425, + "textGot": 6600, + "pureGot": 6600, + "prodGot": 6600, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 168, + "total_tokens": 273, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 53.168180257081985 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 237, + "total_tokens": 430, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 76.84090685844421 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 249, + "total_tokens": 451, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 80.69999700784683 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 20, + "kind": 2, + "question": "A warehouse has 33 shelves, each holding 21 boxes, plus 903 loose boxes. How many boxes are there in total?", + "answer": 1596, + "imageTokens": 425, + "textGot": 1596, + "pureGot": 1596, + "prodGot": 1676, + "textOk": true, + "pureOk": true, + "prodOk": false, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 153, + "total_tokens": 258, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 48.80454409122467 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 326, + "total_tokens": 519, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 102.73181506991386 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 389, + "total_tokens": 591, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 121.42726844549179 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 21, + "kind": 2, + "question": "A warehouse has 21 shelves, each holding 14 boxes, plus 207 loose boxes. How many boxes are there in total?", + "answer": 501, + "imageTokens": 425, + "textGot": 501, + "pureGot": 501, + "prodGot": 501, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 129, + "total_tokens": 234, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 41.82272607088089 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 273, + "total_tokens": 466, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 87.31363373994827 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 311, + "total_tokens": 513, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 98.7363600730896 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 22, + "kind": 2, + "question": "A warehouse has 48 shelves, each holding 21 boxes, plus 419 loose boxes. How many boxes are there in total?", + "answer": 1427, + "imageTokens": 425, + "textGot": 1427, + "pureGot": 1427, + "prodGot": 1427, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 137, + "total_tokens": 242, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 44.1499987244606 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 215, + "total_tokens": 408, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 70.44090700149536 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 240, + "total_tokens": 442, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 78.08181527256966 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 23, + "kind": 2, + "question": "A warehouse has 68 shelves, each holding 50 boxes, plus 443 loose boxes. How many boxes are there in total?", + "answer": 3843, + "imageTokens": 425, + "textGot": 3843, + "pureGot": 3843, + "prodGot": 3843, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 139, + "total_tokens": 244, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 44.73181688785553 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 261, + "total_tokens": 454, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 83.8227247595787 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 272, + "total_tokens": 474, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 87.3909058868885 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 24, + "kind": 2, + "question": "A warehouse has 67 shelves, each holding 50 boxes, plus 211 loose boxes. How many boxes are there in total?", + "answer": 3561, + "imageTokens": 425, + "textGot": 3561, + "pureGot": 3561, + "prodGot": 3561, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 123, + "total_tokens": 228, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 40.077271580696106 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 237, + "total_tokens": 430, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 76.84090679883957 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 207, + "total_tokens": 409, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 68.48181557655334 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 25, + "kind": 2, + "question": "A warehouse has 45 shelves, each holding 27 boxes, plus 411 loose boxes. How many boxes are there in total?", + "answer": 1626, + "imageTokens": 425, + "textGot": 1626, + "pureGot": 1626, + "prodGot": 1626, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 162, + "total_tokens": 267, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 51.4227257668972 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 307, + "total_tokens": 500, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 97.20454251766205 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 341, + "total_tokens": 543, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 107.46363252401352 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 26, + "kind": 2, + "question": "A warehouse has 79 shelves, each holding 55 boxes, plus 563 loose boxes. How many boxes are there in total?", + "answer": 4908, + "imageTokens": 425, + "textGot": 4908, + "pureGot": 4908, + "prodGot": 4908, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 144, + "total_tokens": 249, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 46.18636229634285 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 318, + "total_tokens": 511, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 100.40454241633415 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 230, + "total_tokens": 432, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 75.17272448539734 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 27, + "kind": 2, + "question": "A warehouse has 72 shelves, each holding 73 boxes, plus 951 loose boxes. How many boxes are there in total?", + "answer": 6207, + "imageTokens": 425, + "textGot": 6207, + "pureGot": 6207, + "prodGot": 6207, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 136, + "total_tokens": 241, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 43.85908964276314 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 263, + "total_tokens": 456, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 84.40454292297363 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 277, + "total_tokens": 479, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 88.84545129537582 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 28, + "kind": 2, + "question": "A warehouse has 91 shelves, each holding 78 boxes, plus 395 loose boxes. How many boxes are there in total?", + "answer": 7493, + "imageTokens": 425, + "textGot": 7493, + "pureGot": 7493, + "prodGot": 7493, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 158, + "total_tokens": 263, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 50.259089440107346 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 326, + "total_tokens": 519, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 102.73181506991386 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 469, + "total_tokens": 671, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 144.6999949812889 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 29, + "kind": 2, + "question": "A warehouse has 39 shelves, each holding 40 boxes, plus 475 loose boxes. How many boxes are there in total?", + "answer": 2035, + "imageTokens": 425, + "textGot": 2035, + "pureGot": 2035, + "prodGot": 2035, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 107, + "total_tokens": 212, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 35.42272627353668 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 219, + "total_tokens": 412, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 71.60454332828522 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 245, + "total_tokens": 447, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 79.53636068105698 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 30, + "kind": 2, + "question": "A warehouse has 66 shelves, each holding 32 boxes, plus 803 loose boxes. How many boxes are there in total?", + "answer": 2915, + "imageTokens": 425, + "textGot": 2915, + "pureGot": 2915, + "prodGot": 2915, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 157, + "total_tokens": 262, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 49.96818035840988 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 297, + "total_tokens": 490, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 94.29545170068741 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 224, + "total_tokens": 426, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 73.42726996541023 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 31, + "kind": 2, + "question": "A warehouse has 91 shelves, each holding 30 boxes, plus 575 loose boxes. How many boxes are there in total?", + "answer": 3305, + "imageTokens": 425, + "textGot": 3305, + "pureGot": 3305, + "prodGot": 3305, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 92, + "total_tokens": 197, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 31.059090048074722 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 303, + "total_tokens": 496, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 96.04090619087219 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 335, + "total_tokens": 537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 105.71817803382874 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 32, + "kind": 2, + "question": "A warehouse has 42 shelves, each holding 59 boxes, plus 327 loose boxes. How many boxes are there in total?", + "answer": 2805, + "imageTokens": 425, + "textGot": 2805, + "pureGot": 2805, + "prodGot": 2805, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 144, + "total_tokens": 249, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 46.18636229634285 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 243, + "total_tokens": 436, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 78.58636128902435 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 271, + "total_tokens": 473, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 87.09999680519104 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 33, + "kind": 2, + "question": "A warehouse has 89 shelves, each holding 80 boxes, plus 523 loose boxes. How many boxes are there in total?", + "answer": 7643, + "imageTokens": 425, + "textGot": 7643, + "pureGot": 7643, + "prodGot": 7643, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 87, + "total_tokens": 192, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 29.604544639587402 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 265, + "total_tokens": 458, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 84.98636108636856 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 352, + "total_tokens": 554, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 110.66363242268562 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 34, + "kind": 2, + "question": "A warehouse has 67 shelves, each holding 98 boxes, plus 503 loose boxes. How many boxes are there in total?", + "answer": 7069, + "imageTokens": 425, + "textGot": 7069, + "pureGot": 7149, + "prodGot": 7069, + "textOk": true, + "pureOk": false, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 148, + "total_tokens": 253, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 47.349998623132706 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 267, + "total_tokens": 460, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 85.56817924976349 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 271, + "total_tokens": 473, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 87.09999680519104 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 35, + "kind": 2, + "question": "A warehouse has 43 shelves, each holding 33 boxes, plus 911 loose boxes. How many boxes are there in total?", + "answer": 2330, + "imageTokens": 425, + "textGot": 2330, + "pureGot": 2330, + "prodGot": 2330, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 144, + "total_tokens": 249, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 46.18636229634285 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 282, + "total_tokens": 475, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 89.93181547522545 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 296, + "total_tokens": 498, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 94.37272384762764 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 36, + "kind": 2, + "question": "A warehouse has 45 shelves, each holding 85 boxes, plus 651 loose boxes. How many boxes are there in total?", + "answer": 4476, + "imageTokens": 425, + "textGot": 4476, + "pureGot": 4476, + "prodGot": 4476, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 151, + "total_tokens": 256, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 48.2227258682251 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 346, + "total_tokens": 539, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 108.54999670386314 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 231, + "total_tokens": 433, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 75.46363353729248 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 37, + "kind": 2, + "question": "A warehouse has 55 shelves, each holding 44 boxes, plus 647 loose boxes. How many boxes are there in total?", + "answer": 3067, + "imageTokens": 425, + "textGot": 3067, + "pureGot": 3067, + "prodGot": 3067, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 146, + "total_tokens": 251, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 46.76818045973778 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 234, + "total_tokens": 427, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 75.96817955374718 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 232, + "total_tokens": 434, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 75.75454261898994 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 38, + "kind": 2, + "question": "A warehouse has 20 shelves, each holding 28 boxes, plus 523 loose boxes. How many boxes are there in total?", + "answer": 1083, + "imageTokens": 425, + "textGot": 1083, + "pureGot": 1083, + "prodGot": 1083, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 156, + "total_tokens": 261, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 49.67727127671242 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 293, + "total_tokens": 486, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 93.13181537389755 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 529, + "total_tokens": 731, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 162.15453988313675 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 39, + "kind": 2, + "question": "A warehouse has 83 shelves, each holding 22 boxes, plus 911 loose boxes. How many boxes are there in total?", + "answer": 2737, + "imageTokens": 425, + "textGot": 2737, + "pureGot": 2737, + "prodGot": 2737, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 174, + "total_tokens": 279, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 54.91363474726677 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 201, + "total_tokens": 394, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 66.36817985773087 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 680, + "total_tokens": 882, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 206.0818112194538 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 40, + "kind": 2, + "question": "A warehouse has 59 shelves, each holding 87 boxes, plus 427 loose boxes. How many boxes are there in total?", + "answer": 5560, + "imageTokens": 425, + "textGot": 5560, + "pureGot": 5560, + "prodGot": 5560, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 163, + "total_tokens": 268, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 51.713634848594666 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 247, + "total_tokens": 440, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 79.74999761581421 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 289, + "total_tokens": 491, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 92.33636027574539 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 41, + "kind": 2, + "question": "A warehouse has 55 shelves, each holding 63 boxes, plus 563 loose boxes. How many boxes are there in total?", + "answer": 4028, + "imageTokens": 425, + "textGot": 4028, + "pureGot": 4028, + "prodGot": 4028, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 146, + "total_tokens": 251, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 46.76818045973778 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 340, + "total_tokens": 533, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 106.80454221367836 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 257, + "total_tokens": 459, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 83.02726966142654 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 42, + "kind": 2, + "question": "A warehouse has 91 shelves, each holding 42 boxes, plus 175 loose boxes. How many boxes are there in total?", + "answer": 3997, + "imageTokens": 425, + "textGot": 3997, + "pureGot": 3997, + "prodGot": 3997, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 157, + "total_tokens": 262, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 49.96818035840988 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 286, + "total_tokens": 479, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 91.0954518020153 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 454, + "total_tokens": 656, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 140.33635875582695 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 43, + "kind": 2, + "question": "A warehouse has 25 shelves, each holding 87 boxes, plus 623 loose boxes. How many boxes are there in total?", + "answer": 2798, + "imageTokens": 425, + "textGot": 2798, + "pureGot": 2798, + "prodGot": 2798, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 143, + "total_tokens": 248, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 45.895453214645386 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 283, + "total_tokens": 476, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 90.22272455692291 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 254, + "total_tokens": 456, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 82.15454241633415 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 44, + "kind": 2, + "question": "A warehouse has 93 shelves, each holding 61 boxes, plus 111 loose boxes. How many boxes are there in total?", + "answer": 5784, + "imageTokens": 425, + "textGot": 5784, + "pureGot": 5784, + "prodGot": 5784, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 153, + "total_tokens": 258, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 48.804544031620026 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 318, + "total_tokens": 511, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 100.40454241633415 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 356, + "total_tokens": 558, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 111.8272687792778 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 45, + "kind": 2, + "question": "A warehouse has 63 shelves, each holding 65 boxes, plus 479 loose boxes. How many boxes are there in total?", + "answer": 4574, + "imageTokens": 425, + "textGot": 4574, + "pureGot": 4574, + "prodGot": 4574, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 128, + "total_tokens": 233, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 41.531816989183426 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 320, + "total_tokens": 513, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 100.98636057972908 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 208, + "total_tokens": 410, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 68.77272465825081 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 46, + "kind": 2, + "question": "A warehouse has 54 shelves, each holding 95 boxes, plus 687 loose boxes. How many boxes are there in total?", + "answer": 5817, + "imageTokens": 425, + "textGot": 5817, + "pureGot": 5817, + "prodGot": 5817, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 159, + "total_tokens": 264, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 50.54999852180481 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 337, + "total_tokens": 530, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 105.93181496858597 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 415, + "total_tokens": 617, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 128.99090456962585 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 47, + "kind": 2, + "question": "A warehouse has 48 shelves, each holding 29 boxes, plus 571 loose boxes. How many boxes are there in total?", + "answer": 1963, + "imageTokens": 425, + "textGot": 1963, + "pureGot": 1963, + "prodGot": 1963, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 110, + "total_tokens": 215, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 36.295453518629074 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 319, + "total_tokens": 512, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 100.69545149803162 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 403, + "total_tokens": 605, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 125.49999558925629 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 48, + "kind": 2, + "question": "A warehouse has 71 shelves, each holding 37 boxes, plus 587 loose boxes. How many boxes are there in total?", + "answer": 3214, + "imageTokens": 425, + "textGot": 3214, + "pureGot": 3214, + "prodGot": 3214, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 164, + "total_tokens": 269, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 52.00454393029213 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 274, + "total_tokens": 467, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 87.60454282164574 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 207, + "total_tokens": 409, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 68.48181557655334 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 49, + "kind": 2, + "question": "A warehouse has 41 shelves, each holding 50 boxes, plus 335 loose boxes. How many boxes are there in total?", + "answer": 2385, + "imageTokens": 425, + "textGot": 2385, + "pureGot": 2385, + "prodGot": 2385, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 99, + "total_tokens": 204, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 33.09545361995697 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 353, + "total_tokens": 546, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 110.58636027574539 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 259, + "total_tokens": 461, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 83.60908782482147 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 50, + "kind": 2, + "question": "A warehouse has 39 shelves, each holding 24 boxes, plus 843 loose boxes. How many boxes are there in total?", + "answer": 1779, + "imageTokens": 425, + "textGot": 1779, + "pureGot": 1779, + "prodGot": 1779, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 148, + "total_tokens": 253, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 47.349998623132706 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 282, + "total_tokens": 475, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 89.93181547522545 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 233, + "total_tokens": 435, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 76.04545170068741 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 51, + "kind": 2, + "question": "A warehouse has 28 shelves, each holding 39 boxes, plus 787 loose boxes. How many boxes are there in total?", + "answer": 1879, + "imageTokens": 425, + "textGot": 1879, + "pureGot": 1879, + "prodGot": 1879, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 151, + "total_tokens": 256, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 48.2227258682251 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 219, + "total_tokens": 412, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 71.60454332828522 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 275, + "total_tokens": 477, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 88.2636331319809 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 52, + "kind": 2, + "question": "A warehouse has 97 shelves, each holding 38 boxes, plus 539 loose boxes. How many boxes are there in total?", + "answer": 4225, + "imageTokens": 425, + "textGot": 4225, + "pureGot": 4225, + "prodGot": 4225, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 132, + "total_tokens": 237, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 42.69545331597328 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 470, + "total_tokens": 663, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 144.62272283434868 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 416, + "total_tokens": 618, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 129.28181365132332 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 53, + "kind": 2, + "question": "A warehouse has 67 shelves, each holding 55 boxes, plus 671 loose boxes. How many boxes are there in total?", + "answer": 4356, + "imageTokens": 425, + "textGot": 4356, + "pureGot": 4356, + "prodGot": 4356, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 149, + "total_tokens": 254, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 47.64090770483017 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 343, + "total_tokens": 536, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 107.67726945877075 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 381, + "total_tokens": 583, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 119.09999579191208 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 54, + "kind": 2, + "question": "A warehouse has 89 shelves, each holding 43 boxes, plus 759 loose boxes. How many boxes are there in total?", + "answer": 4586, + "imageTokens": 425, + "textGot": 4586, + "pureGot": 4586, + "prodGot": 4586, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 240, + "total_tokens": 345, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 74.11363413929939 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 281, + "total_tokens": 474, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 89.64090639352798 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 343, + "total_tokens": 545, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 108.04545068740845 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 55, + "kind": 2, + "question": "A warehouse has 68 shelves, each holding 80 boxes, plus 675 loose boxes. How many boxes are there in total?", + "answer": 6115, + "imageTokens": 425, + "textGot": 6115, + "pureGot": 6115, + "prodGot": 6115, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 140, + "total_tokens": 245, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 45.022725969552994 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 487, + "total_tokens": 680, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 149.5681772530079 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 373, + "total_tokens": 575, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 116.77272313833237 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 56, + "kind": 2, + "question": "A warehouse has 95 shelves, each holding 92 boxes, plus 503 loose boxes. How many boxes are there in total?", + "answer": 9243, + "imageTokens": 425, + "textGot": 9243, + "pureGot": 9243, + "prodGot": 9243, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 123, + "total_tokens": 228, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 40.077271580696106 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 334, + "total_tokens": 527, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 105.05908772349358 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 409, + "total_tokens": 611, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 127.24545007944107 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 57, + "kind": 2, + "question": "A warehouse has 23 shelves, each holding 28 boxes, plus 483 loose boxes. How many boxes are there in total?", + "answer": 1127, + "imageTokens": 425, + "textGot": 1127, + "pureGot": 1127, + "prodGot": 1127, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 135, + "total_tokens": 240, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 43.568180561065674 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 328, + "total_tokens": 521, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 103.31363323330879 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 334, + "total_tokens": 536, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 105.42726895213127 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 58, + "kind": 2, + "question": "A warehouse has 65 shelves, each holding 39 boxes, plus 327 loose boxes. How many boxes are there in total?", + "answer": 2862, + "imageTokens": 425, + "textGot": 2862, + "pureGot": 2862, + "prodGot": 2862, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 124, + "total_tokens": 229, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 40.36818066239357 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 241, + "total_tokens": 434, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 78.00454312562943 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 368, + "total_tokens": 570, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 115.31817772984505 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 59, + "kind": 2, + "question": "A warehouse has 50 shelves, each holding 30 boxes, plus 883 loose boxes. How many boxes are there in total?", + "answer": 2383, + "imageTokens": 425, + "textGot": 2383, + "pureGot": 2383, + "prodGot": 2383, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 129, + "total_tokens": 234, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 41.82272607088089 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 262, + "total_tokens": 455, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 84.11363384127617 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 215, + "total_tokens": 417, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 70.80908823013306 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 60, + "kind": 2, + "question": "A warehouse has 75 shelves, each holding 16 boxes, plus 411 loose boxes. How many boxes are there in total?", + "answer": 1611, + "imageTokens": 425, + "textGot": 1611, + "pureGot": 1611, + "prodGot": 1611, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 138, + "total_tokens": 243, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 44.440907806158066 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 253, + "total_tokens": 446, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 81.495452105999 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 399, + "total_tokens": 601, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 124.33635926246643 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 61, + "kind": 2, + "question": "A warehouse has 36 shelves, each holding 58 boxes, plus 167 loose boxes. How many boxes are there in total?", + "answer": 2255, + "imageTokens": 425, + "textGot": 2255, + "pureGot": 2255, + "prodGot": 2255, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 166, + "total_tokens": 271, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 52.58636209368706 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 330, + "total_tokens": 523, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 103.89545139670372 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 242, + "total_tokens": 444, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 78.66363343596458 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 62, + "kind": 2, + "question": "A warehouse has 90 shelves, each holding 73 boxes, plus 775 loose boxes. How many boxes are there in total?", + "answer": 7345, + "imageTokens": 425, + "textGot": 7345, + "pureGot": 7345, + "prodGot": 7345, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 98, + "total_tokens": 203, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 32.804544538259506 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 264, + "total_tokens": 457, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 84.6954520046711 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 298, + "total_tokens": 500, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 94.95454201102257 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 63, + "kind": 2, + "question": "A warehouse has 78 shelves, each holding 48 boxes, plus 535 loose boxes. How many boxes are there in total?", + "answer": 4279, + "imageTokens": 425, + "textGot": 4279, + "pureGot": 4279, + "prodGot": 4279, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 126, + "total_tokens": 231, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 40.9499988257885 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 346, + "total_tokens": 539, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 108.54999676346779 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 238, + "total_tokens": 440, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 77.49999710917473 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 64, + "kind": 2, + "question": "A warehouse has 83 shelves, each holding 93 boxes, plus 771 loose boxes. How many boxes are there in total?", + "answer": 8490, + "imageTokens": 425, + "textGot": 8490, + "pureGot": 8490, + "prodGot": 8490, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 128, + "total_tokens": 233, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 41.531816989183426 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 258, + "total_tokens": 451, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 82.94999751448631 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 236, + "total_tokens": 438, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 76.9181789457798 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 65, + "kind": 2, + "question": "A warehouse has 31 shelves, each holding 30 boxes, plus 691 loose boxes. How many boxes are there in total?", + "answer": 1621, + "imageTokens": 425, + "textGot": 1621, + "pureGot": 1621, + "prodGot": 1621, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 132, + "total_tokens": 237, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 42.69545328617096 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 430, + "total_tokens": 623, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 132.98635956645012 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 261, + "total_tokens": 463, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 84.1909059882164 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 66, + "kind": 2, + "question": "A warehouse has 56 shelves, each holding 28 boxes, plus 619 loose boxes. How many boxes are there in total?", + "answer": 2187, + "imageTokens": 425, + "textGot": 2187, + "pureGot": 2187, + "prodGot": 2187, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 147, + "total_tokens": 252, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 47.05908954143524 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 346, + "total_tokens": 539, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 108.54999670386314 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 244, + "total_tokens": 446, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 79.24545162916183 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 67, + "kind": 2, + "question": "A warehouse has 76 shelves, each holding 46 boxes, plus 695 loose boxes. How many boxes are there in total?", + "answer": 4191, + "imageTokens": 425, + "textGot": 4191, + "pureGot": 4191, + "prodGot": 4191, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 256, + "total_tokens": 361, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 78.76817950606346 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 318, + "total_tokens": 511, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 100.40454241633415 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 380, + "total_tokens": 582, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 118.80908671021461 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 68, + "kind": 2, + "question": "A warehouse has 22 shelves, each holding 99 boxes, plus 107 loose boxes. How many boxes are there in total?", + "answer": 2285, + "imageTokens": 425, + "textGot": 2285, + "pureGot": 2285, + "prodGot": 2285, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 155, + "total_tokens": 260, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 49.386362195014954 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 281, + "total_tokens": 474, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 89.64090639352798 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 228, + "total_tokens": 430, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 74.59090632200241 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 69, + "kind": 2, + "question": "A warehouse has 84 shelves, each holding 75 boxes, plus 343 loose boxes. How many boxes are there in total?", + "answer": 6643, + "imageTokens": 425, + "textGot": 6643, + "pureGot": 6643, + "prodGot": 6643, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 174, + "total_tokens": 279, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 54.91363474726677 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 493, + "total_tokens": 686, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 151.31363171339035 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 382, + "total_tokens": 584, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 119.39090487360954 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 70, + "kind": 2, + "question": "A warehouse has 66 shelves, each holding 22 boxes, plus 131 loose boxes. How many boxes are there in total?", + "answer": 1583, + "imageTokens": 425, + "textGot": 1583, + "pureGot": 1583, + "prodGot": 1583, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 130, + "total_tokens": 235, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 42.11363512277603 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 352, + "total_tokens": 545, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 110.29545119404793 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 218, + "total_tokens": 420, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 71.68181547522545 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 71, + "kind": 2, + "question": "A warehouse has 48 shelves, each holding 99 boxes, plus 735 loose boxes. How many boxes are there in total?", + "answer": 5487, + "imageTokens": 425, + "textGot": 5487, + "pureGot": 5487, + "prodGot": 5487, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 201, + "total_tokens": 306, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 62.76817998290062 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 283, + "total_tokens": 476, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 90.22272458672523 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 213, + "total_tokens": 415, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 70.22727006673813 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 72, + "kind": 2, + "question": "A warehouse has 95 shelves, each holding 71 boxes, plus 119 loose boxes. How many boxes are there in total?", + "answer": 6864, + "imageTokens": 425, + "textGot": 6864, + "pureGot": 6864, + "prodGot": 6864, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 151, + "total_tokens": 256, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 48.2227258682251 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 237, + "total_tokens": 430, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 76.84090679883957 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 366, + "total_tokens": 568, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 114.73635956645012 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 73, + "kind": 2, + "question": "A warehouse has 18 shelves, each holding 47 boxes, plus 583 loose boxes. How many boxes are there in total?", + "answer": 1429, + "imageTokens": 425, + "textGot": 1429, + "pureGot": 1429, + "prodGot": 1429, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 119, + "total_tokens": 224, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 38.913635313510895 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 313, + "total_tokens": 506, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 98.94999700784683 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 320, + "total_tokens": 522, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 101.35454180836678 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 74, + "kind": 2, + "question": "A warehouse has 31 shelves, each holding 12 boxes, plus 579 loose boxes. How many boxes are there in total?", + "answer": 951, + "imageTokens": 425, + "textGot": 951, + "pureGot": 951, + "prodGot": 951, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 92, + "total_tokens": 197, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 31.059090048074722 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 310, + "total_tokens": 503, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 98.07726976275444 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 184, + "total_tokens": 386, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 61.79090669751167 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 75, + "kind": 2, + "question": "A warehouse has 36 shelves, each holding 98 boxes, plus 379 loose boxes. How many boxes are there in total?", + "answer": 3907, + "imageTokens": 425, + "textGot": 3907, + "pureGot": 3907, + "prodGot": 3907, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 150, + "total_tokens": 255, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 47.931816786527634 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 245, + "total_tokens": 438, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 79.16817945241928 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 434, + "total_tokens": 636, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 134.51817712187767 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 76, + "kind": 2, + "question": "A warehouse has 88 shelves, each holding 91 boxes, plus 735 loose boxes. How many boxes are there in total?", + "answer": 8743, + "imageTokens": 425, + "textGot": 8743, + "pureGot": 8743, + "prodGot": 8743, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 170, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 53.74999842047691 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 395, + "total_tokens": 588, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 122.80454170703888 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 234, + "total_tokens": 436, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 76.33636078238487 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 77, + "kind": 2, + "question": "A warehouse has 24 shelves, each holding 64 boxes, plus 715 loose boxes. How many boxes are there in total?", + "answer": 2251, + "imageTokens": 425, + "textGot": 2251, + "pureGot": 2251, + "prodGot": 2251, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 180, + "total_tokens": 285, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 56.65908923745155 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 327, + "total_tokens": 520, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 103.02272415161133 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 445, + "total_tokens": 647, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 137.71817702054977 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 78, + "kind": 2, + "question": "A warehouse has 24 shelves, each holding 73 boxes, plus 595 loose boxes. How many boxes are there in total?", + "answer": 2347, + "imageTokens": 425, + "textGot": 2347, + "pureGot": 2347, + "prodGot": 2347, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 169, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 53.45908933877945 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 326, + "total_tokens": 519, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 102.73181506991386 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 235, + "total_tokens": 437, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 76.62726986408234 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 79, + "kind": 2, + "question": "A warehouse has 24 shelves, each holding 23 boxes, plus 363 loose boxes. How many boxes are there in total?", + "answer": 915, + "imageTokens": 425, + "textGot": 915, + "pureGot": 915, + "prodGot": 915, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 157, + "total_tokens": 262, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 49.96818035840988 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 255, + "total_tokens": 448, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 82.07727026939392 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 310, + "total_tokens": 512, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 98.44545099139214 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 80, + "kind": 2, + "question": "A warehouse has 81 shelves, each holding 27 boxes, plus 979 loose boxes. How many boxes are there in total?", + "answer": 3166, + "imageTokens": 425, + "textGot": 3166, + "pureGot": 3166, + "prodGot": 3166, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 151, + "total_tokens": 256, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 48.2227258682251 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 318, + "total_tokens": 511, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 100.40454241633415 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 326, + "total_tokens": 528, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 103.09999629855156 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 81, + "kind": 2, + "question": "A warehouse has 35 shelves, each holding 72 boxes, plus 623 loose boxes. How many boxes are there in total?", + "answer": 3143, + "imageTokens": 425, + "textGot": 3143, + "pureGot": 3143, + "prodGot": 3143, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 143, + "total_tokens": 248, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 45.895453214645386 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 296, + "total_tokens": 489, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 94.00454261898994 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 247, + "total_tokens": 449, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 80.1181788444519 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 82, + "kind": 2, + "question": "A warehouse has 98 shelves, each holding 92 boxes, plus 123 loose boxes. How many boxes are there in total?", + "answer": 9139, + "imageTokens": 425, + "textGot": 9139, + "pureGot": 9139, + "prodGot": 9139, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 195, + "total_tokens": 300, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 61.02272546291351 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 264, + "total_tokens": 457, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 84.6954520046711 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 269, + "total_tokens": 471, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 86.51817864179611 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 83, + "kind": 2, + "question": "A warehouse has 26 shelves, each holding 43 boxes, plus 291 loose boxes. How many boxes are there in total?", + "answer": 1409, + "imageTokens": 425, + "textGot": 1409, + "pureGot": 1409, + "prodGot": 1409, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 134, + "total_tokens": 239, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 43.27727147936821 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 329, + "total_tokens": 522, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 103.60454231500626 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 194, + "total_tokens": 396, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 64.69999751448631 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 84, + "kind": 2, + "question": "A warehouse has 44 shelves, each holding 77 boxes, plus 911 loose boxes. How many boxes are there in total?", + "answer": 4299, + "imageTokens": 425, + "textGot": 4299, + "pureGot": 4299, + "prodGot": 4299, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 152, + "total_tokens": 257, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 48.51363494992256 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 382, + "total_tokens": 575, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 119.02272364497185 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 372, + "total_tokens": 574, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 116.4818140566349 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 85, + "kind": 2, + "question": "A warehouse has 79 shelves, each holding 75 boxes, plus 739 loose boxes. How many boxes are there in total?", + "answer": 6664, + "imageTokens": 425, + "textGot": 6664, + "pureGot": 6664, + "prodGot": 6664, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 116, + "total_tokens": 221, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 38.04090800881386 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 299, + "total_tokens": 492, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 94.87726986408234 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 482, + "total_tokens": 684, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 148.48181304335594 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 86, + "kind": 2, + "question": "A warehouse has 61 shelves, each holding 45 boxes, plus 283 loose boxes. How many boxes are there in total?", + "answer": 3028, + "imageTokens": 425, + "textGot": 3028, + "pureGot": 3028, + "prodGot": 3028, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 139, + "total_tokens": 244, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 44.73181688785553 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 322, + "total_tokens": 515, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 101.56817874312401 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 328, + "total_tokens": 530, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 103.68181446194649 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 87, + "kind": 2, + "question": "A warehouse has 84 shelves, each holding 78 boxes, plus 195 loose boxes. How many boxes are there in total?", + "answer": 6747, + "imageTokens": 425, + "textGot": 6747, + "pureGot": 6747, + "prodGot": 6747, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 158, + "total_tokens": 263, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 50.259089440107346 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 379, + "total_tokens": 572, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 118.14999639987946 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 253, + "total_tokens": 455, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 81.86363333463669 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 88, + "kind": 2, + "question": "A warehouse has 12 shelves, each holding 19 boxes, plus 695 loose boxes. How many boxes are there in total?", + "answer": 923, + "imageTokens": 425, + "textGot": 923, + "pureGot": 923, + "prodGot": 923, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 87, + "total_tokens": 192, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 29.604544639587402 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 276, + "total_tokens": 469, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 88.18636098504066 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 314, + "total_tokens": 516, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 99.60908731818199 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 89, + "kind": 2, + "question": "A warehouse has 55 shelves, each holding 11 boxes, plus 107 loose boxes. How many boxes are there in total?", + "answer": 712, + "imageTokens": 425, + "textGot": 712, + "pureGot": 712, + "prodGot": 712, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 139, + "total_tokens": 244, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 44.73181688785553 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 325, + "total_tokens": 518, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 102.4409059882164 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 303, + "total_tokens": 505, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 96.40908741950989 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 90, + "kind": 2, + "question": "A warehouse has 52 shelves, each holding 93 boxes, plus 327 loose boxes. How many boxes are there in total?", + "answer": 5163, + "imageTokens": 425, + "textGot": 5163, + "pureGot": 5163, + "prodGot": 5163, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 152, + "total_tokens": 257, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 48.513634979724884 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 374, + "total_tokens": 567, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 116.69545099139214 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 236, + "total_tokens": 438, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 76.9181789457798 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 91, + "kind": 2, + "question": "A warehouse has 41 shelves, each holding 30 boxes, plus 779 loose boxes. How many boxes are there in total?", + "answer": 2009, + "imageTokens": 425, + "textGot": 2009, + "pureGot": 2009, + "prodGot": 2009, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 113, + "total_tokens": 218, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 37.168180763721466 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 242, + "total_tokens": 435, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 78.29545220732689 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 212, + "total_tokens": 414, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 69.93636098504066 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 92, + "kind": 2, + "question": "A warehouse has 38 shelves, each holding 66 boxes, plus 359 loose boxes. How many boxes are there in total?", + "answer": 2867, + "imageTokens": 425, + "textGot": 2867, + "pureGot": 2867, + "prodGot": 2867, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 133, + "total_tokens": 238, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 42.986362397670746 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 351, + "total_tokens": 544, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 110.00454211235046 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 240, + "total_tokens": 442, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 78.08181527256966 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 93, + "kind": 2, + "question": "A warehouse has 83 shelves, each holding 26 boxes, plus 919 loose boxes. How many boxes are there in total?", + "answer": 3077, + "imageTokens": 425, + "textGot": 3077, + "pureGot": 3077, + "prodGot": 3077, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 191, + "total_tokens": 296, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 59.85908913612366 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 272, + "total_tokens": 465, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 87.02272465825081 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 282, + "total_tokens": 484, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 90.29999670386314 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 94, + "kind": 2, + "question": "A warehouse has 49 shelves, each holding 58 boxes, plus 791 loose boxes. How many boxes are there in total?", + "answer": 3633, + "imageTokens": 425, + "textGot": 3633, + "pureGot": 3633, + "prodGot": 3633, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 166, + "total_tokens": 271, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 52.58636209368706 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 405, + "total_tokens": 598, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 125.71363252401352 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 345, + "total_tokens": 547, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 108.62726885080338 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 95, + "kind": 2, + "question": "A warehouse has 73 shelves, each holding 58 boxes, plus 687 loose boxes. How many boxes are there in total?", + "answer": 4921, + "imageTokens": 425, + "textGot": 4921, + "pureGot": 4921, + "prodGot": 4921, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 169, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 53.45908933877945 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 289, + "total_tokens": 482, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 91.9681790471077 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 266, + "total_tokens": 468, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 85.64545139670372 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 96, + "kind": 2, + "question": "A warehouse has 58 shelves, each holding 73 boxes, plus 835 loose boxes. How many boxes are there in total?", + "answer": 5069, + "imageTokens": 425, + "textGot": 5069, + "pureGot": 5069, + "prodGot": 5069, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 144, + "total_tokens": 249, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 46.18636229634285 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 306, + "total_tokens": 499, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 96.9136334657669 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 348, + "total_tokens": 550, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 109.49999609589577 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 97, + "kind": 2, + "question": "A warehouse has 61 shelves, each holding 39 boxes, plus 383 loose boxes. How many boxes are there in total?", + "answer": 2762, + "imageTokens": 425, + "textGot": 2762, + "pureGot": 2762, + "prodGot": 2762, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 157, + "total_tokens": 262, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 49.96818035840988 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 190, + "total_tokens": 383, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 63.16817995905876 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 346, + "total_tokens": 548, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 108.91817793250084 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 98, + "kind": 2, + "question": "A warehouse has 44 shelves, each holding 98 boxes, plus 347 loose boxes. How many boxes are there in total?", + "answer": 4659, + "imageTokens": 425, + "textGot": 4659, + "pureGot": 4659, + "prodGot": 4659, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 117, + "total_tokens": 222, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 38.33181709051132 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 242, + "total_tokens": 435, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 78.29545220732689 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 248, + "total_tokens": 450, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 80.40908792614937 + }, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 99, + "kind": 2, + "question": "A warehouse has 64 shelves, each holding 98 boxes, plus 867 loose boxes. How many boxes are there in total?", + "answer": 7139, + "imageTokens": 425, + "textGot": 7139, + "pureGot": 7139, + "prodGot": 7139, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textUsage": { + "prompt_tokens": 105, + "completion_tokens": 150, + "total_tokens": 255, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 47.931816786527634 + }, + "pureUsage": { + "prompt_tokens": 193, + "completion_tokens": 251, + "total_tokens": 444, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 80.91363394260406 + }, + "prodUsage": { + "prompt_tokens": 202, + "completion_tokens": 229, + "total_tokens": 431, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "neurons": 74.88181537389755 + }, + "textError": null, + "pureError": null, + "prodError": null + } + ] +} \ No newline at end of file diff --git a/eval/qwen-profile/novel-arithmetic.mjs b/eval/qwen-profile/novel-arithmetic.mjs new file mode 100644 index 000000000..335e8f4f3 --- /dev/null +++ b/eval/qwen-profile/novel-arithmetic.mjs @@ -0,0 +1,110 @@ +// Qwen 3.8 novel arithmetic benchmark suite. +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { renderTextToPngs } from '../../dist/core/render.js'; +import { resolveGptProfile } from '../../dist/core/gpt-model-profiles.js'; +import { factSheetText } from '../../dist/core/factsheet.js'; +import { visionTokensForModel } from '../../dist/core/openai.js'; +import { callQwen, resultFilename } from './qwen-client.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const MODEL = process.env.MODEL || 'workers-ai/@cf/qwen/qwen3.8-27b'; +const LIVE = process.env.LIVE === '1'; +const N = Math.max(1, Number(process.env.N || 100)); +const SEED = Number(process.env.SEED || 20260711); +const CONCURRENCY = Math.max(1, Number(process.env.CONCURRENCY || 5)); +const TIMEOUT = Number(process.env.TIMEOUT_MS || 180000); +const profile = resolveGptProfile(MODEL); +const RESULT = join(HERE, resultFilename('novel-arithmetic', MODEL)); + +function lcg(seed) { let s = seed >>> 0; return () => (s = (Math.imul(s, 1664525) + 1013904223) >>> 0); } +function ri(r, a, b) { return a + (r() % (b - a + 1)); } +function num(text) { const m = String(text ?? '').match(/ANSWER:\s*(-?\d+)/i); return m ? Number(m[1]) : NaN; } + +function problems(n, seed) { + const r = lcg(seed), out = []; + for (let i = 0; i < n; i++) { + const k = r() % 4; + let question, answer; + if (k === 0) { + const a = ri(r, 1000, 9999), b = ri(r, 1000, 9999), c = ri(r, 1000, 9999); + question = `A factory produced ${a} units on Monday, ${b} units on Tuesday, and ${c} units on Wednesday. How many units did it produce in total over the three days?`; + answer = a + b + c; + } else if (k === 1) { + const a = ri(r, 3000, 9999), b = ri(r, 100, 999), c = ri(r, 100, 999); + question = `A reservoir contains ${a} gallons of water. ${b} gallons are pumped out, and later ${c} gallons flow in. How many gallons are in the reservoir now?`; + answer = a - b + c; + } else if (k === 2) { + const a = ri(r, 11, 99), b = ri(r, 11, 99), c = ri(r, 100, 999); + question = `A warehouse has ${a} shelves, each holding ${b} boxes, plus ${c} loose boxes. How many boxes are there in total?`; + answer = a * b + c; + } else { + const a = ri(r, 5000, 9999), b = ri(r, 1000, 4999); + question = `A stadium has ${a} seats. ${b} are already sold. How many seats remain unsold?`; + answer = a - b; + } + out.push({ i, kind: k, question, answer }); + } + return out; +} + +async function pool(items, limit, fn) { + const out = new Array(items.length); + let next = 0; + async function w() { + while (next < items.length) { + const i = next++; + out[i] = await fn(items[i]); + } + } + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, w)); + return out; +} + +const ps = problems(N, SEED); +console.log(`novel arithmetic · model=${MODEL} · live=${LIVE} · N=${N}`); + +if (!LIVE) { + console.log('Dry run only. Exiting.'); + process.exit(0); +} + +const rows = await pool(ps, CONCURRENCY, async (p) => { + const ask = "Solve the math word problem. Show brief reasoning and end with exactly 'ANSWER: '."; + const imgs = await renderTextToPngs(p.question, profile.stripCols, profile.style, profile.maxHeightPx); + const urls = imgs.map((im) => ({ type: 'input_image', image_url: `data:image/png;base64,${Buffer.from(im.png).toString('base64')}` })); + const imageTokens = imgs.reduce((n, im) => n + visionTokensForModel(MODEL, im.width, im.height), 0); + + let text, pure, prod; + try { text = await callQwen({ model: MODEL, content: [{ type: 'input_text', text: `${ask}\n\n${p.question}` }], maxOutputTokens: 1024, timeoutMs: TIMEOUT }); } catch (e) { text = { text: '', error: String(e.message || e) }; } + try { pure = await callQwen({ model: MODEL, content: [...urls, { type: 'input_text', text: `The problem is in the image. ${ask}` }], maxOutputTokens: 1024, timeoutMs: TIMEOUT }); } catch (e) { pure = { text: '', error: String(e.message || e) }; } + try { + const fs = factSheetText(p.question, profile.factSheetFormat); + prod = await callQwen({ model: MODEL, content: [...urls, ...(fs ? [{ type: 'input_text', text: fs }] : []), { type: 'input_text', text: `The problem is in the image; use the exact-number factsheet if present. ${ask}` }], maxOutputTokens: 1024, timeoutMs: TIMEOUT }); + } catch (e) { prod = { text: '', error: String(e.message || e) }; } + + const textGot = num(text.text), pureGot = num(pure.text), prodGot = num(prod.text); + const row = { ...p, imageTokens, textGot, pureGot, prodGot, textOk: textGot === p.answer, pureOk: pureGot === p.answer, prodOk: prodGot === p.answer, textUsage: text.usage || null, pureUsage: pure.usage || null, prodUsage: prod.usage || null, textError: text.error || null, pureError: pure.error || null, prodError: prod.error || null }; + console.log(`q${p.i} text=${row.textOk ? 'Y' : 'N'}(${textGot}) pure=${row.pureOk ? 'Y' : 'N'}(${pureGot}) prod=${row.prodOk ? 'Y' : 'N'}(${prodGot}) gold=${p.answer}`); + return row; +}); + +const count = (k) => rows.filter((r) => r[k]).length; +const summary = { + generatedAt: new Date().toISOString(), + model: MODEL, + live: true, + n: N, + seed: SEED, + recipe: { cols: profile.stripCols, maxH: profile.maxHeightPx, style: profile.style, factsheet: true }, + textCorrect: count('textOk'), + pureCorrect: count('pureOk'), + prodCorrect: count('prodOk'), + textPct: 100 * count('textOk') / N, + purePct: 100 * count('pureOk') / N, + prodPct: 100 * count('prodOk') / N, + rows, +}; +writeFileSync(RESULT, JSON.stringify(summary, null, 2)); +console.log(`\nSUMMARY text ${summary.textCorrect}/${N} (${summary.textPct}%) · pure ${summary.pureCorrect}/${N} (${summary.purePct}%) · prod ${summary.prodCorrect}/${N} (${summary.prodPct}%)`); diff --git a/eval/qwen-profile/qwen-client.mjs b/eval/qwen-profile/qwen-client.mjs new file mode 100644 index 000000000..11b20c84f --- /dev/null +++ b/eval/qwen-profile/qwen-client.mjs @@ -0,0 +1,116 @@ +// Qwen 3.8 Workers AI client through ocproxy / Cloudflare AI Gateway. +import { readFileSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; + +function getAuthToken() { + if (process.env.OPENAI_API_KEY) return process.env.OPENAI_API_KEY; + if (process.env.OCPROXY_TOKEN) return process.env.OCPROXY_TOKEN; + const tokenPath = join(process.env.HOME || '', '.local/state/ocproxy/dashboard.token'); + if (existsSync(tokenPath)) { + return readFileSync(tokenPath, 'utf8').trim(); + } + return ''; +} + +function gatewayEndpoint() { + const base = (process.env.OPENAI_BASE_URL || 'http://127.0.0.1:8082/v1').replace(/\/$/, ''); + return base.endsWith('/chat/completions') ? base : `${base}/chat/completions`; +} + +export function resultFilename(base, model) { + const clean = model.replace(/^workers-ai\//, '').replace(/^@cf\//, ''); + if (clean === 'qwen/qwen3.8-27b' || clean === 'qwen3.8-27b') return `${base}-results.json`; + return `${base}-${clean.replace(/[^a-zA-Z0-9._-]+/g, '_')}-results.json`; +} + +export async function callQwen({ + model = 'workers-ai/@cf/qwen/qwen3.8-27b', + content, + maxOutputTokens = 2048, + timeoutMs = 120000, +}) { + const token = getAuthToken(); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + const started = Date.now(); + + const messages = [ + { + role: 'user', + content: content.map((part) => { + if (part.type === 'input_text' || part.type === 'text') { + return { type: 'text', text: part.text }; + } + if (part.type === 'input_image' || part.type === 'image_url') { + const url = typeof part.image_url === 'string' ? part.image_url : part.image_url?.url; + return { type: 'image_url', image_url: { url } }; + } + throw new Error(`unsupported content part: ${part.type}`); + }), + }, + ]; + + try { + const response = await fetch(gatewayEndpoint(), { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(token ? { Authorization: `Bearer ${token}` } : {}), + }, + body: JSON.stringify({ + model, + messages, + stream: true, + max_tokens: maxOutputTokens, + }), + signal: controller.signal, + }); + + if (!response.ok) { + const raw = await response.text(); + let msg = raw; + try { + const j = JSON.parse(raw); + msg = j?.error?.message || j?.message || raw; + } catch {} + throw new Error(`HTTP ${response.status}: ${msg}`); + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let text = ''; + let reasoning = ''; + let usage = null; + let buffer = ''; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed.startsWith('data:')) continue; + const payload = trimmed.slice(5).trim(); + if (payload === '[DONE]') continue; + try { + const json = JSON.parse(payload); + const delta = json.choices?.[0]?.delta; + if (delta?.content) text += delta.content; + if (delta?.reasoning) reasoning += delta.reasoning; + if (json.usage) usage = json.usage; + } catch {} + } + } + + return { + text: text.trim(), + reasoning: reasoning.trim(), + usage, + ms: Date.now() - started, + }; + } finally { + clearTimeout(timer); + } +} diff --git a/eval/qwen-profile/verbatim-hex-14px-results.json b/eval/qwen-profile/verbatim-hex-14px-results.json new file mode 100644 index 000000000..fd2418220 --- /dev/null +++ b/eval/qwen-profile/verbatim-hex-14px-results.json @@ -0,0 +1,130 @@ +{ + "model": "workers-ai/@cf/qwen/qwen3.8-27b", + "font": "jetbrains-mono-14", + "cols": 84, + "maxH": 512, + "correct": 11, + "n": 15, + "rows": [ + { + "page": 0, + "dur": 4439, + "gold": "c9c947f680ec", + "got": "c9c947f680ec", + "ok": true, + "raw": "c9c947f680ec" + }, + { + "page": 0, + "dur": 812, + "gold": "851eb3af1bd1", + "got": "851eb3af1bd1", + "ok": true, + "raw": "851eb3af1bd1" + }, + { + "page": 0, + "dur": 6150, + "gold": "ade34f70fd73", + "got": "ade34f70fd73", + "ok": true, + "raw": "ade34f70fd73" + }, + { + "page": 1, + "dur": 7978, + "gold": "c5d68855f46d", + "got": "c5d68855f46d", + "ok": true, + "raw": "c5d68855f46d" + }, + { + "page": 1, + "dur": 8071, + "gold": "92abade01aad", + "got": "92abade01aad", + "ok": true, + "raw": "92abade01aad" + }, + { + "page": 1, + "dur": 3309, + "gold": "ffe21785b09d", + "got": "ffe21785b09d", + "ok": true, + "raw": "ffe21785b09d" + }, + { + "page": 2, + "dur": 7215, + "gold": "87cb51eb0e99", + "got": "87cb51eb0e99", + "ok": true, + "raw": "87cb51eb0e99" + }, + { + "page": 2, + "dur": 4397, + "gold": "93c3ced96dac", + "got": "93c3ced96dac", + "ok": true, + "raw": "93c3ced96dac" + }, + { + "page": 2, + "dur": 4495, + "gold": "f152ae9bfb8f", + "got": "ERR: This operation was aborted", + "ok": false, + "raw": "" + }, + { + "page": 3, + "dur": 1622, + "gold": "5a7373d4187f", + "got": "5a7373d4187f", + "ok": true, + "raw": "5a7373d4187f" + }, + { + "page": 3, + "dur": 2025, + "gold": "44ea8c7aeedd", + "got": "", + "ok": false, + "raw": "" + }, + { + "page": 3, + "dur": 6533, + "gold": "8145b5a0fd46", + "got": "81458a5b0fd6", + "ok": false, + "raw": "81458a5b0fd6" + }, + { + "page": 4, + "dur": 2921, + "gold": "b8fce698f971", + "got": "b8fce698f971", + "ok": true, + "raw": "b8fce698f971" + }, + { + "page": 4, + "dur": 8475, + "gold": "4a8164556b99", + "got": "4a816455b699", + "ok": false, + "raw": "4a816455b699" + }, + { + "page": 4, + "dur": 8799, + "gold": "e53112c4b5a4", + "got": "e53112c4b5a4", + "ok": true, + "raw": "e53112c4b5a4" + } + ] +} \ No newline at end of file diff --git a/eval/qwen-profile/verbatim-hex-results.json b/eval/qwen-profile/verbatim-hex-results.json new file mode 100644 index 000000000..39e0f88a8 --- /dev/null +++ b/eval/qwen-profile/verbatim-hex-results.json @@ -0,0 +1,161 @@ +{ + "generatedAt": "2026-08-19T22:19:45.996Z", + "model": "workers-ai/@cf/qwen/qwen3.8-27b", + "live": true, + "correct": 0, + "completed": 15, + "errors": 0, + "n": 15, + "rows": [ + { + "page": 0, + "dur": 4439, + "gold": "c9c947f680ec", + "got": "", + "ok": false, + "raw": "", + "ms": 43179, + "error": null + }, + { + "page": 0, + "dur": 812, + "gold": "851eb3af1bd1", + "got": "", + "ok": false, + "raw": "", + "ms": 58714, + "error": null + }, + { + "page": 0, + "dur": 6150, + "gold": "ade34f70fd73", + "got": "2a19e563d3e4", + "ok": false, + "raw": "2a19e563d3e4", + "ms": 5244, + "error": null + }, + { + "page": 1, + "dur": 7978, + "gold": "c5d68855f46d", + "got": "", + "ok": false, + "raw": "", + "ms": 81157, + "error": null + }, + { + "page": 1, + "dur": 8071, + "gold": "92abade01aad", + "got": "", + "ok": false, + "raw": "", + "ms": 108315, + "error": null + }, + { + "page": 1, + "dur": 3309, + "gold": "ffe21785b09d", + "got": "", + "ok": false, + "raw": "", + "ms": 97382, + "error": null + }, + { + "page": 2, + "dur": 7215, + "gold": "87cb51eb0e99", + "got": "162594c33a1f", + "ok": false, + "raw": "162594c33a1f", + "ms": 50995, + "error": null + }, + { + "page": 2, + "dur": 4397, + "gold": "93c3ced96dac", + "got": "830c34948575", + "ok": false, + "raw": "830c34948575", + "ms": 35886, + "error": null + }, + { + "page": 2, + "dur": 4495, + "gold": "f152ae9bfb8f", + "got": "0b55208f4c4c", + "ok": false, + "raw": "0b55208f4c4c", + "ms": 34411, + "error": null + }, + { + "page": 3, + "dur": 1622, + "gold": "5a7373d4187f", + "got": "50753a6071c4", + "ok": false, + "raw": "50753a6071c4", + "ms": 26216, + "error": null + }, + { + "page": 3, + "dur": 2025, + "gold": "44ea8c7aeedd", + "got": "0414c660d684", + "ok": false, + "raw": "0414c660d684", + "ms": 33255, + "error": null + }, + { + "page": 3, + "dur": 6533, + "gold": "8145b5a0fd46", + "got": "", + "ok": false, + "raw": "", + "ms": 75078, + "error": null + }, + { + "page": 4, + "dur": 2921, + "gold": "b8fce698f971", + "got": "4c7fc1d2d818", + "ok": false, + "raw": "4c7fc1d2d818", + "ms": 15583, + "error": null + }, + { + "page": 4, + "dur": 8475, + "gold": "4a8164556b99", + "got": "ca31b9e5b92d", + "ok": false, + "raw": "ca31b9e5b92d", + "ms": 33850, + "error": null + }, + { + "page": 4, + "dur": 8799, + "gold": "e53112c4b5a4", + "got": "", + "ok": false, + "raw": "", + "ms": 75345, + "error": null + } + ] +} \ No newline at end of file diff --git a/eval/qwen-profile/verbatim-hex.mjs b/eval/qwen-profile/verbatim-hex.mjs new file mode 100644 index 000000000..5f3e0a311 --- /dev/null +++ b/eval/qwen-profile/verbatim-hex.mjs @@ -0,0 +1,90 @@ +// Qwen 3.8 verbatim hex evaluation suite. +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { callQwen, resultFilename } from './qwen-client.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = join(HERE, '../verbatim-15'); +const MODEL = process.env.MODEL || 'workers-ai/@cf/qwen/qwen3.8-27b'; +const LIVE = process.env.LIVE === '1'; +const TIMEOUT = Number(process.env.TIMEOUT_MS || 120000); +const MAX_OUTPUT_TOKENS = Number(process.env.MAX_OUTPUT_TOKENS || 4096); +const trials = JSON.parse(readFileSync(join(ROOT, 'golds.json'), 'utf8')); +const RESULT = join(HERE, resultFilename('verbatim-hex', MODEL)); + +function writeResult(rows) { + const completed = rows.filter((r) => !r.error); + const result = { + generatedAt: new Date().toISOString(), + model: MODEL, + live: LIVE, + correct: completed.filter((r) => r.ok).length, + completed: completed.length, + errors: rows.length - completed.length, + n: trials.length, + rows, + }; + writeFileSync(RESULT, JSON.stringify(result, null, 2)); + return result; +} + +async function callImage(trial) { + const png = readFileSync(join(ROOT, `page${trial.page}.png`)); + const content = [ + { + type: 'input_image', + image_url: `data:image/png;base64,${png.toString('base64')}`, + }, + { + type: 'input_text', + text: `Read the image visually. Find the JSON line whose dur_ms is exactly ${trial.dur}. Return only its id field, exactly 12 lowercase hex characters.`, + }, + ]; + return callQwen({ model: MODEL, content, maxOutputTokens: MAX_OUTPUT_TOKENS, timeoutMs: TIMEOUT }); +} + +const existingRows = existsSync(RESULT) + ? JSON.parse(readFileSync(RESULT, 'utf8')).rows || [] + : []; +const completedIndices = new Set( + existingRows.filter((r) => !r.error).map((r, i) => i) +); + +const rows = [...existingRows]; + +for (let i = 0; i < trials.length; i++) { + const t = trials[i]; + if (i < rows.length && rows[i] && !rows[i].error && rows[i].got !== undefined) { + console.log(`trial ${i + 1}/${trials.length} page${t.page} dur=${t.dur} (cached: ${rows[i].ok ? 'HIT' : 'MISS'})`); + continue; + } + + let out = ''; + let ms = null; + let err = null; + process.stdout.write(`trial ${i + 1}/${trials.length} page${t.page} dur=${t.dur} ... `); + try { + if (LIVE) { + const r = await callImage(t); + out = r.text; + ms = r.ms; + } + } catch (e) { + err = String(e?.message || e); + out = err; + } + const got = out.match(/[0-9a-f]{12}/i)?.[0]?.toLowerCase() || ''; + const ok = got === t.gold; + rows[i] = { ...t, got, ok, raw: out, ms, error: err }; + if (LIVE) writeResult(rows); + console.log(`${ok ? 'HIT' : 'MISS'} gold=${t.gold} got=${got || '-'}${ms != null ? ` ${ms}ms` : ''}${err ? ` ERR ${err.slice(0, 100)}` : ''}`); +} + +if (!LIVE) { + console.log('Dry run only; no receipt written'); + process.exit(0); +} + +const result = writeResult(rows); +console.log(`SUMMARY ${result.correct}/${result.completed} completed (${result.errors} errors) -> ${RESULT}`); From 542d4e3e04dfcddca24487f878e0f9ed67b82ac8 Mon Sep 17 00:00:00 2001 From: Steven Chong <25894545+teamchong@users.noreply.github.com> Date: Wed, 19 Aug 2026 21:59:40 -0400 Subject: [PATCH 02/11] fix(openai): use 'high' detail for non-gpt5 models and address review findings --- eval/qwen-profile/gist-recall.mjs | 7 ++++-- eval/qwen-profile/novel-arithmetic.mjs | 4 +-- eval/qwen-profile/qwen-client.mjs | 4 +++ eval/qwen-profile/verbatim-hex.mjs | 17 +++++++++---- src/core/openai.ts | 22 ++++++++++------- tests/openai-gpt5.test.ts | 34 ++++++++++++++++++++++++++ 6 files changed, 70 insertions(+), 18 deletions(-) diff --git a/eval/qwen-profile/gist-recall.mjs b/eval/qwen-profile/gist-recall.mjs index e124fb3c6..75078ad30 100644 --- a/eval/qwen-profile/gist-recall.mjs +++ b/eval/qwen-profile/gist-recall.mjs @@ -37,7 +37,10 @@ function correct(p, a) { const g = norm(p.gold); if (p.type === 'unanswerable') return x === 'unknown'; if (p.type === 'numeric') return new RegExp(`(?:^|\\D)${g}(?:\\D|$)`).test(x); - if (p.type === 'negation') return x.includes('off') && !x.includes('enabled'); + if (p.type === 'negation') { + if (g === 'off') return x.includes('off') && !x.includes('enabled'); + if (g === 'enabled') return x.includes('enabled') && !x.includes('off'); + } return x.includes(g); } @@ -57,7 +60,7 @@ function writeProgress(rows) { unanswerable: { confabulated: done(guards).filter((r) => !r.ok).length, completed: done(guards).length, n: guards.length }, rows, }; - writeFileSync(RESULT, JSON.stringify(out, null, 2)); + writeFileSync(RESULT, JSON.stringify(out, null, 2) + '\n'); return out; } diff --git a/eval/qwen-profile/novel-arithmetic.mjs b/eval/qwen-profile/novel-arithmetic.mjs index 335e8f4f3..4e170ea80 100644 --- a/eval/qwen-profile/novel-arithmetic.mjs +++ b/eval/qwen-profile/novel-arithmetic.mjs @@ -94,7 +94,7 @@ const count = (k) => rows.filter((r) => r[k]).length; const summary = { generatedAt: new Date().toISOString(), model: MODEL, - live: true, + live: LIVE, n: N, seed: SEED, recipe: { cols: profile.stripCols, maxH: profile.maxHeightPx, style: profile.style, factsheet: true }, @@ -106,5 +106,5 @@ const summary = { prodPct: 100 * count('prodOk') / N, rows, }; -writeFileSync(RESULT, JSON.stringify(summary, null, 2)); +writeFileSync(RESULT, JSON.stringify(summary, null, 2) + '\n'); console.log(`\nSUMMARY text ${summary.textCorrect}/${N} (${summary.textPct}%) · pure ${summary.pureCorrect}/${N} (${summary.purePct}%) · prod ${summary.prodCorrect}/${N} (${summary.prodPct}%)`); diff --git a/eval/qwen-profile/qwen-client.mjs b/eval/qwen-profile/qwen-client.mjs index 11b20c84f..d86756b5b 100644 --- a/eval/qwen-profile/qwen-client.mjs +++ b/eval/qwen-profile/qwen-client.mjs @@ -76,6 +76,10 @@ export async function callQwen({ throw new Error(`HTTP ${response.status}: ${msg}`); } + if (!response.body) { + throw new Error(`HTTP ${response.status}: empty body`); + } + const reader = response.body.getReader(); const decoder = new TextDecoder(); let text = ''; diff --git a/eval/qwen-profile/verbatim-hex.mjs b/eval/qwen-profile/verbatim-hex.mjs index 5f3e0a311..6cbc3cd1d 100644 --- a/eval/qwen-profile/verbatim-hex.mjs +++ b/eval/qwen-profile/verbatim-hex.mjs @@ -25,12 +25,22 @@ function writeResult(rows) { n: trials.length, rows, }; - writeFileSync(RESULT, JSON.stringify(result, null, 2)); + writeFileSync(RESULT, JSON.stringify(result, null, 2) + '\n'); return result; } +const pageCache = new Map(); +function getPagePng(page) { + let png = pageCache.get(page); + if (!png) { + png = readFileSync(join(ROOT, `page${page}.png`)); + pageCache.set(page, png); + } + return png; +} + async function callImage(trial) { - const png = readFileSync(join(ROOT, `page${trial.page}.png`)); + const png = getPagePng(trial.page); const content = [ { type: 'input_image', @@ -47,9 +57,6 @@ async function callImage(trial) { const existingRows = existsSync(RESULT) ? JSON.parse(readFileSync(RESULT, 'utf8')).rows || [] : []; -const completedIndices = new Set( - existingRows.filter((r) => !r.error).map((r, i) => i) -); const rows = [...existingRows]; diff --git a/src/core/openai.ts b/src/core/openai.ts index 9301aa269..e815d10aa 100644 --- a/src/core/openai.ts +++ b/src/core/openai.ts @@ -475,22 +475,26 @@ function rewriteFlatToolsForGpt(tools: unknown[] | undefined): { return { tools: changed ? rewritten : tools, docs: docs.join('\n\n') }; } -function openAIImagePart(img: RenderedImage): OpenAIImagePart { +function isGpt5Family(model?: string): boolean { + return typeof model === 'string' && /^gpt-5/i.test(model); +} + +function openAIImagePart(img: RenderedImage, model?: string): OpenAIImagePart { return { type: 'image_url', image_url: { url: `data:image/png;base64,${bytesToBase64(img.png)}`, - detail: 'original', // GPT-5.6 preserves submitted dimensions; older profiles retain their own cost caps. + detail: isGpt5Family(model) ? 'original' : 'high', }, }; } /** Build a Responses API input_image part. */ -function responsesImagePart(img: RenderedImage): ResponsesInputImagePart { +function responsesImagePart(img: RenderedImage, model?: string): ResponsesInputImagePart { return { type: 'input_image', image_url: `data:image/png;base64,${bytesToBase64(img.png)}`, - detail: 'original', // see openAIImagePart: avoid 'high' downscale of dense text + detail: isGpt5Family(model) ? 'original' : 'high', }; } @@ -820,10 +824,10 @@ async function applyChatHistoryCollapse( const outro = compactFraming ? COMPACT_HISTORY_TRANSCRIPT_OUTRO : HISTORY_TRANSCRIPT_OUTRO; const histFactSheet = factSheetText(plan.text, profile.factSheetFormat); const content: OpenAIContentPart[] = [{ type: 'text', text: intro }]; - for (const img of plan.images) content.push(openAIImagePart(img)); + for (const img of plan.images) content.push(openAIImagePart(img, req.model)); if (plan.pinText !== undefined) { content.push({ type: 'text', text: pinnedRequestBlock(plan.pinText) }); - for (const img of plan.imagesAfter) content.push(openAIImagePart(img)); + for (const img of plan.imagesAfter) content.push(openAIImagePart(img, req.model)); } if (histFactSheet) content.push({ type: 'text', text: histFactSheet }); content.push({ type: 'text', text: outro }); @@ -890,7 +894,7 @@ async function applyResponsesHistoryCollapse( const segment = plan.segments[segmentIndex]!; const content: ResponsesContentPart[] = [ { type: 'input_text', text: intro }, - ...segment.images.map(responsesImagePart), + ...segment.images.map((img) => responsesImagePart(img, req.model)), ]; const sheet = profile.history.factSheetScope === 'combined' ? (segmentIndex === plan.segments.length - 1 ? combinedSheet : '') @@ -1052,7 +1056,7 @@ export async function transformOpenAIChatCompletions( const topDropped = droppedCodepointsTop(droppedCodepoints); if (topDropped) info.droppedCodepointsTop = topDropped; - const imageParts: OpenAIImagePart[] = images.map(openAIImagePart); + const imageParts: OpenAIImagePart[] = images.map((img) => openAIImagePart(img, req.model)); info.imageCount = images.length; // GPT savings basis: vision tokens the images actually cost vs the text tokens // the same content would have cost unproxied. req.tools is still the original @@ -1296,7 +1300,7 @@ export async function transformOpenAIResponses( info.imageSourceText = renderedText.slice(0, 65_536); info.imageSourceTexts = images.map(() => info.imageSourceText); - const imagePartsResp: ResponsesInputImagePart[] = images.map(responsesImagePart); + const imagePartsResp: ResponsesInputImagePart[] = images.map((img) => responsesImagePart(img, req.model)); const endMarker: ResponsesInputTextPart = { type: 'input_text', text: '[End of rendered GPT system/tool context.]' }; // Verbatim fact-sheet (see src/core/factsheet.ts): exact tokens that survive OCR loss. const slabFactSheet = factSheetText(combinedRaw, profile.factSheetFormat); diff --git a/tests/openai-gpt5.test.ts b/tests/openai-gpt5.test.ts index ecc7a585a..0da2cae81 100644 --- a/tests/openai-gpt5.test.ts +++ b/tests/openai-gpt5.test.ts @@ -1035,6 +1035,24 @@ describe('image parts request detail = "original" (avoid downscale of dense text for (const p of imgs) expect(p.image_url!.detail).toBe('original'); }); + it('Chat Completions image_url parts use detail:"high" for non-gpt5 models (Workers AI / Qwen / GPT-4o)', async () => { + const body = enc.encode(JSON.stringify({ + model: 'workers-ai/@cf/qwen/qwen3.8-27b', + messages: [ + { role: 'system', content: BIG_SYSTEM }, + { role: 'user', content: 'hello' }, + ], + })); + const result = await transformOpenAIChatCompletions(body, { charsPerToken: 1, minCompressChars: 1 }); + expect(result.info.compressed).toBe(true); + const out = JSON.parse(dec.decode(result.body)) as { messages: Array<{ role: string; content: unknown }> }; + const firstUser = out.messages.find((m) => m.role === 'user')!; + const parts = firstUser.content as Array<{ type: string; image_url?: { detail?: string } }>; + const imgs = parts.filter((p) => p.type === 'image_url'); + expect(imgs.length).toBeGreaterThan(0); + for (const p of imgs) expect(p.image_url!.detail).toBe('high'); + }); + it('Responses input_image parts use detail:"original"', async () => { const body = enc.encode(JSON.stringify({ model: 'gpt-5.6-sol', @@ -1050,6 +1068,22 @@ describe('image parts request detail = "original" (avoid downscale of dense text expect(imgs.length).toBeGreaterThan(0); for (const p of imgs) expect(p.detail).toBe('original'); }); + + it('Responses input_image parts use detail:"high" for non-gpt5 models', async () => { + const body = enc.encode(JSON.stringify({ + model: 'workers-ai/@cf/qwen/qwen3.8-27b', + instructions: BIG_INSTRUCTIONS, + input: [{ role: 'user', content: [{ type: 'input_text', text: 'hello' }] }], + })); + const result = await transformOpenAIResponses(body, { charsPerToken: 1, minCompressChars: 1 }); + expect(result.info.compressed).toBe(true); + const out = JSON.parse(dec.decode(result.body)) as { input: Array<{ role?: string; content?: unknown }> }; + const firstUser = out.input.find((m) => m.role === 'user')!; + const parts = firstUser.content as Array<{ type: string; detail?: string }>; + const imgs = parts.filter((p) => p.type === 'input_image'); + expect(imgs.length).toBeGreaterThan(0); + for (const p of imgs) expect(p.detail).toBe('high'); + }); }); From 82ebbb010d73d00a9b55bc4fb75d542b8d4ce646 Mon Sep 17 00:00:00 2001 From: Steven Chong <25894545+teamchong@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:56:59 -0400 Subject: [PATCH 03/11] feat(qwen): add dedicated Qwen profile and dynamic image cap under 32 images --- src/core/gpt-model-profiles.ts | 28 ++++++++++++++++++++++++++++ src/core/openai.ts | 25 ++++++++++++++++++++++--- tests/openai-gpt5.test.ts | 11 +++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/src/core/gpt-model-profiles.ts b/src/core/gpt-model-profiles.ts index 72826166b..24a8608f0 100644 --- a/src/core/gpt-model-profiles.ts +++ b/src/core/gpt-model-profiles.ts @@ -229,6 +229,9 @@ const isMiniNanoPatch = (m: string): boolean => /** Grok ids pxpipe has a measured profile for. */ const isGrokModel = (m: string): boolean => /^grok-/.test(m); +/** Qwen ids pxpipe has a measured profile for (e.g. Qwen 3.8 / Qwen 2.5). */ +const isQwenModel = (m: string): boolean => /qwen/i.test(m); + /** Shared GPT geometry for the small patch-billed models; only the patch * multiplier and the family list prices differ between the rules below. */ const miniNanoProfile = ( @@ -323,6 +326,30 @@ const BUILTIN_RULES: ProfileRule[] = [ }, }, }, + + // Qwen (Workers AI / open weights). Native 14px / 84 cols / maxH 512 is required + // because 5x8 bitmap glyphs are illegible to Qwen vision (0/15 hex vs 11/15 on 14px). + // History image count capped at 24 images to stay safely below Workers AI's 32-image limit. + { + test: isQwenModel, + profile: { + vision: { regime: 'mpix', tokensPerMegapixel: 1000 }, + cacheReadRate: 0.25, + outputRate: 3, + stripCols: 84, + maxHeightPx: 512, + minCompressTokens: 500, + factSheetFormat: 'full', + history: { ...BASE_HISTORY, maxImages: 24 }, + style: { + ...BASE_STYLE, + font: 'jetbrains-mono-14', + aa: true, + grid: false, + gridCols: 0, + }, + }, + }, ]; /** @@ -333,6 +360,7 @@ const BUILTIN_RULES: ProfileRule[] = [ const FAMILY_ID_GUARDS: ReadonlyArray<{ mentions: RegExp; matches: (m: string) => boolean }> = [ { mentions: /gemini/, matches: hasGeminiMeasuredProfile }, { mentions: /grok/, matches: isGrokModel }, + { mentions: /qwen/, matches: isQwenModel }, ]; /** True when the operator declared this id in PXPIPE_GPT_PROFILES. The guards diff --git a/src/core/openai.ts b/src/core/openai.ts index e815d10aa..5f6ce6ef6 100644 --- a/src/core/openai.ts +++ b/src/core/openai.ts @@ -249,11 +249,28 @@ function configuredHistoryMaxImages(model: string): number { return Number.isFinite(parsed) ? Math.max(1, Math.min(100, parsed)) : fallback; } +function countRequestImages(messages: OpenAIChatMessage[]): number { + let count = 0; + for (const msg of messages) { + if (Array.isArray(msg.content)) { + for (const part of msg.content) { + if (typeof part === 'object' && part !== null && 'type' in part && (part as { type?: string }).type === 'image_url') { + count++; + } + } + } + } + return count; +} + function gptHistoryOpts( model: string, o: OpenAIResolvedOptions, profile: ReturnType, + existingImages = 0, ): Partial { + const configuredMax = o.gptHistory?.maxImages ?? configuredHistoryMaxImages(model); + const remainingMax = Math.max(0, configuredMax - existingImages); return { ...o.gptHistory, reflow: o.reflow, @@ -264,7 +281,7 @@ function gptHistoryOpts( cols: o.gptHistory?.cols ?? profile.stripCols, maxHeightPx: o.gptHistory?.maxHeightPx ?? profile.maxHeightPx, style: o.gptHistory?.style ?? profile.style, - maxImages: o.gptHistory?.maxImages ?? configuredHistoryMaxImages(model), + maxImages: remainingMax, }; } @@ -809,11 +826,12 @@ async function applyChatHistoryCollapse( ): Promise { const profitable = (text: string, cols: number, baselineTextTokens?: number) => evalOpenAIGate(req.model, text, cols, o.charsPerToken, baselineTextTokens).profitable; + const existingImages = (info.imageCount ?? 0) + countRequestImages(req.messages); const plan = await planGptCollapse( chatMessagesToTurns(req.messages), protectedPrefix, profitable, - gptHistoryOpts(req.model, o, profile), + gptHistoryOpts(req.model, o, profile, existingImages), ); foldGptHistory(info, req.model, plan); const allImages = [...plan.images, ...plan.imagesAfter]; @@ -853,10 +871,11 @@ async function applyResponsesHistoryCollapse( ): Promise { const profitable = (text: string, cols: number, baselineTextTokens?: number) => evalOpenAIGate(req.model, text, cols, o.charsPerToken, baselineTextTokens).profitable; + const existingImages = info.imageCount ?? 0; const plan = await planResponsesPairCollapse( inputItems, profitable, - gptHistoryOpts(req.model, o, profile), + gptHistoryOpts(req.model, o, profile, existingImages), ); const ps = plan.pairState; const rc = info.responsesComposition!; diff --git a/tests/openai-gpt5.test.ts b/tests/openai-gpt5.test.ts index 0da2cae81..a738c2072 100644 --- a/tests/openai-gpt5.test.ts +++ b/tests/openai-gpt5.test.ts @@ -1208,6 +1208,17 @@ describe('resolveGptProfile (Grok)', () => { }); }); +describe('resolveGptProfile (Qwen)', () => { + it('uses native 14px packing and 24 max images under Workers AI 32-image cap', () => { + const p = resolveGptProfile('workers-ai/@cf/qwen/qwen3.8-27b'); + expect(p.stripCols).toBe(84); + expect(p.maxHeightPx).toBe(512); + expect(p.style.font).toBe('jetbrains-mono-14'); + expect(p.history.maxImages).toBe(24); + expect(resolveGptProfile('qwen-3.8-27b').stripCols).toBe(84); + }); +}); + describe('resolveGptProfile style overrides', () => { it('merges every render knob into the selected model profile', () => { const prev = process.env.PXPIPE_GPT_PROFILES; From a60f3372244ff9806815ac3b3ecb4e19e5db0ca4 Mon Sep 17 00:00:00 2001 From: Steven Chong <25894545+teamchong@users.noreply.github.com> Date: Thu, 20 Aug 2026 09:25:20 -0400 Subject: [PATCH 04/11] fix(bridge): use detail high for non-gpt5 models in messages-responses-bridge --- src/core/messages-responses-bridge.ts | 4 ++-- tests/proxy-usage.test.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/messages-responses-bridge.ts b/src/core/messages-responses-bridge.ts index 9b89b9119..1deedc756 100644 --- a/src/core/messages-responses-bridge.ts +++ b/src/core/messages-responses-bridge.ts @@ -51,7 +51,7 @@ function inputParts(content: unknown, location = 'message', role = 'user'): Json } else if (part?.type === 'image') { const image_url = imageUrl(part.source); if (!image_url) invalidRequest(`Unsupported ${location} image source`); - out.push({ type: 'input_image', image_url, detail: 'original' }); + out.push({ type: 'input_image', image_url, detail: 'high' }); } else { invalidRequest(`Unsupported ${location} content block: ${String(part?.type ?? 'invalid')}`); } @@ -81,7 +81,7 @@ function functionOutput(content: unknown, isError: boolean): string | JsonObject else if (part?.type === 'image') { const image_url = imageUrl(part.source); if (!image_url) invalidRequest('Unsupported tool_result image source'); - pieces.push({ type: 'input_image', image_url, detail: 'original' }); + pieces.push({ type: 'input_image', image_url, detail: 'high' }); } else { invalidRequest(`Unsupported tool_result content block: ${String(part?.type ?? 'invalid')}`); } diff --git a/tests/proxy-usage.test.ts b/tests/proxy-usage.test.ts index f7e0b5bbe..27466b252 100644 --- a/tests/proxy-usage.test.ts +++ b/tests/proxy-usage.test.ts @@ -652,7 +652,7 @@ describe('proxy usage extraction', () => { expect(output).toEqual([ { type: 'input_text', text: '[Tool execution failed]' }, { type: 'input_text', text: 'failed' }, - { type: 'input_image', image_url: 'data:image/png;base64,YWJj', detail: 'original' }, + { type: 'input_image', image_url: 'data:image/png;base64,YWJj', detail: 'high' }, ]); }); From 83c8911dbc39ef1ea84ae48f1943da9dc9a748f6 Mon Sep 17 00:00:00 2001 From: Steven Chong <25894545+teamchong@users.noreply.github.com> Date: Thu, 20 Aug 2026 12:51:46 -0400 Subject: [PATCH 05/11] fix(openai): count slab images once in the history-collapse budget info.imageCount already tracked the static slab, and the slabs are inserted into req.messages before applyChatHistoryCollapse runs, so adding both double-counted them. With 14 slab images the budget went 28 > 24 and remainingMax clamped to 0, disabling history collapse for Qwen-sized contexts. Walk the request messages only, matching the Responses path. --- src/core/openai.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core/openai.ts b/src/core/openai.ts index 5f6ce6ef6..6bc3f0e31 100644 --- a/src/core/openai.ts +++ b/src/core/openai.ts @@ -826,7 +826,11 @@ async function applyChatHistoryCollapse( ): Promise { const profitable = (text: string, cols: number, baselineTextTokens?: number) => evalOpenAIGate(req.model, text, cols, o.charsPerToken, baselineTextTokens).profitable; - const existingImages = (info.imageCount ?? 0) + countRequestImages(req.messages); + // Count images already in the request. At the main call site the static slab + // images are inserted into req.messages before this runs, so walking the + // messages (instead of also adding info.imageCount) avoids double-counting + // the slab and matches the Responses path's budget math. + const existingImages = countRequestImages(req.messages); const plan = await planGptCollapse( chatMessagesToTurns(req.messages), protectedPrefix, From cf07f44b0394f88dc574002c09fe7201bddc04b2 Mon Sep 17 00:00:00 2001 From: Steven Chong <25894545+teamchong@users.noreply.github.com> Date: Thu, 20 Aug 2026 12:51:48 -0400 Subject: [PATCH 06/11] fix(bridge): pick image detail from the request model GPT-5 keeps detail:'original'; every other upstream (Workers AI, GPT-4o, ...) validates against auto|low|high and 400s on 'original'. The bridge already has req.model, so derive the detail from it instead of hard-coding. --- src/core/messages-responses-bridge.ts | 22 +++++++++++++------- tests/proxy-usage.test.ts | 30 +++++++++++++-------------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/core/messages-responses-bridge.ts b/src/core/messages-responses-bridge.ts index 1deedc756..7d2bb64c6 100644 --- a/src/core/messages-responses-bridge.ts +++ b/src/core/messages-responses-bridge.ts @@ -36,7 +36,14 @@ function imageUrl(source: unknown): string | undefined { return undefined; } -function inputParts(content: unknown, location = 'message', role = 'user'): JsonObject[] { +/** Mirrors isGpt5Family in src/core/openai.ts: only the GPT-5 family accepts + * detail:'original'; every other upstream (Workers AI, GPT-4o, …) validates + * detail against 'auto'|'low'|'high' and 400s on 'original'. */ +function imageDetailFor(model: unknown): string { + return typeof model === 'string' && /^gpt-5/i.test(model) ? 'original' : 'high'; +} + +function inputParts(content: unknown, location = 'message', role = 'user', imageDetail = 'high'): JsonObject[] { // Responses requires assistant-role message text to be `output_text`; // `input_text` is only valid for user/system. Emitting input_text under // role:"assistant" (any replayed assistant turn) is a 400. @@ -51,7 +58,7 @@ function inputParts(content: unknown, location = 'message', role = 'user'): Json } else if (part?.type === 'image') { const image_url = imageUrl(part.source); if (!image_url) invalidRequest(`Unsupported ${location} image source`); - out.push({ type: 'input_image', image_url, detail: 'high' }); + out.push({ type: 'input_image', image_url, detail: imageDetail }); } else { invalidRequest(`Unsupported ${location} content block: ${String(part?.type ?? 'invalid')}`); } @@ -59,7 +66,7 @@ function inputParts(content: unknown, location = 'message', role = 'user'): Json return out; } -function functionOutput(content: unknown, isError: boolean): string | JsonObject[] { +function functionOutput(content: unknown, isError: boolean, imageDetail = 'high'): string | JsonObject[] { if (typeof content === 'string') { return isError ? [{ type: 'input_text', text: '[Tool execution failed]' }, { type: 'input_text', text: content }] @@ -81,7 +88,7 @@ function functionOutput(content: unknown, isError: boolean): string | JsonObject else if (part?.type === 'image') { const image_url = imageUrl(part.source); if (!image_url) invalidRequest('Unsupported tool_result image source'); - pieces.push({ type: 'input_image', image_url, detail: 'high' }); + pieces.push({ type: 'input_image', image_url, detail: imageDetail }); } else { invalidRequest(`Unsupported tool_result content block: ${String(part?.type ?? 'invalid')}`); } @@ -106,6 +113,7 @@ function mapToolChoice(value: unknown): unknown { /** Convert a Claude Code Messages request into an OpenAI Responses request. */ export function anthropicMessagesToOpenAIResponses(body: Uint8Array): Uint8Array { const req = JSON.parse(new TextDecoder().decode(body)) as JsonObject; + const imageDetail = imageDetailFor(req.model); const input: JsonObject[] = []; if (Array.isArray(req.messages)) { @@ -124,7 +132,7 @@ export function anthropicMessagesToOpenAIResponses(body: Uint8Array): Uint8Array } const content = message.content; if (!Array.isArray(content)) { - const ordinary = inputParts(content, `${String(message.role)} message`, String(message.role)); + const ordinary = inputParts(content, `${String(message.role)} message`, String(message.role), imageDetail); if (ordinary.length) input.push({ role: message.role, content: ordinary }); continue; } @@ -150,10 +158,10 @@ export function anthropicMessagesToOpenAIResponses(body: Uint8Array): Uint8Array input.push({ type: 'function_call_output', call_id: part.tool_use_id, - output: functionOutput(part.content, part.is_error === true), + output: functionOutput(part.content, part.is_error === true, imageDetail), }); } else { - ordinary.push(...inputParts([rawPart], `${String(message.role)} message`, String(message.role))); + ordinary.push(...inputParts([rawPart], `${String(message.role)} message`, String(message.role), imageDetail)); } } flushOrdinary(); diff --git a/tests/proxy-usage.test.ts b/tests/proxy-usage.test.ts index 27466b252..013d4bf5b 100644 --- a/tests/proxy-usage.test.ts +++ b/tests/proxy-usage.test.ts @@ -105,7 +105,7 @@ describe('proxy usage extraction', () => { let captured: ProxyEvent | undefined; const proxy = createProxy({ - upstream: 'http://ocproxy.test', + upstream: 'http://gateway.test', transform: { compress: false }, onRequest: (event) => { captured = event; @@ -157,7 +157,7 @@ describe('proxy usage extraction', () => { }); let captured: ProxyEvent | undefined; const proxy = createProxy({ - upstream: 'http://ocproxy.test', + upstream: 'http://gateway.test', transform: { compress: true }, onRequest: (event) => { captured = event; }, }); @@ -204,7 +204,7 @@ describe('proxy usage extraction', () => { }); let captured: ProxyEvent | undefined; const proxy = createProxy({ - upstream: 'http://ocproxy.test', + upstream: 'http://gateway.test', transform: { compress: true }, onRequest: (event) => { captured = event; }, }); @@ -238,7 +238,7 @@ describe('proxy usage extraction', () => { }); let captured: ProxyEvent | undefined; const proxy = createProxy({ - upstream: 'http://ocproxy.test', + upstream: 'http://gateway.test', transform: { compress: true }, onRequest: (event) => { captured = event; }, }); @@ -284,7 +284,7 @@ describe('proxy usage extraction', () => { const restore = restoreFetch; let captured: ProxyEvent | undefined; const proxy = createProxy({ - upstream: 'http://ocproxy.test', + upstream: 'http://gateway.test', transform: { compress: true }, onRequest: (event) => { captured = event; }, }); @@ -321,7 +321,7 @@ describe('proxy usage extraction', () => { }); let captured: ProxyEvent | undefined; const proxy = createProxy({ - upstream: 'http://ocproxy.test', + upstream: 'http://gateway.test', transform: { compress: true }, onRequest: (event) => { captured = event; }, }); @@ -354,7 +354,7 @@ describe('proxy usage extraction', () => { }); let captured: ProxyEvent | undefined; const proxy = createProxy({ - upstream: 'http://ocproxy.test', + upstream: 'http://gateway.test', onRequest: (event) => { captured = event; }, }); const body = JSON.stringify({ contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }); @@ -430,7 +430,7 @@ describe('proxy usage extraction', () => { let captured: ProxyEvent | undefined; const proxy = createProxy({ - upstream: 'http://ocproxy.test', + upstream: 'http://gateway.test', apiKey: 'sk-anthropic-test', transform: { charsPerToken: 1, minCompressChars: 1 }, onRequest: (e) => { @@ -456,13 +456,13 @@ describe('proxy usage extraction', () => { await new Promise((r) => setTimeout(r, 20)); restore(); - const main = upstreamRequests.find((r) => r.url === 'http://ocproxy.test/anthropic/messages'); + const main = upstreamRequests.find((r) => r.url === 'http://gateway.test/anthropic/messages'); expect(main).toBeDefined(); expect(captured?.model).toBe('claude-fable-5'); expect(captured?.info?.compressed).toBe(true); // count_tokens probe mirrors the request path under the same prefix. expect( - upstreamRequests.some((r) => r.url === 'http://ocproxy.test/anthropic/messages/count_tokens'), + upstreamRequests.some((r) => r.url === 'http://gateway.test/anthropic/messages/count_tokens'), ).toBe(true); }); @@ -652,7 +652,7 @@ describe('proxy usage extraction', () => { expect(output).toEqual([ { type: 'input_text', text: '[Tool execution failed]' }, { type: 'input_text', text: 'failed' }, - { type: 'input_image', image_url: 'data:image/png;base64,YWJj', detail: 'high' }, + { type: 'input_image', image_url: 'data:image/png;base64,YWJj', detail: 'original' }, ]); }); @@ -1206,7 +1206,7 @@ describe('proxy usage extraction', () => { }); const proxy = createProxy({ - upstream: 'http://ocproxy.test', + upstream: 'http://gateway.test', openAIUpstream: 'https://api.openai.test', transform: { charsPerToken: 1, minCompressChars: 1 }, }); @@ -1230,7 +1230,7 @@ describe('proxy usage extraction', () => { restore(); expect(upstreamRequests).toHaveLength(1); - expect(upstreamRequests[0]!.url).toBe('http://ocproxy.test/openai/v1/chat/completions'); + expect(upstreamRequests[0]!.url).toBe('http://gateway.test/openai/v1/chat/completions'); expect(upstreamRequests[0]!.headers.get('authorization')).toBe('Bearer local-token'); const sent = JSON.parse(await upstreamRequests[0]!.text()) as any; const firstUser = sent.messages.find((m: any) => m.role === 'user'); @@ -1254,7 +1254,7 @@ describe('proxy usage extraction', () => { let captured: ProxyEvent | undefined; const proxy = createProxy({ - upstream: 'http://ocproxy.test', + upstream: 'http://gateway.test', openAIUpstream: 'https://api.openai.test', transform: { charsPerToken: 1, minCompressChars: 1 }, onRequest: (e) => { @@ -1280,7 +1280,7 @@ describe('proxy usage extraction', () => { restore(); expect(upstreamRequests).toHaveLength(1); - expect(upstreamRequests[0]!.url).toBe('http://ocproxy.test/openai/responses'); + expect(upstreamRequests[0]!.url).toBe('http://gateway.test/openai/responses'); expect(upstreamRequests[0]!.headers.get('authorization')).toBe('Bearer local-token'); const sent = JSON.parse(await upstreamRequests[0]!.text()) as any; const firstUser = sent.input.find((m: any) => m.role === 'user'); From 4e9f97f902886ad18cd983cf5a2b6468b535b150 Mon Sep 17 00:00:00 2001 From: Steven Chong <25894545+teamchong@users.noreply.github.com> Date: Thu, 20 Aug 2026 12:51:49 -0400 Subject: [PATCH 07/11] test(openai-gpt5): guard find() results instead of non-null assertions --- tests/openai-gpt5.test.ts | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/openai-gpt5.test.ts b/tests/openai-gpt5.test.ts index a738c2072..97f8dc92e 100644 --- a/tests/openai-gpt5.test.ts +++ b/tests/openai-gpt5.test.ts @@ -5,7 +5,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { isPxpipeSupportedGptModel } from '../src/core/applicability.js'; import { openAIVisionTokens, visionTokensForModel, isClaudeModel, resolveVisionCost, transformOpenAIChatCompletions, transformOpenAIResponses } from '../src/core/openai.js'; -import { resolveGptProfile } from '../src/core/gpt-model-profiles.js'; +import { resolveGptProfile, isMisresolvedModelId } from '../src/core/gpt-model-profiles.js'; const enc = new TextEncoder(); const dec = new TextDecoder(); @@ -1028,7 +1028,8 @@ describe('image parts request detail = "original" (avoid downscale of dense text const result = await transformOpenAIChatCompletions(body, { charsPerToken: 1, minCompressChars: 1 }); expect(result.info.compressed).toBe(true); const out = JSON.parse(dec.decode(result.body)) as { messages: Array<{ role: string; content: unknown }> }; - const firstUser = out.messages.find((m) => m.role === 'user')!; + const firstUser = out.messages.find((m) => m.role === 'user'); + if (!firstUser) throw new Error('transformed output has no user item'); const parts = firstUser.content as Array<{ type: string; image_url?: { detail?: string } }>; const imgs = parts.filter((p) => p.type === 'image_url'); expect(imgs.length).toBeGreaterThan(0); @@ -1046,7 +1047,8 @@ describe('image parts request detail = "original" (avoid downscale of dense text const result = await transformOpenAIChatCompletions(body, { charsPerToken: 1, minCompressChars: 1 }); expect(result.info.compressed).toBe(true); const out = JSON.parse(dec.decode(result.body)) as { messages: Array<{ role: string; content: unknown }> }; - const firstUser = out.messages.find((m) => m.role === 'user')!; + const firstUser = out.messages.find((m) => m.role === 'user'); + if (!firstUser) throw new Error('transformed output has no user item'); const parts = firstUser.content as Array<{ type: string; image_url?: { detail?: string } }>; const imgs = parts.filter((p) => p.type === 'image_url'); expect(imgs.length).toBeGreaterThan(0); @@ -1062,7 +1064,8 @@ describe('image parts request detail = "original" (avoid downscale of dense text const result = await transformOpenAIResponses(body, { charsPerToken: 1, minCompressChars: 1 }); expect(result.info.compressed).toBe(true); const out = JSON.parse(dec.decode(result.body)) as { input: Array<{ role?: string; content?: unknown }> }; - const firstUser = out.input.find((m) => m.role === 'user')!; + const firstUser = out.input.find((m) => m.role === 'user'); + if (!firstUser) throw new Error('transformed output has no user item'); const parts = firstUser.content as Array<{ type: string; detail?: string }>; const imgs = parts.filter((p) => p.type === 'input_image'); expect(imgs.length).toBeGreaterThan(0); @@ -1078,7 +1081,8 @@ describe('image parts request detail = "original" (avoid downscale of dense text const result = await transformOpenAIResponses(body, { charsPerToken: 1, minCompressChars: 1 }); expect(result.info.compressed).toBe(true); const out = JSON.parse(dec.decode(result.body)) as { input: Array<{ role?: string; content?: unknown }> }; - const firstUser = out.input.find((m) => m.role === 'user')!; + const firstUser = out.input.find((m) => m.role === 'user'); + if (!firstUser) throw new Error('transformed output has no user item'); const parts = firstUser.content as Array<{ type: string; detail?: string }>; const imgs = parts.filter((p) => p.type === 'input_image'); expect(imgs.length).toBeGreaterThan(0); @@ -1215,7 +1219,15 @@ describe('resolveGptProfile (Qwen)', () => { expect(p.maxHeightPx).toBe(512); expect(p.style.font).toBe('jetbrains-mono-14'); expect(p.history.maxImages).toBe(24); - expect(resolveGptProfile('qwen-3.8-27b').stripCols).toBe(84); + // Keyed to the exact measured model id, not to any 'qwen' substring. + expect(resolveGptProfile('qwen3.8-27b').stripCols).toBe(84); + }); + + it('refuses unmeasured Qwen variants instead of applying this profile', () => { + for (const id of ['qwen2.5-72b-instruct', 'qwen3-30b', 'qwen-3.8-27b']) { + expect(resolveGptProfile(id).stripCols).not.toBe(84); + expect(isMisresolvedModelId(id)).toBe(true); + } }); }); From 1693de1f5af0aa33fbfa6f7d7c99cf634840014b Mon Sep 17 00:00:00 2001 From: Steven Chong <25894545+teamchong@users.noreply.github.com> Date: Thu, 20 Aug 2026 12:51:50 -0400 Subject: [PATCH 08/11] feat(profiles): key the Qwen profile to qwen3.8-27b exactly The measured 14px/84-col/512px geometry is specific to Qwen 3.8 27B. Other Qwen variants now fall to the family-id guard (refused) instead of being gated with an unmeasured profile. --- src/core/gpt-model-profiles.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/core/gpt-model-profiles.ts b/src/core/gpt-model-profiles.ts index 24a8608f0..2ab431121 100644 --- a/src/core/gpt-model-profiles.ts +++ b/src/core/gpt-model-profiles.ts @@ -229,8 +229,10 @@ const isMiniNanoPatch = (m: string): boolean => /** Grok ids pxpipe has a measured profile for. */ const isGrokModel = (m: string): boolean => /^grok-/.test(m); -/** Qwen ids pxpipe has a measured profile for (e.g. Qwen 3.8 / Qwen 2.5). */ -const isQwenModel = (m: string): boolean => /qwen/i.test(m); +/** Qwen 3.8 27B ids — the only Qwen geometry pxpipe has measured. Other Qwen + * variants deliberately do NOT match: the family-id guard below refuses them + * instead of gating an unmeasured model with this profile. */ +const isQwenModel = (m: string): boolean => /qwen3\.8-27b/i.test(m); /** Shared GPT geometry for the small patch-billed models; only the patch * multiplier and the family list prices differ between the rules below. */ @@ -327,7 +329,7 @@ const BUILTIN_RULES: ProfileRule[] = [ }, }, - // Qwen (Workers AI / open weights). Native 14px / 84 cols / maxH 512 is required + // Qwen 3.8 27B (Workers AI / open weights). Native 14px / 84 cols / maxH 512 is required // because 5x8 bitmap glyphs are illegible to Qwen vision (0/15 hex vs 11/15 on 14px). // History image count capped at 24 images to stay safely below Workers AI's 32-image limit. { From 6b536d8452e3308ba9783485c55d731836d3b67d Mon Sep 17 00:00:00 2001 From: Steven Chong <25894545+teamchong@users.noreply.github.com> Date: Thu, 20 Aug 2026 12:51:52 -0400 Subject: [PATCH 09/11] chore: drop private gateway references from code and docs The local gateway is private tooling; the repo now references it only as 'a local gateway' with neutral example ports, and eval clients require OPENAI_BASE_URL instead of assuming a default. --- README.md | 2 +- eval/gemini-profile/gemini-client.mjs | 3 ++- eval/gist-recall/OPUS_14PX.md | 2 +- eval/glyph-matrix/PLAN.md | 2 +- eval/grok-density/QUALITY_SUITE.md | 2 +- eval/grok-density/native-sweep/README.md | 2 +- eval/grok-density/native-sweep/RESULTS.md | 4 ++-- eval/grok-density/native-sweep/ask.mjs | 2 +- eval/grok-profile/QUALITY_RESULTS.md | 2 +- eval/qwen-profile/QUALITY_RESULTS.md | 2 +- eval/qwen-profile/qwen-client.mjs | 18 ++++++------------ eval/sol-profile/novel-arithmetic-results.json | 2 +- eval/sol-profile/run.mjs | 2 +- src/core/proxy.ts | 2 +- src/core/transform.ts | 2 +- src/dashboard/fragments.ts | 2 +- src/node.ts | 2 +- src/warp/route.ts | 2 +- tests/gateway.test.ts | 8 ++++---- tests/warp-route.test.ts | 8 ++++---- 20 files changed, 33 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index db2038e61..612de7af0 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ provider over some other base URL need a rule for it, and a rule that names a port matches only that port: ```bash -pxpipe warp --route '127.0.0.1:8082/v1/*=http://127.0.0.1:47821' -- codex +pxpipe warp --route '127.0.0.1:9090/v1/*=http://127.0.0.1:47821' -- codex ``` ## Offline export (no proxy) diff --git a/eval/gemini-profile/gemini-client.mjs b/eval/gemini-profile/gemini-client.mjs index e6cabf769..e29ef66f8 100644 --- a/eval/gemini-profile/gemini-client.mjs +++ b/eval/gemini-profile/gemini-client.mjs @@ -1,7 +1,8 @@ // Google AI Studio eval client through the local Cloudflare AI Gateway. function gatewayOrigin() { - const base = (process.env.OPENAI_BASE_URL || 'http://127.0.0.1:8082/v1').replace(/\/$/, ''); + const base = (process.env.OPENAI_BASE_URL ?? '').replace(/\/$/, ''); + if (!base) throw new Error('OPENAI_BASE_URL is required (point it at your gateway)'); return new URL(base).origin; } diff --git a/eval/gist-recall/OPUS_14PX.md b/eval/gist-recall/OPUS_14PX.md index f5cfa41b1..28fba3df2 100644 --- a/eval/gist-recall/OPUS_14PX.md +++ b/eval/gist-recall/OPUS_14PX.md @@ -20,7 +20,7 @@ density, matching how fable's matrix cells are scored at fable's `5x8`. pageWidthPx}`; `run*.py` **preflight-aborts before any API call** if the corpus manifest model ≠ the model under test (guards silent profile/model mismatch). -- Model reached via **upstream (`:8082`), pxpipe-bypassed** — so pxpipe's own +- Model reached via **the gateway upstream, pxpipe-bypassed** — so pxpipe's own text→image rendering does not contaminate the arms; the model still gets the images. Concurrency 3, `CCI_READY_TIMEOUT=120`, `CCI_TIMEOUT=210`. - Raw results: `work/results.opus-14px.jsonl`, `work2/…`, `work3/…`. diff --git a/eval/glyph-matrix/PLAN.md b/eval/glyph-matrix/PLAN.md index c5ae2c412..b715b5822 100644 --- a/eval/glyph-matrix/PLAN.md +++ b/eval/glyph-matrix/PLAN.md @@ -7,7 +7,7 @@ ## What this measures Per-character confusion matrix for 12-char hex IDs read back through the -pixelpipe render path, across render-style arms — the precision-tier +pxpipe render path, across render-style arms — the precision-tier failure mode that gist-recall (98/98) structurally cannot surface and needle-haystack only bracketed (~87% exact-match on worst-case hex at prod style). diff --git a/eval/grok-density/QUALITY_SUITE.md b/eval/grok-density/QUALITY_SUITE.md index 9aefabb21..0804aa05f 100644 --- a/eval/grok-density/QUALITY_SUITE.md +++ b/eval/grok-density/QUALITY_SUITE.md @@ -4,7 +4,7 @@ Grok is evaluated through the OpenAI-compatible Responses endpoint used by Codex. Fable and Opus use the Claude harnesses. ```bash -export OPENAI_BASE_URL=http://127.0.0.1:8082/v1 +export OPENAI_BASE_URL=http://127.0.0.1:/v1 export OPENAI_API_KEY=… export SOL_QUALITY_MODEL=grok-4.5 export SOL_QUALITY_LIVE=1 diff --git a/eval/grok-density/native-sweep/README.md b/eval/grok-density/native-sweep/README.md index 50eb2bacc..42dccdd74 100644 --- a/eval/grok-density/native-sweep/README.md +++ b/eval/grok-density/native-sweep/README.md @@ -10,7 +10,7 @@ atlas swap → rebuild → render, live ask locks answers before truth is scored ```bash node eval/grok-density/native-sweep/gen-fixture.mjs node eval/grok-density/native-sweep/render-ladder.mjs -OPENAI_BASE_URL=http://127.0.0.1:8082/v1 OPENAI_API_KEY=… \ +OPENAI_BASE_URL=http://127.0.0.1:/v1 OPENAI_API_KEY=… \ bash eval/grok-density/native-sweep/_ask_all.sh node eval/grok-density/native-sweep/score-all.mjs ``` diff --git a/eval/grok-density/native-sweep/RESULTS.md b/eval/grok-density/native-sweep/RESULTS.md index 67fbbdc06..cfa54e8a0 100644 --- a/eval/grok-density/native-sweep/RESULTS.md +++ b/eval/grok-density/native-sweep/RESULTS.md @@ -1,7 +1,7 @@ # Grok 4.5 native JetBrains Mono 8–16px blind sweep Date: 2026-07-23. Model: `grok-4.5` via direct Responses (`OPENAI_BASE_URL` -upstream `:8082`, pxpipe port 47821 rejected by the client). +upstream local gateway; pxpipe port 47821 rejected by the client). ## Protocol (same spirit as Opus / Sol native sweeps) @@ -58,7 +58,7 @@ evidence; this sweep is the native-size negative result. # atlases already built under eval/opus-density/native-sweep/atlases (symlinked) node eval/grok-density/native-sweep/gen-fixture.mjs # fresh truth (optional) node eval/grok-density/native-sweep/render-ladder.mjs # swaps atlas*, rebuilds, restores -OPENAI_BASE_URL=http://127.0.0.1:8082/v1 OPENAI_API_KEY=… \ +OPENAI_BASE_URL=http://127.0.0.1:/v1 OPENAI_API_KEY=… \ bash eval/grok-density/native-sweep/_ask_all.sh # locks answers-*.json node eval/grok-density/native-sweep/score-all.mjs ``` diff --git a/eval/grok-density/native-sweep/ask.mjs b/eval/grok-density/native-sweep/ask.mjs index 09e4cdc49..e207ed06a 100644 --- a/eval/grok-density/native-sweep/ask.mjs +++ b/eval/grok-density/native-sweep/ask.mjs @@ -1,6 +1,6 @@ // Live-call grok-4.5 on one rendered rung. Blind: never opens truth.json. // Writes answers-