+ Issue #65 · dev-only · WebGPU required · not bundled into production.
+
+
+ Runs the selected model against every prompt variant against every
+ fixture under tests/fixtures/rewrite/. Deterministic
+ rubric only — no judge model is loaded.
+
+
+ One model per tab on purpose: cycling multiple multi-GB models in a
+ single browser tab is fragile on consumer GPUs (eviction-then-reload
+ can OOM). Run each model in a fresh tab, download its report, then
+ open a new tab for the next one.
+
+ The downloaded report goes under
+ tests/fixtures/rewrite/reports/ when you commit a fresh
+ run. See src/lib/webllm/eval/README.md for the
+ report-commit workflow.
+
+
+
+
diff --git a/package.json b/package.json
index 72af2251..bef84df8 100644
--- a/package.json
+++ b/package.json
@@ -11,6 +11,7 @@
"test": "vitest run",
"test:watch": "vitest",
"bake-fixtures": "UPDATE_FIXTURES=1 vitest run src/lib/heuristics/corpus.test.ts",
+ "eval:rewrite": "vite --open=/resumelint/eval-rewrite.html",
"typecheck": "tsc -b --noEmit",
"lint": "eslint .",
"deploy": "./scripts/deploy_resumelint.sh",
diff --git a/scripts/run_resumelint.sh b/scripts/run_resumelint.sh
index 29342592..ab711e5d 100755
--- a/scripts/run_resumelint.sh
+++ b/scripts/run_resumelint.sh
@@ -8,16 +8,17 @@ set -euo pipefail
# Commands: ./scripts/run_resumelint.sh [args]
#
# Commands:
-# dev Start Vite dev server (http://localhost:5173)
-# build Build static bundle into dist/
-# preview Serve the built bundle (builds first if dist/ is missing)
-# test Run vitest
-# test:watch Run vitest in watch mode
-# typecheck tsc -b --noEmit (lint alias)
-# install npm install
-# clean Remove dist/ and node_modules/ (asks for confirmation)
-# deploy Build and deploy to GCS — forwards args to deploy_resumelint.sh
-# (e.g. ./scripts/run_resumelint.sh deploy --dry-run)
+# dev Start Vite dev server (http://localhost:5173)
+# build Build static bundle into dist/
+# preview Serve the built bundle (builds first if dist/ is missing)
+# test Run vitest
+# test:watch Run vitest in watch mode
+# typecheck tsc -b --noEmit (lint alias)
+# install npm install
+# clean Remove dist/ and node_modules/ (asks for confirmation)
+# eval:rewrite Open the dev-only rewrite-quality eval page (#65). WebGPU required.
+# deploy Build and deploy to GCS — forwards args to deploy_resumelint.sh
+# (e.g. ./scripts/run_resumelint.sh deploy --dry-run)
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
@@ -85,6 +86,14 @@ cmd_clean() {
npm_clean "$WEB_DIR"
}
+cmd_eval_rewrite() {
+ cd "$WEB_DIR"
+ ensure_npm_deps "$WEB_DIR"
+ log_info "Opening rewrite-eval page on http://localhost:$DEV_PORT/resumelint/eval-rewrite.html"
+ log_info "Ctrl+C to stop the dev server when done. WebGPU required to run inference."
+ npm run eval:rewrite
+}
+
cmd_deploy() {
# Forward any remaining args (e.g. --dry-run, --mode=modified) verbatim.
if [[ ! -x "$DEPLOY_SCRIPT" ]]; then
@@ -111,6 +120,7 @@ ${BLUE}4)${NC} Run tests (vitest run)
${BLUE}5)${NC} Test watch (vitest, watch mode)
${BLUE}6)${NC} Typecheck (tsc -b --noEmit)
${BLUE}7)${NC} Install deps (npm install)
+${BLUE}e)${NC} Rewrite eval page (dev-only, WebGPU; issue #65)
${BLUE}d)${NC} Deploy to GCS (scripts/deploy_resumelint.sh)
${BLUE}p)${NC} Deploy --dry-run (preview what would upload)
${BLUE}c)${NC} Clean (rm -rf dist/ node_modules/)
@@ -134,6 +144,7 @@ interactive_menu() {
5) cmd_test_watch || true ;;
6) cmd_typecheck || true ;;
7) cmd_install || true ;;
+ e|E) cmd_eval_rewrite || true ;;
d|D) cmd_deploy || true ;;
p|P) cmd_deploy --dry-run || true ;;
c|C) cmd_clean || true ;;
@@ -142,7 +153,7 @@ interactive_menu() {
esac
# Skip the Enter-prompt for foregrounded long-runners — they already
# blocked until the user was ready to come back.
- if [[ ! "$choice" =~ ^[135]$ ]] && [[ "${choice:-}" != "p" ]] && [[ "${choice:-}" != "P" ]]; then
+ if [[ ! "$choice" =~ ^[135]$ ]] && [[ "${choice:-}" != "p" ]] && [[ "${choice:-}" != "P" ]] && [[ "${choice:-}" != "e" ]] && [[ "${choice:-}" != "E" ]]; then
echo ""
read -rp "Press Enter to continue..."
fi
@@ -167,13 +178,14 @@ else
typecheck|lint) cmd_typecheck ;;
install) cmd_install ;;
clean) cmd_clean ;;
+ eval:rewrite|eval-rewrite) cmd_eval_rewrite ;;
deploy) cmd_deploy "$@" ;;
-h|--help|help)
sed -n '3,21p' "${BASH_SOURCE[0]}"
;;
*)
log_error "Unknown command: $subcommand"
- echo "Usage: $0 [dev|build|preview|test|test:watch|typecheck|install|clean|deploy [args...]]"
+ echo "Usage: $0 [dev|build|preview|test|test:watch|typecheck|install|clean|eval:rewrite|deploy [args...]]"
echo " $0 (interactive menu)"
exit 1
;;
diff --git a/src/hooks/useModelSelection.integration.test.tsx b/src/hooks/useModelSelection.integration.test.tsx
index 87350fda..de1a110a 100644
--- a/src/hooks/useModelSelection.integration.test.tsx
+++ b/src/hooks/useModelSelection.integration.test.tsx
@@ -67,7 +67,12 @@ function Probe({
}
beforeEach(() => {
- localStorage.clear();
+ // Node 22+ ships a built-in global `localStorage` that shadows jsdom's
+ // `Storage` and exposes no `clear()` method, so a bare `localStorage.clear()`
+ // throws on newer runtimes (green on CI's Node 20, red on Node 25 locally).
+ // Optional-chain it to match the store's own defensive access; the per-key
+ // cleanup that actually matters is done by the reset helper below.
+ globalThis.localStorage?.clear?.();
_resetPersistedModelSelectionForTesting();
});
diff --git a/src/lib/score/score.ts b/src/lib/score/score.ts
index ded959f5..78fcc7ef 100644
--- a/src/lib/score/score.ts
+++ b/src/lib/score/score.ts
@@ -142,7 +142,19 @@ function bulletHasMetric(text: string): boolean {
return ANY_DIGIT.test(stripped);
}
-const ACTION_VERBS = new Set([
+/**
+ * Curated past-tense action verbs used to grade the user's *existing*
+ * bullets. Exported so the rewrite eval (`src/lib/webllm/eval/verbs.ts`)
+ * can reuse this as the base set without duplicating it — the eval set
+ * adds present-tense and IC-discipline verbs on top, but the scorer's
+ * specificity-dimension semantics stay anchored here.
+ *
+ * Kept narrow on purpose: weak generic verbs ("worked", "helped",
+ * "supported", "responsible", "assisted", "participated") are deliberately
+ * NOT here. A bullet leading with one of those SHOULD fail the
+ * specificity check.
+ */
+export const ACTION_VERBS: ReadonlySet = new Set([
"led", "managed", "developed", "built", "designed", "implemented",
"created", "launched", "drove", "increased", "reduced", "improved",
"delivered", "established", "optimized", "architected", "scaled",
diff --git a/src/lib/webllm/eval/README.md b/src/lib/webllm/eval/README.md
new file mode 100644
index 00000000..6e1af272
--- /dev/null
+++ b/src/lib/webllm/eval/README.md
@@ -0,0 +1,114 @@
+# Rewrite-quality eval harness
+
+Phase 3 of the in-browser AI rewrite epic (issue #65). Scores
+section-rewrite outputs against a deterministic rubric so the default
+model + prompt are picked from measurement rather than vibes.
+
+## Layout
+
+```
+src/lib/webllm/eval/
+├── types.ts # FixtureKind, RubricResult, EvalReport, RewriteFn
+├── verbs.ts # curated action-verb set (superset of scorer's)
+├── fixtures.ts # loads + validates JSON fixtures
+├── rubric.ts # the six deterministic criteria
+├── prompt-variants.ts # the shipped prompt + experimental variants
+├── runner.ts # iterates (model × variant × fixture)
+├── report.ts # JSON + Markdown formatters
+└── run-eval-browser.ts # browser entry — wires real WebLLM engine
+```
+
+Fixtures live under `tests/fixtures/rewrite/`; reports get committed to
+`tests/fixtures/rewrite/reports/`.
+
+## Two execution legs
+
+### 1. Scoring leg (CI)
+
+Pure scoring logic — rubric, runner, formatters, fixture loading — all
+unit-tested under `*.test.ts` siblings. Runs in the default
+`npm run test` and is exercised on every PR via the existing CI gate.
+No model, no WebGPU, no network.
+
+### 2. Inference leg (local, WebGPU)
+
+Real models run only in a browser. The entry point is the dev-only
+`eval-rewrite.html` page at the project root:
+
+```sh
+npm run eval:rewrite
+# opens http://localhost:5173/resumelint/eval-rewrite.html
+```
+
+**One model per tab.** The page asks you to pick a model from the
+dropdown, then click **Run eval** — it loads that model only, runs every
+prompt variant against every fixture, scores with the rubric, and
+exposes JSON + Markdown report downloads. To compare another model,
+open a fresh tab (or refresh) and pick a different one.
+
+This is intentional: cycling several multi-GB models in a single tab
+kept crashing Chrome on consumer GPUs during the WebGPU
+eviction-then-reload path. Closing and reopening the tab between
+models reclaims VRAM cleanly. The downside is the maintainer commits
+one report file per model and reviewers compare them side-by-side —
+still cheap.
+
+Each downloaded report includes the model slug in the filename
+(`eval-rewrite-qwen2-5-1-5b-…-{timestamp}.{json,md}`) so the three
+per-model files coexist under `tests/fixtures/rewrite/reports/` without
+collision. Reports are append-only — never overwrite a prior run.
+
+`eval-rewrite.html` is NOT included in `build.rollupOptions.input`, so
+the production bundle is unaffected.
+
+## Reading the report
+
+The Markdown report leads with a per-`(model, variant)` aggregate row.
+Six rates (numbers / one-line / verb / length / no-preamble / dedup) and
+the equal-weight composite `Aggregate` column drive the model choice.
+Per-cell records below the aggregate let you trace a failure to a
+specific fixture.
+
+The dedup column is `—` for non-redundant fixtures (the criterion
+doesn't apply); the aggregate's dedup rate is computed over `redundant`
+fixtures only.
+
+The judge column is `—` until the optional LLM-judge gate is enabled.
+That path is flag-plumbed (`runEval({ judgeEnabled })`) but the
+implementation is intentionally stubbed — coherence judging is a follow-up.
+
+## Adding a fixture
+
+Drop a JSON file under `tests/fixtures/rewrite/` with this shape:
+
+```json
+{
+ "id": "kebab-case-id",
+ "kind": "weak | strong | numeric | redundant",
+ "description": "What this fixture stresses, for the report's prose.",
+ "bullets": ["...", "..."]
+}
+```
+
+Then append an `import` + entry in `fixtures.ts::REWRITE_FIXTURES`.
+`parseFixture` validates shape at module load — a malformed fixture
+throws with a precise pointer before any eval runs.
+
+**PII policy still applies.** Bullet fixtures are persona-free by
+construction (no contact info), but keep employer names, dates, and
+résumé details synthetic. The repo is public.
+
+## Adding a prompt variant
+
+Append to `prompt-variants.ts::PROMPT_VARIANTS`. The runner enumerates
+the array; the browser entry picks all of them up automatically. Keep
+deltas small — one or two rule changes per variant — so a regression in
+any one criterion traces cleanly to the prompt change.
+
+## Choosing a default model
+
+The aggregate's `Aggregate` column is the equal-weight mean of the
+deterministic rates. If two models tie within ~3 points, prefer the
+smaller / Apache-2.0 one — the eval is a measurement floor, not the only
+input (license, download size, and consent friction matter for the
+shipped default).
diff --git a/src/lib/webllm/eval/fixtures.test.ts b/src/lib/webllm/eval/fixtures.test.ts
new file mode 100644
index 00000000..5f8154eb
--- /dev/null
+++ b/src/lib/webllm/eval/fixtures.test.ts
@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 The resumelint Authors
+
+import { describe, expect, it } from "vitest";
+
+import { parseFixture, REWRITE_FIXTURES, getFixtureById } from "./fixtures.ts";
+
+describe("REWRITE_FIXTURES", () => {
+ it("loads exactly the four canonical fixture kinds", () => {
+ const kinds = REWRITE_FIXTURES.map((f) => f.kind).sort();
+ expect(kinds).toEqual(["numeric", "redundant", "strong", "weak"]);
+ });
+
+ it("every fixture has a non-empty id, description, and bullets", () => {
+ for (const f of REWRITE_FIXTURES) {
+ expect(f.id.length).toBeGreaterThan(0);
+ expect(f.description.length).toBeGreaterThan(0);
+ expect(f.bullets.length).toBeGreaterThan(0);
+ for (const b of f.bullets) expect(b.length).toBeGreaterThan(0);
+ }
+ });
+
+ it("fixture ids are unique", () => {
+ const ids = REWRITE_FIXTURES.map((f) => f.id);
+ expect(new Set(ids).size).toBe(ids.length);
+ });
+
+ it("getFixtureById finds known ids and misses unknown ones", () => {
+ expect(getFixtureById(REWRITE_FIXTURES[0].id)).toBeDefined();
+ expect(getFixtureById("nope")).toBeUndefined();
+ });
+});
+
+describe("parseFixture", () => {
+ it("rejects a fixture missing an id", () => {
+ expect(() =>
+ parseFixture(
+ { kind: "weak", description: "x", bullets: ["a"] },
+ "test.json",
+ ),
+ ).toThrow(/missing\/empty 'id'/);
+ });
+
+ it("rejects an unknown kind", () => {
+ expect(() =>
+ parseFixture(
+ { id: "x", kind: "bogus", description: "x", bullets: ["a"] },
+ "test.json",
+ ),
+ ).toThrow(/'kind' must be one of/);
+ });
+
+ it("rejects an empty bullets array", () => {
+ expect(() =>
+ parseFixture(
+ { id: "x", kind: "weak", description: "x", bullets: [] },
+ "test.json",
+ ),
+ ).toThrow(/non-empty string/);
+ });
+});
diff --git a/src/lib/webllm/eval/fixtures.ts b/src/lib/webllm/eval/fixtures.ts
new file mode 100644
index 00000000..0f600655
--- /dev/null
+++ b/src/lib/webllm/eval/fixtures.ts
@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 The resumelint Authors
+
+import weak from "../../../../tests/fixtures/rewrite/weak.json" with { type: "json" };
+import strong from "../../../../tests/fixtures/rewrite/strong.json" with { type: "json" };
+import numeric from "../../../../tests/fixtures/rewrite/numeric.json" with { type: "json" };
+import redundant from "../../../../tests/fixtures/rewrite/redundant.json" with { type: "json" };
+
+import type { FixtureKind, RewriteFixture } from "./types.ts";
+
+/**
+ * The fixture set the eval iterates. JSON files are imported directly
+ * (Vite handles bundling for the browser entry; Node 20+ + Vitest both
+ * resolve `with { type: "json" }` natively), so adding a fixture means
+ * dropping a JSON file under `tests/fixtures/rewrite/` AND appending an
+ * import here. The explicit list is deliberate — autoloading via dynamic
+ * `import.meta.glob` would invert the test/code dependency and surprise
+ * a reader of the file.
+ *
+ * Every fixture is validated by `parseFixture` at module load time so a
+ * broken fixture file throws before any eval run, with a precise pointer
+ * to which file is malformed.
+ */
+
+const FIXTURE_KINDS: readonly FixtureKind[] = [
+ "weak",
+ "strong",
+ "numeric",
+ "redundant",
+];
+
+function isFixtureKind(value: unknown): value is FixtureKind {
+ return (
+ typeof value === "string" &&
+ (FIXTURE_KINDS as readonly string[]).includes(value)
+ );
+}
+
+/**
+ * Validate one fixture's JSON shape. Throws with the source path so a
+ * malformed fixture is easy to find.
+ */
+export function parseFixture(raw: unknown, source: string): RewriteFixture {
+ if (typeof raw !== "object" || raw === null) {
+ throw new Error(`[rewrite-fixture] ${source}: not an object`);
+ }
+ const obj = raw as Record;
+ if (typeof obj.id !== "string" || obj.id.length === 0) {
+ throw new Error(`[rewrite-fixture] ${source}: missing/empty 'id'`);
+ }
+ if (!isFixtureKind(obj.kind)) {
+ throw new Error(
+ `[rewrite-fixture] ${source}: 'kind' must be one of ${FIXTURE_KINDS.join(", ")}`,
+ );
+ }
+ if (typeof obj.description !== "string") {
+ throw new Error(`[rewrite-fixture] ${source}: missing 'description'`);
+ }
+ if (
+ !Array.isArray(obj.bullets) ||
+ obj.bullets.length === 0 ||
+ !obj.bullets.every((b): b is string => typeof b === "string")
+ ) {
+ throw new Error(
+ `[rewrite-fixture] ${source}: 'bullets' must be a non-empty string[]`,
+ );
+ }
+ return {
+ id: obj.id,
+ kind: obj.kind,
+ description: obj.description,
+ bullets: obj.bullets,
+ };
+}
+
+/**
+ * All fixtures, parsed at module load. Order is stable and used as the
+ * report's row order.
+ */
+export const REWRITE_FIXTURES: readonly RewriteFixture[] = [
+ parseFixture(weak, "tests/fixtures/rewrite/weak.json"),
+ parseFixture(strong, "tests/fixtures/rewrite/strong.json"),
+ parseFixture(numeric, "tests/fixtures/rewrite/numeric.json"),
+ parseFixture(redundant, "tests/fixtures/rewrite/redundant.json"),
+];
+
+/** Look up a fixture by id. */
+export function getFixtureById(id: string): RewriteFixture | undefined {
+ return REWRITE_FIXTURES.find((f) => f.id === id);
+}
diff --git a/src/lib/webllm/eval/prompt-variants.ts b/src/lib/webllm/eval/prompt-variants.ts
new file mode 100644
index 00000000..4ae3586e
--- /dev/null
+++ b/src/lib/webllm/eval/prompt-variants.ts
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 The resumelint Authors
+
+import { SECTION_REWRITE_SYSTEM_PROMPT } from "../rewrite-section.ts";
+import type { PromptVariant } from "./types.ts";
+
+/**
+ * Prompt variants the eval compares.
+ *
+ * `baseline` is the SHIPPED system prompt in `rewrite-section.ts` —
+ * imported directly so a tweak there is automatically reflected in the
+ * baseline column of the eval. The other variants are deliberately small
+ * deltas (one or two rule changes) so a regression in any one criterion
+ * traces cleanly to a single prompt change rather than a wholesale
+ * rewrite.
+ *
+ * Add a variant by appending here; the runner enumerates this array.
+ * Variant ids must be stable kebab-case so committed reports remain
+ * diffable across runs.
+ */
+export const PROMPT_VARIANTS: readonly PromptVariant[] = [
+ {
+ id: "baseline",
+ label: "Baseline (shipped)",
+ systemPrompt: SECTION_REWRITE_SYSTEM_PROMPT,
+ },
+ {
+ id: "terse",
+ label: "Terse (rules-only)",
+ systemPrompt: `Rewrite each resume bullet to be more specific and outcome-oriented.
+- One bullet per line. No numbering, markers, quotes, or preamble.
+- Start each bullet with a strong action verb.
+- Preserve every number from the input EXACTLY.
+- Merge weak duplicates. Drop pure filler. Vary the verbs.`,
+ },
+ {
+ id: "examples-led",
+ label: "Examples-led (few-shot)",
+ systemPrompt: `You are rewriting resume bullets to be more specific and outcome-oriented.
+
+Rules:
+- One bullet per line. No numbering. No bullet markers. No quotes. No preamble.
+- Lead every bullet with a strong action verb.
+- Preserve every concrete number EXACTLY. Do not invent numbers.
+- Merge weak duplicates. Drop pure filler. Vary verbs across bullets.
+
+Example weak → strong:
+- "Helped with marketing things" → "Drove a 4-touchpoint nurture sequence that lifted lead-to-MQL conversion 12%."
+- "Worked on backend stuff" → "Migrated the order-processing pipeline to Kafka, cutting median latency 38%."`,
+ },
+];
+
+/** Look up a variant by id. */
+export function getVariantById(id: string): PromptVariant | undefined {
+ return PROMPT_VARIANTS.find((v) => v.id === id);
+}
diff --git a/src/lib/webllm/eval/report.test.ts b/src/lib/webllm/eval/report.test.ts
new file mode 100644
index 00000000..32c29b69
--- /dev/null
+++ b/src/lib/webllm/eval/report.test.ts
@@ -0,0 +1,114 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 The resumelint Authors
+
+import { describe, expect, it } from "vitest";
+
+import { renderJsonReport, renderMarkdownReport } from "./report.ts";
+import type { EvalReport, RunRecord } from "./types.ts";
+
+function passingRecord(modelId: string, variantId: string, fixtureId: string): RunRecord {
+ return {
+ modelId,
+ variantId,
+ fixtureId,
+ fixtureKind: "weak",
+ inputBulletCount: 5,
+ outputBulletCount: 5,
+ rubric: {
+ numbersPreserved: true,
+ oneLinePerBullet: true,
+ actionVerbLead: true,
+ lengthSanity: true,
+ noPreambleLeak: true,
+ dedupEffective: null,
+ judgeCoherence: null,
+ perBullet: [],
+ droppedNumbers: [],
+ addedNumbers: [],
+ },
+ rewriteDurationMs: 1200,
+ error: null,
+ };
+}
+
+const sampleReport: EvalReport = {
+ startedAt: "2026-06-23T00:00:00.000Z",
+ appVersion: "abc1234",
+ modelIds: ["Qwen2.5-1.5B-Instruct-q4f16_1-MLC"],
+ variantIds: ["baseline"],
+ fixtureIds: ["fx-weak", "fx-strong"],
+ judgeEnabled: false,
+ records: [
+ passingRecord("Qwen2.5-1.5B-Instruct-q4f16_1-MLC", "baseline", "fx-weak"),
+ passingRecord("Qwen2.5-1.5B-Instruct-q4f16_1-MLC", "baseline", "fx-strong"),
+ ],
+ aggregates: [
+ {
+ modelId: "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
+ variantId: "baseline",
+ scoredFixtures: 2,
+ numbersPreservedRate: 1,
+ oneLineRate: 1,
+ actionVerbRate: 1,
+ lengthSanityRate: 1,
+ noPreambleLeakRate: 1,
+ dedupEffectiveRate: null,
+ judgeMean: null,
+ aggregateScore: 1,
+ },
+ ],
+};
+
+describe("renderJsonReport", () => {
+ it("renders pretty-printed JSON with a trailing newline", () => {
+ const out = renderJsonReport(sampleReport);
+ expect(out.endsWith("\n")).toBe(true);
+ const parsed = JSON.parse(out);
+ expect(parsed.startedAt).toBe("2026-06-23T00:00:00.000Z");
+ expect(parsed.aggregates).toHaveLength(1);
+ });
+});
+
+describe("renderMarkdownReport", () => {
+ it("includes the header metadata", () => {
+ const md = renderMarkdownReport(sampleReport);
+ expect(md).toContain("# Rewrite eval report");
+ expect(md).toContain("**Started:** 2026-06-23T00:00:00.000Z");
+ expect(md).toContain("**App version:** `abc1234`");
+ expect(md).toContain("**LLM judge:** disabled (default)");
+ });
+
+ it("renders the aggregate table with model name resolved from the registry", () => {
+ const md = renderMarkdownReport(sampleReport);
+ // The model id resolves to its registry name via getModelById.
+ expect(md).toContain("| Qwen 2.5 (1.5B) | Baseline (shipped) |");
+ expect(md).toContain("**100%**");
+ });
+
+ it("renders `—` for dedup and judge when they don't apply", () => {
+ const md = renderMarkdownReport(sampleReport);
+ // The aggregate row's dedup + judge columns should render `—`.
+ expect(md).toMatch(/\| — \| — \| \*\*100%\*\* \|/);
+ });
+
+ it("renders an error column for errored cells", () => {
+ const errReport: EvalReport = {
+ ...sampleReport,
+ records: [
+ {
+ ...passingRecord("Qwen2.5-1.5B-Instruct-q4f16_1-MLC", "baseline", "fx-weak"),
+ error: "model OOM",
+ outputBulletCount: 0,
+ rubric: {
+ ...passingRecord("M", "V", "F").rubric,
+ numbersPreserved: false,
+ actionVerbLead: false,
+ },
+ },
+ ],
+ };
+ const md = renderMarkdownReport(errReport);
+ expect(md).toContain("`model OOM`");
+ expect(md).toContain("fail");
+ });
+});
diff --git a/src/lib/webllm/eval/report.ts b/src/lib/webllm/eval/report.ts
new file mode 100644
index 00000000..c87c411d
--- /dev/null
+++ b/src/lib/webllm/eval/report.ts
@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 The resumelint Authors
+
+import { getModelById } from "../models.ts";
+import { getVariantById } from "./prompt-variants.ts";
+import type { EvalReport } from "./types.ts";
+
+/**
+ * Render the eval report in two flavors:
+ *
+ * - `renderJsonReport` — stable structured artifact (committed under
+ * `tests/fixtures/rewrite/reports/`). Machine-diffable across runs.
+ * - `renderMarkdownReport` — human-readable table for PR / issue
+ * comments. Same shape as the JSON; just the prose layer.
+ *
+ * The renderer is pure over `EvalReport` so a snapshot test fixes the
+ * formatting against drift. Reports are intentionally lossy on
+ * model-output text (per-bullet text is included but per-cell raw
+ * responses are not) so the artifact stays under a kilobyte per cell.
+ */
+
+export function renderJsonReport(report: EvalReport): string {
+ return `${JSON.stringify(report, null, 2)}\n`;
+}
+
+export function renderMarkdownReport(report: EvalReport): string {
+ const lines: string[] = [];
+ lines.push("# Rewrite eval report");
+ lines.push("");
+ lines.push(`- **Started:** ${report.startedAt}`);
+ if (report.appVersion) lines.push(`- **App version:** \`${report.appVersion}\``);
+ lines.push(`- **Models:** ${report.modelIds.length}`);
+ lines.push(`- **Prompt variants:** ${report.variantIds.length}`);
+ lines.push(`- **Fixtures:** ${report.fixtureIds.length}`);
+ lines.push(
+ `- **LLM judge:** ${report.judgeEnabled ? "enabled" : "disabled (default)"}`,
+ );
+ lines.push("");
+
+ lines.push("## Aggregate (per model × variant)");
+ lines.push("");
+ lines.push(
+ "| Model | Variant | Numbers | One-line | Verb | Length | No-preamble | Dedup | Judge | **Aggregate** |",
+ );
+ lines.push(
+ "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |",
+ );
+ for (const row of report.aggregates) {
+ const modelLabel = getModelById(row.modelId)?.name ?? row.modelId;
+ const variantLabel = getVariantById(row.variantId)?.label ?? row.variantId;
+ lines.push(
+ `| ${modelLabel} | ${variantLabel} | ${pct(row.numbersPreservedRate)} | ${pct(row.oneLineRate)} | ${pct(row.actionVerbRate)} | ${pct(row.lengthSanityRate)} | ${pct(row.noPreambleLeakRate)} | ${pctOrDash(row.dedupEffectiveRate)} | ${numOrDash(row.judgeMean)} | **${pct(row.aggregateScore)}** |`,
+ );
+ }
+ lines.push("");
+
+ lines.push("## Per-cell records");
+ lines.push("");
+ for (const modelId of report.modelIds) {
+ const modelLabel = getModelById(modelId)?.name ?? modelId;
+ lines.push(`### ${modelLabel}`);
+ lines.push("");
+ for (const variantId of report.variantIds) {
+ const variantLabel = getVariantById(variantId)?.label ?? variantId;
+ lines.push(`#### ${variantLabel}`);
+ lines.push("");
+ lines.push(
+ "| Fixture | Kind | In → Out | Numbers | Verb | Length | Preamble | Dedup | Error |",
+ );
+ lines.push(
+ "| --- | --- | --- | --- | --- | --- | --- | --- | --- |",
+ );
+ for (const r of report.records) {
+ if (r.modelId !== modelId || r.variantId !== variantId) continue;
+ lines.push(
+ `| ${r.fixtureId} | ${r.fixtureKind} | ${r.inputBulletCount} → ${r.outputBulletCount} | ${tick(r.rubric.numbersPreserved)} | ${tick(r.rubric.actionVerbLead)} | ${tick(r.rubric.lengthSanity)} | ${tick(r.rubric.noPreambleLeak)} | ${tickOrDash(r.rubric.dedupEffective)} | ${r.error ? `\`${r.error}\`` : ""} |`,
+ );
+ }
+ lines.push("");
+ }
+ }
+
+ return `${lines.join("\n")}\n`;
+}
+
+function pct(v: number): string {
+ return `${Math.round(v * 100)}%`;
+}
+
+function pctOrDash(v: number | null): string {
+ return v === null ? "—" : pct(v);
+}
+
+function numOrDash(v: number | null): string {
+ return v === null ? "—" : v.toFixed(2);
+}
+
+function tick(v: boolean): string {
+ return v ? "PASS" : "fail";
+}
+
+function tickOrDash(v: boolean | null): string {
+ return v === null ? "—" : tick(v);
+}
diff --git a/src/lib/webllm/eval/rubric.test.ts b/src/lib/webllm/eval/rubric.test.ts
new file mode 100644
index 00000000..ed0e0672
--- /dev/null
+++ b/src/lib/webllm/eval/rubric.test.ts
@@ -0,0 +1,181 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 The resumelint Authors
+
+import { describe, expect, it } from "vitest";
+
+import { emptyRubricForError, scoreRubric } from "./rubric.ts";
+import type { RawRewriteOutput } from "./types.ts";
+
+function out(bullets: string[], raw?: string): RawRewriteOutput {
+ return { bullets, raw: raw ?? bullets.join("\n") };
+}
+
+describe("scoreRubric — canned good outputs", () => {
+ it("passes a strong rewrite of a weak input without inventing numbers", () => {
+ // Input has zero numeric tokens, so the output must also have zero —
+ // inventing a metric the input didn't contain is a number-preservation
+ // failure (the guardrail's "none invented" half).
+ const input = [
+ "Responsible for handling marketing tasks and supporting the team as needed.",
+ "Worked on campaigns to drive engagement.",
+ ];
+ const output = out([
+ "Drove the email nurture sequence rollout, lifting lead-to-MQL conversion across the brand portfolio.",
+ "Launched paid-social experiments and ran weekly performance reviews with the brand team.",
+ ]);
+ const r = scoreRubric({ input, output, fixtureKind: "weak" });
+ expect(r.actionVerbLead).toBe(true);
+ expect(r.oneLinePerBullet).toBe(true);
+ expect(r.lengthSanity).toBe(true);
+ expect(r.noPreambleLeak).toBe(true);
+ expect(r.numbersPreserved).toBe(true);
+ expect(r.dedupEffective).toBeNull(); // n/a for weak fixtures
+ });
+
+ it("passes a numeric input whose output preserves every metric", () => {
+ const input = [
+ "Grew users from 120K to 1.8M between 2022 and 2024, lifting retention 14%.",
+ "Drove $4.2M ARR through a 2-tier paywall redesign.",
+ ];
+ const output = out([
+ "Grew weekly actives from 120K to 1.8M (2022-2024), lifting day-7 retention 14% via 7 onboarding tests.",
+ "Drove $4.2M incremental ARR with a 2-tier paywall redesign across 3 surfaces.",
+ ]);
+ const r = scoreRubric({ input, output, fixtureKind: "numeric" });
+ expect(r.numbersPreserved).toBe(true);
+ expect(r.droppedNumbers).toEqual([]);
+ expect(r.addedNumbers).toEqual([]);
+ });
+
+ it("passes a redundant input whose output collapses duplicates", () => {
+ const input = [
+ "Triaged 200+ support tickets per week.",
+ "Managed a 200/week ticket queue.",
+ "Handled support tickets at 200 per week.",
+ "Resolved $85K in disputed enterprise charges.",
+ ];
+ const output = out([
+ "Triaged 200+ inbound support tickets per week across email and chat.",
+ "Resolved escalated billing disputes for 40 enterprise accounts, recovering $85K.",
+ ]);
+ const r = scoreRubric({ input, output, fixtureKind: "redundant" });
+ expect(r.dedupEffective).toBe(true);
+ });
+});
+
+describe("scoreRubric — canned bad outputs", () => {
+ it("flags a dropped number", () => {
+ const input = ["Drove $4.2M ARR via a 2-tier paywall lift of 23%."];
+ const output = out(["Drove ARR via a paywall redesign lifting conversion 23%."]);
+ const r = scoreRubric({ input, output, fixtureKind: "numeric" });
+ expect(r.numbersPreserved).toBe(false);
+ expect(r.droppedNumbers).toContain("$4.2M");
+ });
+
+ it("flags an invented number", () => {
+ const input = ["Drove ARR via a paywall redesign."];
+ const output = out(["Drove $1.2M ARR via a paywall redesign."]);
+ const r = scoreRubric({ input, output, fixtureKind: "numeric" });
+ expect(r.numbersPreserved).toBe(false);
+ expect(r.addedNumbers).toContain("$1.2M");
+ });
+
+ it("flags a weak verb lead", () => {
+ const input = ["Responsible for marketing."];
+ const output = out([
+ "Worked on marketing campaigns across 3 channels with the brand team.",
+ ]);
+ const r = scoreRubric({ input, output, fixtureKind: "weak" });
+ expect(r.actionVerbLead).toBe(false);
+ });
+
+ it("flags a too-short bullet (length sanity floor)", () => {
+ const input = ["Worked on stuff."];
+ const output = out(["Did things."]);
+ const r = scoreRubric({ input, output, fixtureKind: "weak" });
+ expect(r.lengthSanity).toBe(false);
+ });
+
+ it("flags a too-long bullet (length sanity ceiling)", () => {
+ const input = ["Worked on stuff."];
+ const long = `Drove ${"a ".repeat(150)}thing across multiple teams.`;
+ const r = scoreRubric({
+ input,
+ output: out([long]),
+ fixtureKind: "weak",
+ });
+ expect(r.lengthSanity).toBe(false);
+ });
+
+ it("flags preamble leakage in the raw response", () => {
+ const input = ["Worked on marketing."];
+ const output = out(
+ ["Drove a 4-touchpoint nurture sequence that lifted MQL conversion 12%."],
+ "Here is the rewritten bullets:\nDrove a 4-touchpoint nurture sequence that lifted MQL conversion 12%.",
+ );
+ const r = scoreRubric({ input, output, fixtureKind: "weak" });
+ expect(r.noPreambleLeak).toBe(false);
+ });
+
+ it("does NOT flag preamble when the phrase appears only inside a bullet", () => {
+ const input = ["Drafted the rules of engagement for the eng-marketing handoff."];
+ const output = out([
+ "Drafted the rules of engagement document for cross-team handoffs across 4 teams.",
+ ]);
+ const r = scoreRubric({ input, output, fixtureKind: "weak" });
+ expect(r.noPreambleLeak).toBe(true);
+ });
+
+ it("flags a redundant fixture whose output did NOT collapse", () => {
+ const input = ["A", "B", "C"];
+ const output = out([
+ "Triaged 200+ support tickets per week across email channels.",
+ "Triaged 200+ inbound queue items weekly via the email pipeline.",
+ "Handled 200 tickets per week through the support inbox.",
+ ]);
+ const r = scoreRubric({ input, output, fixtureKind: "redundant" });
+ expect(r.dedupEffective).toBe(false);
+ });
+
+ it("flags an embedded newline as a one-line violation", () => {
+ const input = ["Worked on X."];
+ const output = out(["Drove a multi-team launch\nacross the org with strong outcomes."]);
+ const r = scoreRubric({ input, output, fixtureKind: "weak" });
+ expect(r.oneLinePerBullet).toBe(false);
+ });
+});
+
+describe("scoreRubric — empty output (model returned nothing parseable)", () => {
+ it("fails one-line, verb, length, and dedup criteria when bullets is empty", () => {
+ // A model returning zero bullets is a failure, not a vacuous pass.
+ // The criteria that quantify per-bullet quality must reflect "no
+ // bullets to score" as a fail, including dedup (which would otherwise
+ // trivially satisfy `output < input`).
+ const r = scoreRubric({
+ input: ["A", "B", "C"],
+ output: { bullets: [], raw: "" },
+ fixtureKind: "redundant",
+ });
+ expect(r.oneLinePerBullet).toBe(false);
+ expect(r.actionVerbLead).toBe(false);
+ expect(r.lengthSanity).toBe(false);
+ expect(r.dedupEffective).toBe(false);
+ // The non-bullet-dependent criteria still report honestly.
+ expect(r.numbersPreserved).toBe(true); // input had no numeric tokens
+ expect(r.noPreambleLeak).toBe(true); // raw was empty
+ });
+});
+
+describe("emptyRubricForError", () => {
+ it("returns all-fail with no per-bullet rows", () => {
+ const r = emptyRubricForError();
+ expect(r.numbersPreserved).toBe(false);
+ expect(r.actionVerbLead).toBe(false);
+ expect(r.lengthSanity).toBe(false);
+ expect(r.noPreambleLeak).toBe(false);
+ expect(r.oneLinePerBullet).toBe(false);
+ expect(r.dedupEffective).toBeNull();
+ expect(r.judgeCoherence).toBeNull();
+ expect(r.perBullet).toEqual([]);
+ });
+});
diff --git a/src/lib/webllm/eval/rubric.ts b/src/lib/webllm/eval/rubric.ts
new file mode 100644
index 00000000..68e02f03
--- /dev/null
+++ b/src/lib/webllm/eval/rubric.ts
@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 The resumelint Authors
+
+import { checkNumbersPreserved } from "../preserve-numbers.ts";
+import type {
+ FixtureKind,
+ PerBulletDiagnostic,
+ RawRewriteOutput,
+ RubricResult,
+} from "./types.ts";
+import { startsWithActionVerb } from "./verbs.ts";
+
+/**
+ * Deterministic rubric: takes the input bullets + a model's raw rewrite
+ * output and emits a per-criterion pass/fail record. No judge model.
+ *
+ * The six criteria are the issue #65 AC list:
+ *
+ * 1. numbersPreserved — multiset of numeric tokens unchanged
+ * 2. oneLinePerBullet — no embedded `\n` after the runner's split
+ * 3. actionVerbLead — first token of each bullet in the curated set
+ * 4. lengthSanity — each bullet in a sane char band
+ * 5. noPreambleLeak — output doesn't echo prompt scaffolding
+ * 6. dedupEffective — for `redundant` fixtures only: output < input
+ *
+ * Each criterion is computed independently — one failing does NOT
+ * short-circuit the others, because the report's per-criterion pass rate
+ * is more useful than a single composite verdict.
+ */
+
+/** Sanity band for a single bullet. Below MIN reads as a truncated
+ * fragment; above MAX reads as a run-on or multi-bullet collapse. The
+ * band intentionally covers strong real-world bullets (most of which
+ * land in the 60–180 range). */
+const BULLET_MIN_CHARS = 25;
+const BULLET_MAX_CHARS = 260;
+
+/**
+ * Phrases that indicate the model echoed prompt scaffolding into the
+ * output. Cleaned by `cleanRewriteLine` already, but the rubric also
+ * scans the RAW pre-split output to catch leakage that survived (e.g.
+ * spread across multiple lines, or with non-standard capitalization).
+ *
+ * The check is substring (case-insensitive) against the raw response
+ * with the bullet lines stripped, so a legitimate bullet that contains
+ * "the rules of engagement" doesn't trip the criterion.
+ */
+const PREAMBLE_LEAK_PHRASES = [
+ "rewritten bullets:",
+ "original bullets:",
+ "here are the rewritten",
+ "here is the rewritten",
+ "rules:",
+ "system:",
+ "as an ai",
+ "as a language model",
+];
+
+/**
+ * Returns the empty rubric used for an error row (RewriteFn threw, or
+ * returned an unparseable response). All criteria fail so the row
+ * surfaces in the report instead of being silently scored as a pass.
+ */
+export function emptyRubricForError(): RubricResult {
+ return {
+ numbersPreserved: false,
+ oneLinePerBullet: false,
+ actionVerbLead: false,
+ lengthSanity: false,
+ noPreambleLeak: false,
+ dedupEffective: null,
+ judgeCoherence: null,
+ perBullet: [],
+ droppedNumbers: [],
+ addedNumbers: [],
+ };
+}
+
+export interface ScoreRubricInput {
+ input: readonly string[];
+ output: RawRewriteOutput;
+ fixtureKind: FixtureKind;
+}
+
+export function scoreRubric({
+ input,
+ output,
+ fixtureKind,
+}: ScoreRubricInput): RubricResult {
+ const outputBullets = output.bullets;
+
+ // ── (1) Numbers preserved ─────────────────────────────────────────────
+ const preservation = checkNumbersPreserved(input, outputBullets);
+
+ // ── (2) One line per bullet ───────────────────────────────────────────
+ // The runner already split on `\n`, so an embedded `\n` here would
+ // only appear if the post-process kept a literal `\n` token (e.g. a
+ // Windows `\r` survived). Explicit check on each bullet keeps the
+ // criterion honest if the splitting strategy changes.
+ //
+ // Empty output is NOT vacuously a pass: the model produced no bullets
+ // at all, so the "every bullet is one line" claim has nothing to back
+ // it. Require at least one bullet for the criterion to be true.
+ const oneLinePerBullet =
+ outputBullets.length > 0 && outputBullets.every((b) => !/[\r\n]/.test(b));
+
+ // ── (3) Action-verb lead ──────────────────────────────────────────────
+ const verbResults = outputBullets.map((b) => startsWithActionVerb(b));
+ const actionVerbLead =
+ outputBullets.length > 0 && verbResults.every((v) => v);
+
+ // ── (4) Length sanity ─────────────────────────────────────────────────
+ const lengthResults = outputBullets.map(
+ (b) => b.length >= BULLET_MIN_CHARS && b.length <= BULLET_MAX_CHARS,
+ );
+ const lengthSanity =
+ outputBullets.length > 0 && lengthResults.every((v) => v);
+
+ // ── (5) No preamble leakage ───────────────────────────────────────────
+ // Scan the RAW response (pre-split) with the bullet text stripped out
+ // so a phrase like "rules:" inside a legitimate bullet doesn't trip.
+ // Lowercased substring match; the phrase list is conservative.
+ let rawMinusBullets = output.raw.toLowerCase();
+ for (const b of outputBullets) {
+ rawMinusBullets = rawMinusBullets.replace(b.toLowerCase(), "");
+ }
+ const noPreambleLeak = !PREAMBLE_LEAK_PHRASES.some((p) =>
+ rawMinusBullets.includes(p),
+ );
+
+ // ── (6) Dedup effectiveness ───────────────────────────────────────────
+ // Only meaningful for fixtures that explicitly stage redundancy. For
+ // other kinds, the criterion is `null` (not applicable) — the report
+ // displays `—` and the aggregate ignores them.
+ //
+ // The non-empty guard matters: a model returning zero bullets would
+ // trivially satisfy `output < input`, but that's a model failure, not
+ // a dedup win. Require at least one bullet.
+ const dedupEffective: boolean | null =
+ fixtureKind === "redundant"
+ ? outputBullets.length > 0 && outputBullets.length < input.length
+ : null;
+
+ const perBullet: PerBulletDiagnostic[] = outputBullets.map((b, i) => ({
+ index: i,
+ text: b,
+ startsWithActionVerb: verbResults[i] ?? false,
+ lengthOk: lengthResults[i] ?? false,
+ oneLine: !/[\r\n]/.test(b),
+ }));
+
+ return {
+ numbersPreserved: preservation.ok,
+ oneLinePerBullet,
+ actionVerbLead,
+ lengthSanity,
+ noPreambleLeak,
+ dedupEffective,
+ judgeCoherence: null,
+ perBullet,
+ droppedNumbers: preservation.dropped,
+ addedNumbers: preservation.added,
+ };
+}
diff --git a/src/lib/webllm/eval/run-eval-browser.ts b/src/lib/webllm/eval/run-eval-browser.ts
new file mode 100644
index 00000000..0574b492
--- /dev/null
+++ b/src/lib/webllm/eval/run-eval-browser.ts
@@ -0,0 +1,231 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 The resumelint Authors
+
+/**
+ * Browser entry for the rewrite-quality eval harness.
+ *
+ * Reached via `npm run eval:rewrite` → opens
+ * `/resumelint/eval-rewrite.html` in the dev server. Loads the model
+ * the user picked, runs every prompt variant against every fixture,
+ * scores with the deterministic rubric, and renders a downloadable
+ * JSON + Markdown report.
+ *
+ * One model per tab on purpose. Cycling all three registry models in a
+ * single tab kept crashing Chrome on consumer GPUs during the
+ * eviction-then-reload path — eviction calls `.unload()` but VRAM
+ * isn't guaranteed to be free by the time the next model starts
+ * downloading. A fresh tab per model sidesteps that entirely: the
+ * prior tab's GPU resources are reclaimed on close. The downside is
+ * the maintainer commits one report file per model and a reviewer
+ * compares them side-by-side — still cheap.
+ *
+ * This file is deliberately NOT imported by `src/main.tsx`, so it does
+ * not contribute to the production bundle. The Vite root has
+ * `index.html` as the sole prod input — `eval-rewrite.html` is a
+ * dev-only sibling page that Vite serves but does not build.
+ *
+ * Telemetry: the eval explicitly skips analytics. The shipped rewrite
+ * APIs fire telemetry, so the eval reaches into the engine directly
+ * instead of calling `rewriteSectionWithLlm`. That keeps an eval run
+ * from polluting `webllm_section_rewrite_*` counters during local
+ * benchmarking.
+ */
+
+import { cleanRewriteLine } from "../post-process.ts";
+import {
+ buildSectionUserPrompt,
+ sectionMaxTokens,
+} from "../rewrite-section.ts";
+import { MODEL_REGISTRY, getModelById } from "../models.ts";
+import { loadEngine } from "../web-llm.ts";
+import { detectWebGpu } from "../capability.ts";
+import type { WebLlmEngine } from "../types.ts";
+
+import { REWRITE_FIXTURES } from "./fixtures.ts";
+import { PROMPT_VARIANTS } from "./prompt-variants.ts";
+import { renderJsonReport, renderMarkdownReport } from "./report.ts";
+import { runEval } from "./runner.ts";
+import type { RawRewriteOutput, RewriteFn } from "./types.ts";
+
+declare const __APP_VERSION__: string;
+
+const SECTION_TEMPERATURE = 0.3;
+
+/**
+ * Build a `RewriteFn` backed by a real WebLLM engine + custom prompt.
+ * The system prompt is the variant's; the user prompt is the shared
+ * `buildSectionUserPrompt` shape from production so output framing
+ * stays comparable to the shipped path.
+ */
+function makeRealRewriteFn(engine: WebLlmEngine): RewriteFn {
+ return async ({ variantId, fixture }) => {
+ const variant = PROMPT_VARIANTS.find((v) => v.id === variantId);
+ if (!variant) throw new Error(`unknown variant: ${variantId}`);
+
+ const response = await engine.chat.completions.create({
+ messages: [
+ { role: "system", content: variant.systemPrompt },
+ { role: "user", content: buildSectionUserPrompt(fixture.bullets) },
+ ],
+ temperature: SECTION_TEMPERATURE,
+ max_tokens: sectionMaxTokens(fixture.bullets.length),
+ });
+
+ const raw = response.choices[0]?.message?.content ?? "";
+ const bullets = raw
+ .split("\n")
+ .map((line) => cleanRewriteLine(line))
+ .filter((line) => line.length > 0);
+ return { bullets, raw } satisfies RawRewriteOutput;
+ };
+}
+
+interface DomRefs {
+ status: HTMLElement;
+ progress: HTMLElement;
+ log: HTMLElement;
+ downloadJson: HTMLAnchorElement;
+ downloadMd: HTMLAnchorElement;
+ runBtn: HTMLButtonElement;
+ modelSelect: HTMLSelectElement;
+}
+
+function getDomRefs(): DomRefs {
+ return {
+ status: document.getElementById("status")!,
+ progress: document.getElementById("progress")!,
+ log: document.getElementById("log")!,
+ downloadJson: document.getElementById("download-json") as HTMLAnchorElement,
+ downloadMd: document.getElementById("download-md") as HTMLAnchorElement,
+ runBtn: document.getElementById("run") as HTMLButtonElement,
+ modelSelect: document.getElementById("model") as HTMLSelectElement,
+ };
+}
+
+function setStatus(refs: DomRefs, text: string): void {
+ refs.status.textContent = text;
+}
+
+function appendLog(refs: DomRefs, line: string): void {
+ const time = new Date().toISOString().slice(11, 19);
+ refs.log.textContent = `${refs.log.textContent ?? ""}[${time}] ${line}\n`;
+ refs.log.scrollTop = refs.log.scrollHeight;
+}
+
+function wireDownload(
+ anchor: HTMLAnchorElement,
+ filename: string,
+ contents: string,
+ mime: string,
+): void {
+ const blob = new Blob([contents], { type: mime });
+ const url = URL.createObjectURL(blob);
+ anchor.href = url;
+ anchor.download = filename;
+ anchor.removeAttribute("hidden");
+}
+
+function populateModelPicker(refs: DomRefs): void {
+ refs.modelSelect.innerHTML = "";
+ for (const model of MODEL_REGISTRY) {
+ const option = document.createElement("option");
+ option.value = model.id;
+ option.textContent = `${model.name} · ${model.licenseType} · ~${model.downloadSizeMb} MB`;
+ refs.modelSelect.appendChild(option);
+ }
+}
+
+async function runForModel(refs: DomRefs, modelId: string): Promise {
+ const meta = getModelById(modelId);
+ const display = meta?.name ?? modelId;
+
+ appendLog(refs, `loading model ${modelId}`);
+ setStatus(refs, `Loading ${display} …`);
+ const engine = await loadEngine(modelId, (update) => {
+ refs.progress.textContent = `${display}: ${(update.progress * 100).toFixed(0)}% — ${update.text}`;
+ });
+ appendLog(
+ refs,
+ `model loaded; running ${PROMPT_VARIANTS.length} variants × ${REWRITE_FIXTURES.length} fixtures`,
+ );
+ // Refresh the status line so it reflects "running" instead of staying
+ // on "Loading …" for the whole cell loop.
+ setStatus(refs, `Running ${display} (${PROMPT_VARIANTS.length} variants × ${REWRITE_FIXTURES.length} fixtures) …`);
+
+ const report = await runEval({
+ modelIds: [modelId],
+ variantIds: PROMPT_VARIANTS.map((v) => v.id),
+ fixtures: REWRITE_FIXTURES,
+ rewriteFn: makeRealRewriteFn(engine),
+ appVersion: typeof __APP_VERSION__ === "string" ? __APP_VERSION__ : null,
+ onProgress: (done, total, cell) => {
+ refs.progress.textContent = `${display}: ${done}/${total} — ${cell.variantId} × ${cell.fixtureId}`;
+ },
+ });
+
+ const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+ // Slugify the model id for the filename so it's filesystem-safe and
+ // easy to read in the reports/ directory.
+ const slug = modelId.toLowerCase().replace(/[^a-z0-9]+/g, "-");
+ // Explicit `;charset=utf-8` so the saved files don't get re-decoded as
+ // Latin-1 by some text viewers — without it, multi-byte chars like
+ // `×` and `—` in the markdown render as mojibake.
+ wireDownload(
+ refs.downloadJson,
+ `eval-rewrite-${slug}-${stamp}.json`,
+ renderJsonReport(report),
+ "application/json;charset=utf-8",
+ );
+ wireDownload(
+ refs.downloadMd,
+ `eval-rewrite-${slug}-${stamp}.md`,
+ renderMarkdownReport(report),
+ "text/markdown;charset=utf-8",
+ );
+ setStatus(
+ refs,
+ `Done. ${report.records.length} records scored for ${display}.`,
+ );
+ appendLog(
+ refs,
+ "report ready — download below and commit under tests/fixtures/rewrite/reports/",
+ );
+}
+
+async function main(): Promise {
+ const refs = getDomRefs();
+ populateModelPicker(refs);
+
+ refs.runBtn.addEventListener("click", async () => {
+ refs.runBtn.disabled = true;
+ refs.modelSelect.disabled = true;
+ refs.downloadJson.setAttribute("hidden", "");
+ refs.downloadMd.setAttribute("hidden", "");
+ refs.log.textContent = "";
+
+ try {
+ const capability = await detectWebGpu();
+ if (capability !== "available") {
+ setStatus(refs, `WebGPU not available: ${capability}`);
+ return;
+ }
+ const modelId = refs.modelSelect.value;
+ const meta = getModelById(modelId);
+ if (!meta) {
+ setStatus(refs, `Unknown model: ${modelId}`);
+ return;
+ }
+ appendLog(refs, `WebGPU available; running ${meta.name}`);
+ await runForModel(refs, modelId);
+ } catch (err) {
+ const message = err instanceof Error ? err.message : String(err);
+ setStatus(refs, `Failed: ${message}`);
+ appendLog(refs, `ERROR: ${message}`);
+ } finally {
+ refs.runBtn.disabled = false;
+ refs.modelSelect.disabled = false;
+ }
+ });
+}
+
+void main();
diff --git a/src/lib/webllm/eval/runner.test.ts b/src/lib/webllm/eval/runner.test.ts
new file mode 100644
index 00000000..fc81cfd0
--- /dev/null
+++ b/src/lib/webllm/eval/runner.test.ts
@@ -0,0 +1,177 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 The resumelint Authors
+
+import { describe, expect, it } from "vitest";
+
+import { runEval } from "./runner.ts";
+import type {
+ FixtureKind,
+ RawRewriteOutput,
+ RewriteFixture,
+ RewriteFn,
+} from "./types.ts";
+
+function fixture(id: string, kind: FixtureKind, bullets: string[]): RewriteFixture {
+ return { id, kind, description: `fixture ${id}`, bullets };
+}
+
+/** Stable per-(model, variant, fixture) output dispatch for runner tests. */
+function dispatchFn(
+ table: Record,
+): RewriteFn {
+ return async ({ modelId, variantId, fixture: f }) => {
+ const key = `${modelId}|${variantId}|${f.id}`;
+ const v = table[key];
+ if (!v) throw new Error(`no canned output for ${key}`);
+ if (v instanceof Error) throw v;
+ return v;
+ };
+}
+
+describe("runEval", () => {
+ it("iterates (model × variant × fixture) and emits one record per cell", async () => {
+ const fx1 = fixture("f1", "weak", ["Worked on stuff."]);
+ const fx2 = fixture("f2", "strong", ["Built X."]);
+ const goodOut: RawRewriteOutput = {
+ bullets: ["Drove a 4-touchpoint nurture sequence lifting MQL 12% across 3 channels."],
+ raw: "Drove a 4-touchpoint nurture sequence lifting MQL 12% across 3 channels.",
+ };
+
+ const table: Record = {};
+ for (const m of ["M-A", "M-B"]) {
+ for (const v of ["V-baseline", "V-terse"]) {
+ for (const f of [fx1, fx2]) {
+ table[`${m}|${v}|${f.id}`] = goodOut;
+ }
+ }
+ }
+
+ const report = await runEval({
+ modelIds: ["M-A", "M-B"],
+ variantIds: ["V-baseline", "V-terse"],
+ fixtures: [fx1, fx2],
+ rewriteFn: dispatchFn(table),
+ });
+
+ expect(report.records).toHaveLength(2 * 2 * 2);
+ expect(report.aggregates).toHaveLength(2 * 2);
+ // Order: M-A × V-baseline × (fx1, fx2), then M-A × V-terse × ...
+ expect(report.records[0]).toMatchObject({
+ modelId: "M-A",
+ variantId: "V-baseline",
+ fixtureId: "f1",
+ });
+ expect(report.records[7]).toMatchObject({
+ modelId: "M-B",
+ variantId: "V-terse",
+ fixtureId: "f2",
+ });
+ });
+
+ it("records error rows without aborting the run, scoring them 0", async () => {
+ const fx1 = fixture("f1", "weak", ["Worked on stuff."]);
+ const fx2 = fixture("f2", "strong", ["Built X."]);
+ const good: RawRewriteOutput = {
+ bullets: ["Drove a 4-touchpoint nurture sequence lifting MQL 12% across 3 channels."],
+ raw: "Drove a 4-touchpoint nurture sequence lifting MQL 12% across 3 channels.",
+ };
+
+ const table = {
+ "M-A|V-baseline|f1": good,
+ "M-A|V-baseline|f2": new Error("model OOM"),
+ };
+
+ const report = await runEval({
+ modelIds: ["M-A"],
+ variantIds: ["V-baseline"],
+ fixtures: [fx1, fx2],
+ rewriteFn: dispatchFn(table),
+ });
+
+ expect(report.records).toHaveLength(2);
+ expect(report.records[1].error).toBe("model OOM");
+ expect(report.records[1].rubric.actionVerbLead).toBe(false);
+
+ // The aggregate counts the error row OUT of `scoredFixtures` but
+ // still computes rates over scored ones only.
+ expect(report.aggregates[0].scoredFixtures).toBe(1);
+ expect(report.aggregates[0].actionVerbRate).toBe(1);
+ });
+
+ it("computes a dedup rate over redundant fixtures only", async () => {
+ const fxRed = fixture("r", "redundant", ["A", "B", "C"]);
+ const fxStr = fixture("s", "strong", ["X"]);
+ const dedupedOk: RawRewriteOutput = {
+ bullets: ["Triaged 200+ inbound support tickets weekly across email and chat."],
+ raw: "",
+ };
+ const noChange: RawRewriteOutput = {
+ bullets: ["Built X across 3 teams to scale operations across the org."],
+ raw: "",
+ };
+
+ const report = await runEval({
+ modelIds: ["M"],
+ variantIds: ["V"],
+ fixtures: [fxRed, fxStr],
+ rewriteFn: dispatchFn({
+ "M|V|r": dedupedOk,
+ "M|V|s": noChange,
+ }),
+ });
+
+ expect(report.aggregates[0].dedupEffectiveRate).toBe(1);
+
+ // A run with no redundant fixtures yields null, not 0.
+ const reportNoRed = await runEval({
+ modelIds: ["M"],
+ variantIds: ["V"],
+ fixtures: [fxStr],
+ rewriteFn: dispatchFn({ "M|V|s": noChange }),
+ });
+ expect(reportNoRed.aggregates[0].dedupEffectiveRate).toBeNull();
+ });
+
+ it("threads modelIds / variantIds / judgeEnabled into the report header", async () => {
+ const fx = fixture("f", "weak", ["W"]);
+ const r = await runEval({
+ modelIds: ["M-A"],
+ variantIds: ["V-1"],
+ fixtures: [fx],
+ rewriteFn: dispatchFn({
+ "M-A|V-1|f": { bullets: ["Drove a 4-touchpoint nurture sequence lifting MQL 12% across 3 channels."], raw: "" },
+ }),
+ judgeEnabled: true,
+ appVersion: "abc1234",
+ now: () => 1735689600000, // 2025-01-01T00:00:00Z
+ });
+ expect(r.judgeEnabled).toBe(true);
+ expect(r.appVersion).toBe("abc1234");
+ expect(r.startedAt).toBe("2025-01-01T00:00:00.000Z");
+ expect(r.modelIds).toEqual(["M-A"]);
+ expect(r.variantIds).toEqual(["V-1"]);
+ expect(r.fixtureIds).toEqual(["f"]);
+ });
+
+ it("invokes onProgress once per cell with running counts", async () => {
+ const fx = fixture("f", "weak", ["W"]);
+ const good: RawRewriteOutput = {
+ bullets: ["Drove a 4-touchpoint nurture sequence lifting MQL 12% across 3 channels."],
+ raw: "",
+ };
+ const progress: Array<[number, number, string]> = [];
+ await runEval({
+ modelIds: ["M-A", "M-B"],
+ variantIds: ["V"],
+ fixtures: [fx],
+ rewriteFn: dispatchFn({ "M-A|V|f": good, "M-B|V|f": good }),
+ onProgress: (done, total, cell) => {
+ progress.push([done, total, cell.modelId]);
+ },
+ });
+ expect(progress).toEqual([
+ [1, 2, "M-A"],
+ [2, 2, "M-B"],
+ ]);
+ });
+});
diff --git a/src/lib/webllm/eval/runner.ts b/src/lib/webllm/eval/runner.ts
new file mode 100644
index 00000000..f7bae08f
--- /dev/null
+++ b/src/lib/webllm/eval/runner.ts
@@ -0,0 +1,221 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 The resumelint Authors
+
+import { emptyRubricForError, scoreRubric } from "./rubric.ts";
+import type {
+ AggregateRow,
+ EvalReport,
+ RewriteFixture,
+ RewriteFn,
+ RunRecord,
+} from "./types.ts";
+
+/**
+ * Iterate (model × variant × fixture), invoke `rewriteFn` for each cell,
+ * score with the rubric, and emit a structured report.
+ *
+ * The runner is engine-agnostic: tests pass a stub `rewriteFn` that
+ * returns canned outputs (no model, no WebGPU), and the browser entry
+ * passes a real WebLLM-backed `rewriteFn`. The runner itself never
+ * touches `@mlc-ai/web-llm`, so this file is safe to import in Node
+ * test environments.
+ *
+ * Failure handling: a single failing cell logs an `error` on its
+ * `RunRecord` and the iteration continues. The row scores 0 across all
+ * criteria so the report shows the failure instead of silently dropping
+ * the cell. That keeps a flaky model from quietly inflating an
+ * aggregate by skipping its own failures.
+ *
+ * Sequential, not parallel — the browser entry can only host one model
+ * in WebGPU at a time, and the eviction guard in `web-llm.ts` already
+ * serializes cross-model loads. Running variants concurrently inside one
+ * model would also race the engine cache. Sequential is the only sound
+ * order.
+ */
+
+export interface RunEvalInput {
+ /** Models to compare. Each one must be loadable by the `rewriteFn`. */
+ modelIds: readonly string[];
+ /** Prompt variants to compare. */
+ variantIds: readonly string[];
+ /** Fixtures to evaluate. */
+ fixtures: readonly RewriteFixture[];
+ /** Inference seam — stub in tests, real-engine in the browser. */
+ rewriteFn: RewriteFn;
+ /**
+ * Optional cell-level progress callback. The browser entry uses this
+ * to update the page UI; tests pass nothing.
+ */
+ onProgress?: (
+ completed: number,
+ total: number,
+ cell: { modelId: string; variantId: string; fixtureId: string },
+ ) => void;
+ /**
+ * Set when the LLM-judge flag is on. The flag itself doesn't compute
+ * the judge here — the rubric's `judgeCoherence` slot stays `null`
+ * unless a future patch fills it in — but it's threaded through so the
+ * report can label which runs were judge-enabled.
+ */
+ judgeEnabled?: boolean;
+ /**
+ * Resumelint commit SHA the eval ran against, surfaced in the report.
+ * The browser entry passes `__APP_VERSION__`; tests pass `null`.
+ */
+ appVersion?: string | null;
+ /**
+ * Clock override for deterministic timing in tests. Defaults to
+ * `Date.now`. Tests pass a step-by-step ticker so duration assertions
+ * stay stable.
+ */
+ now?: () => number;
+}
+
+export async function runEval({
+ modelIds,
+ variantIds,
+ fixtures,
+ rewriteFn,
+ onProgress,
+ judgeEnabled = false,
+ appVersion = null,
+ now = Date.now,
+}: RunEvalInput): Promise {
+ const records: RunRecord[] = [];
+ const total = modelIds.length * variantIds.length * fixtures.length;
+ let completed = 0;
+ const startedAt = new Date(now()).toISOString();
+
+ for (const modelId of modelIds) {
+ for (const variantId of variantIds) {
+ for (const fixture of fixtures) {
+ const cellStart = now();
+ let record: RunRecord;
+ try {
+ const output = await rewriteFn({ modelId, variantId, fixture });
+ const rubric = scoreRubric({
+ input: fixture.bullets,
+ output,
+ fixtureKind: fixture.kind,
+ });
+ record = {
+ modelId,
+ variantId,
+ fixtureId: fixture.id,
+ fixtureKind: fixture.kind,
+ inputBulletCount: fixture.bullets.length,
+ outputBulletCount: output.bullets.length,
+ rubric,
+ rewriteDurationMs: now() - cellStart,
+ error: null,
+ };
+ } catch (err) {
+ record = {
+ modelId,
+ variantId,
+ fixtureId: fixture.id,
+ fixtureKind: fixture.kind,
+ inputBulletCount: fixture.bullets.length,
+ outputBulletCount: 0,
+ rubric: emptyRubricForError(),
+ rewriteDurationMs: now() - cellStart,
+ error: err instanceof Error ? err.message : String(err),
+ };
+ }
+ records.push(record);
+ completed += 1;
+ onProgress?.(completed, total, {
+ modelId,
+ variantId,
+ fixtureId: fixture.id,
+ });
+ }
+ }
+ }
+
+ return {
+ startedAt,
+ appVersion,
+ modelIds,
+ variantIds,
+ fixtureIds: fixtures.map((f) => f.id),
+ judgeEnabled,
+ records,
+ aggregates: aggregateRecords(records, modelIds, variantIds),
+ };
+}
+
+/**
+ * Per-(model, variant) aggregate. The dedup rate is computed over
+ * `redundant` fixtures only — if the set has none, the slot is `null`
+ * and the report renders `—`. The composite `aggregateScore` is the
+ * equal-weight mean of the deterministic rates (judge excluded) so
+ * choosing a default-model from the report is one column.
+ */
+function aggregateRecords(
+ records: readonly RunRecord[],
+ modelIds: readonly string[],
+ variantIds: readonly string[],
+): AggregateRow[] {
+ const rows: AggregateRow[] = [];
+ for (const modelId of modelIds) {
+ for (const variantId of variantIds) {
+ const cell = records.filter(
+ (r) => r.modelId === modelId && r.variantId === variantId,
+ );
+ const scored = cell.filter((r) => r.error === null);
+
+ const numbersPreservedRate = rate(scored, (r) => r.rubric.numbersPreserved);
+ const oneLineRate = rate(scored, (r) => r.rubric.oneLinePerBullet);
+ const actionVerbRate = rate(scored, (r) => r.rubric.actionVerbLead);
+ const lengthSanityRate = rate(scored, (r) => r.rubric.lengthSanity);
+ const noPreambleLeakRate = rate(scored, (r) => r.rubric.noPreambleLeak);
+
+ const redundantCell = scored.filter((r) => r.fixtureKind === "redundant");
+ const dedupEffectiveRate =
+ redundantCell.length === 0
+ ? null
+ : redundantCell.filter((r) => r.rubric.dedupEffective === true).length /
+ redundantCell.length;
+
+ const judgeScores = scored
+ .map((r) => r.rubric.judgeCoherence)
+ .filter((v): v is number => v !== null);
+ const judgeMean =
+ judgeScores.length === 0
+ ? null
+ : judgeScores.reduce((s, v) => s + v, 0) / judgeScores.length;
+
+ const deterministicRates = [
+ numbersPreservedRate,
+ oneLineRate,
+ actionVerbRate,
+ lengthSanityRate,
+ noPreambleLeakRate,
+ ...(dedupEffectiveRate === null ? [] : [dedupEffectiveRate]),
+ ];
+ const aggregateScore =
+ deterministicRates.reduce((s, v) => s + v, 0) / deterministicRates.length;
+
+ rows.push({
+ modelId,
+ variantId,
+ scoredFixtures: scored.length,
+ numbersPreservedRate,
+ oneLineRate,
+ actionVerbRate,
+ lengthSanityRate,
+ noPreambleLeakRate,
+ dedupEffectiveRate,
+ judgeMean,
+ aggregateScore,
+ });
+ }
+ }
+ return rows;
+}
+
+function rate(records: readonly RunRecord[], pred: (r: RunRecord) => boolean): number {
+ if (records.length === 0) return 0;
+ return records.filter(pred).length / records.length;
+}
diff --git a/src/lib/webllm/eval/types.ts b/src/lib/webllm/eval/types.ts
new file mode 100644
index 00000000..3c26522a
--- /dev/null
+++ b/src/lib/webllm/eval/types.ts
@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 The resumelint Authors
+
+/**
+ * Shared types for the rewrite-quality eval harness (issue #65).
+ *
+ * The harness is a Node/Vitest-runnable scoring pipeline that grades the
+ * output of a section-rewrite against a deterministic rubric. The model
+ * inference leg is browser-only (WebGPU); the scoring leg is
+ * model-agnostic and ships with full unit coverage in CI.
+ *
+ * The shapes here are the seam between those two legs: a `RewriteFn`
+ * implementation (real engine in the browser, stub in CI/tests) produces
+ * `RawRewriteOutput` records, and the runner feeds them into `scoreRubric`.
+ */
+
+/**
+ * Fixture kind drives which rubric criteria are applicable. `redundant`
+ * fixtures expect dedup; `numeric` fixtures expect strict number
+ * preservation; `strong` fixtures expect minimal change. The kind is the
+ * fixture's claim about itself, not a measured property — it gates rule
+ * application, not pass/fail.
+ */
+export type FixtureKind = "weak" | "strong" | "numeric" | "redundant";
+
+/**
+ * One résumé-section fixture: a tagged set of input bullets.
+ *
+ * `description` is for humans reading the committed report — it explains
+ * what the fixture is testing. `bullets` are the input to the rewrite; the
+ * runner does NOT mutate or filter them.
+ */
+export interface RewriteFixture {
+ /** Stable identifier used in report tables. Kebab-case. */
+ id: string;
+ /** Which rubric criteria apply (see `FixtureKind`). */
+ kind: FixtureKind;
+ /** Human-readable description for the committed report. */
+ description: string;
+ /** The input bullets passed to the rewrite. */
+ bullets: readonly string[];
+}
+
+/**
+ * Per-criterion pass/fail booleans + diagnostic detail. Each field is a
+ * deterministic, model-free check; no field requires a judge model. The
+ * optional `judge` slot is the gated coherence score from #65's optional
+ * AC — null when the flag is off (the default).
+ */
+export interface RubricResult {
+ /** Every numeric token from input survived; none invented. */
+ numbersPreserved: boolean;
+ /** Every output bullet is a single line (no embedded `\n`). */
+ oneLinePerBullet: boolean;
+ /** Every output bullet's first token is in the curated verb list. */
+ actionVerbLead: boolean;
+ /** Every output bullet length lies inside the sanity band. */
+ lengthSanity: boolean;
+ /** Output contains none of the prompt-scaffolding echo phrases. */
+ noPreambleLeak: boolean;
+ /**
+ * For `redundant` fixtures: output bullet count < input bullet count.
+ * `null` for non-redundant fixtures (the criterion does not apply).
+ */
+ dedupEffective: boolean | null;
+ /**
+ * Flag-gated LLM-judge coherence score, 0..1. `null` when the judge is
+ * off (default in CI and the committed scripts). Never required for any
+ * acceptance gate — the harness reports it advisory-only.
+ */
+ judgeCoherence: number | null;
+ /** Per-bullet diagnostic detail surfaced in the report. */
+ perBullet: PerBulletDiagnostic[];
+ /**
+ * Numbers that the model dropped from input (multiset diff). Empty when
+ * numbersPreserved is true.
+ */
+ droppedNumbers: string[];
+ /**
+ * Numbers that appeared in output but not input (multiset diff). Empty
+ * when numbersPreserved is true.
+ */
+ addedNumbers: string[];
+}
+
+export interface PerBulletDiagnostic {
+ /** Index in the output (0-based). */
+ index: number;
+ /** The bullet text, post-cleanup. */
+ text: string;
+ /** First-token check (one of the rubric criteria). */
+ startsWithActionVerb: boolean;
+ /** Length-sanity check (one of the rubric criteria). */
+ lengthOk: boolean;
+ /** Single-line check (the input line had no embedded `\n`). */
+ oneLine: boolean;
+}
+
+/**
+ * Raw rewrite output produced by a `RewriteFn`. The runner feeds this
+ * straight into `scoreRubric` — the rubric does NOT call the model.
+ */
+export interface RawRewriteOutput {
+ /** Rewritten bullets, post the shared `cleanRewriteLine` cleanup. */
+ bullets: readonly string[];
+ /**
+ * Raw model output before line-splitting, kept so the rubric can spot
+ * preamble leakage across the whole response (not just per-bullet).
+ */
+ raw: string;
+}
+
+/**
+ * The pluggable inference seam. The Node scoring tests pass a stub that
+ * returns canned outputs; the browser entry passes a real WebLLM-backed
+ * implementation. Neither leg owns the rubric — they only produce the
+ * output the rubric consumes.
+ */
+export type RewriteFn = (input: {
+ modelId: string;
+ variantId: string;
+ fixture: RewriteFixture;
+}) => Promise;
+
+/** A prompt variant in the compare matrix. */
+export interface PromptVariant {
+ /** Stable identifier used in report tables. Kebab-case. */
+ id: string;
+ /** Human-readable label for the committed report. */
+ label: string;
+ /** System prompt the model is asked to follow. */
+ systemPrompt: string;
+}
+
+/** One row in the (model × variant × fixture) matrix. */
+export interface RunRecord {
+ modelId: string;
+ variantId: string;
+ fixtureId: string;
+ fixtureKind: FixtureKind;
+ inputBulletCount: number;
+ outputBulletCount: number;
+ rubric: RubricResult;
+ /** Wall-clock ms spent inside the `RewriteFn` (browser-leg only). */
+ rewriteDurationMs: number | null;
+ /**
+ * Set when the `RewriteFn` threw or returned an unparseable response.
+ * The runner records the error and moves on — the row scores 0 across
+ * all criteria so it shows up in the report instead of being silently
+ * skipped.
+ */
+ error: string | null;
+}
+
+/** Aggregate report shape that report.ts formats. */
+export interface EvalReport {
+ /** ISO-8601 timestamp the run started. */
+ startedAt: string;
+ /** Resumelint commit SHA the eval ran against, if resolvable. */
+ appVersion: string | null;
+ /** Models compared in this run. */
+ modelIds: readonly string[];
+ /** Prompt variants compared in this run. */
+ variantIds: readonly string[];
+ /** Fixtures evaluated. */
+ fixtureIds: readonly string[];
+ /** Whether the judge flag was set when this run executed. */
+ judgeEnabled: boolean;
+ /** Per-row records. */
+ records: readonly RunRecord[];
+ /** Per-(model, variant) aggregate over fixtures. */
+ aggregates: readonly AggregateRow[];
+}
+
+export interface AggregateRow {
+ modelId: string;
+ variantId: string;
+ /** Number of fixtures that produced a usable rubric (i.e. not errored). */
+ scoredFixtures: number;
+ /** 0..1 per-criterion pass rate across scored fixtures. */
+ numbersPreservedRate: number;
+ oneLineRate: number;
+ actionVerbRate: number;
+ lengthSanityRate: number;
+ noPreambleLeakRate: number;
+ /** 0..1 across `redundant` fixtures only; `null` if none in the set. */
+ dedupEffectiveRate: number | null;
+ /** Mean judge score across scored fixtures; `null` when judge is off. */
+ judgeMean: number | null;
+ /** Equal-weight mean of the deterministic rates (judge excluded). */
+ aggregateScore: number;
+}
diff --git a/src/lib/webllm/eval/verbs.ts b/src/lib/webllm/eval/verbs.ts
new file mode 100644
index 00000000..44e54b8b
--- /dev/null
+++ b/src/lib/webllm/eval/verbs.ts
@@ -0,0 +1,69 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 The resumelint Authors
+
+import { ACTION_VERBS as SCORER_ACTION_VERBS } from "../../score/score.ts";
+
+/**
+ * Action-verb list used by the eval rubric's `actionVerbLead` criterion.
+ *
+ * Built as the scorer's curated past-tense set + a small eval-only
+ * extension. The scorer set lives in `src/lib/score/score.ts` so when a
+ * verb is added there it lights up here automatically — single source of
+ * truth, no drift.
+ *
+ * The extension covers two cases the scorer set doesn't:
+ *
+ * 1. Tense breadth — small instruct models occasionally emit
+ * present-progressive forms ("Building / Driving / Owning") in
+ * rewrite output. The scorer never sees those because users write
+ * résumés in past tense.
+ * 2. Cross-discipline verbs — "analyzed / authored / wrote / programmed"
+ * are normal for IC and writing-heavy roles. The scorer set leans
+ * eng/PM and would over-penalize a research-coded résumé.
+ *
+ * Weak generic verbs ("worked", "helped", "responsible", "assisted",
+ * "participated") are deliberately absent — a bullet leading with one of
+ * those SHOULD fail the criterion. That's the whole point.
+ */
+
+const EVAL_ONLY_EXTENSIONS: readonly string[] = [
+ // Eng / data IC verbs the scorer set doesn't cover.
+ "analyzed", "authored", "configured", "debugged", "deployed",
+ "engineered", "investigated", "prototyped", "rewrote", "shipped",
+ "tested", "validated", "wrote",
+ // Cross-discipline (research / ops / comms) IC verbs.
+ "completed", "conducted", "drafted", "identified", "owned",
+ "performed", "planned", "presented", "produced", "published",
+ "secured", "tracked",
+ // Present-progressive forms small models sometimes emit.
+ "building", "driving", "leading", "managing", "designing",
+ "shipping", "scaling", "owning",
+];
+
+// Module-internal: only `startsWithActionVerb` is consumed by the rubric.
+// Not exported — keeping it local avoids a dead public export and a
+// name collision with `score.ts`'s `ACTION_VERBS` (both flagged by fallow).
+const ACTION_VERBS: ReadonlySet = new Set([
+ ...SCORER_ACTION_VERBS,
+ ...EVAL_ONLY_EXTENSIONS,
+]);
+
+/**
+ * First-token check that mirrors `score.ts::startsWithActionVerb`:
+ * lowercase the first whitespace-delimited token, strip everything that
+ * isn't a-z, and look up in the union set. The strip handles trailing
+ * punctuation (`Led,`, `Shipped:`) without expanding the set with
+ * decorated variants.
+ *
+ * Returns `false` for an empty bullet — empty bullets should never make
+ * it past the rubric's line-splitting cleanup, but the guard keeps the
+ * behavior defined.
+ */
+export function startsWithActionVerb(bullet: string): boolean {
+ const firstWord = bullet
+ .split(/\s/)[0]
+ ?.toLowerCase()
+ .replace(/[^a-z]/g, "");
+ if (!firstWord) return false;
+ return ACTION_VERBS.has(firstWord);
+}
diff --git a/tests/fixtures/rewrite/README.md b/tests/fixtures/rewrite/README.md
new file mode 100644
index 00000000..c9c9d52a
--- /dev/null
+++ b/tests/fixtures/rewrite/README.md
@@ -0,0 +1,52 @@
+# Rewrite eval fixtures
+
+Synthetic résumé-section fixtures consumed by the rewrite-quality eval
+harness (issue #65; see `src/lib/webllm/eval/README.md`).
+
+## Layout
+
+```
+tests/fixtures/rewrite/
+├── weak.json # vague bullets, no metrics, weak verbs
+├── strong.json # already-strong bullets the rewrite should leave intact
+├── numeric.json # bullets dense with metrics — number-preservation stress
+├── redundant.json # deliberate duplicates the rewrite should collapse
+└── reports/ # committed reports from local WebGPU runs
+```
+
+## Fixture shape
+
+```json
+{
+ "id": "kebab-case-id",
+ "kind": "weak | strong | numeric | redundant",
+ "description": "What this fixture is exercising, surfaced in the report.",
+ "bullets": ["...", "..."]
+}
+```
+
+`parseFixture` in `src/lib/webllm/eval/fixtures.ts` validates the shape
+at module load; a malformed file throws before any eval runs.
+
+## PII policy
+
+The repo's general PII rule
+(`tests/fixtures/pdfs/README.md` Privacy section) applies to these
+fixtures too: **synthetic personas only**. Bullet fixtures don't carry
+contact info, but keep employer names, dates, locations, and project
+details fictional. The repo is public — anything you commit is
+permanently searchable, even after a later removal commit.
+
+The committed reports under `reports/` are derived from these fixtures
+running through models locally; they inherit the same PII-cleanliness as
+long as the fixtures themselves stay clean.
+
+## Adding a fixture
+
+1. Drop a new JSON file in this directory matching the shape above.
+2. Append an `import` + `parseFixture(...)` line in
+ `src/lib/webllm/eval/fixtures.ts::REWRITE_FIXTURES`.
+3. Run `npm run test src/lib/webllm/eval/fixtures.test.ts` to verify it
+ loads.
+4. Run `npm run eval:rewrite` locally (WebGPU required) to regenerate
+ committed reports.
diff --git a/tests/fixtures/rewrite/numeric.json b/tests/fixtures/rewrite/numeric.json
new file mode 100644
index 00000000..9f69fa85
--- /dev/null
+++ b/tests/fixtures/rewrite/numeric.json
@@ -0,0 +1,12 @@
+{
+ "id": "numeric-growth-pm",
+ "kind": "numeric",
+ "description": "Growth-PM bullets dense with concrete numbers — currency, percent, magnitude, headcount, years. The number-preservation guardrail must catch any drop or invention.",
+ "bullets": [
+ "Grew weekly active users from 120K to 1.8M between 2022 and 2024 by shipping 7 onboarding experiments, lifting day-7 retention 14%.",
+ "Led a 6-person growth squad that drove $4.2M incremental ARR through pricing-page experiments and a 2-tier paywall redesign.",
+ "Reduced free-to-paid conversion friction with a checkout overhaul that lifted conversion 23% and cut median time-to-purchase from 4.5 minutes to 90 seconds.",
+ "Owned 3 quarterly OKRs spanning acquisition, activation, and retention; hit 100% of activation targets and 85% of acquisition targets in FY2023.",
+ "Negotiated a $250K co-marketing budget with 2 partner brands, delivering 11.5M impressions and 38K signups across a 6-week campaign."
+ ]
+}
diff --git a/tests/fixtures/rewrite/redundant.json b/tests/fixtures/rewrite/redundant.json
new file mode 100644
index 00000000..758b3bef
--- /dev/null
+++ b/tests/fixtures/rewrite/redundant.json
@@ -0,0 +1,12 @@
+{
+ "id": "redundant-support-lead",
+ "kind": "redundant",
+ "description": "Support-lead bullets with deliberate overlap — three near-duplicate descriptions of the same ticket-triage work, plus an unrelated onboarding bullet that should survive. A good rewrite should collapse the overlap to one or two strong bullets, so output count must drop below input count.",
+ "bullets": [
+ "Triaged 200+ inbound customer support tickets per week across email and chat channels.",
+ "Managed a 200/week ticket queue across email and chat, prioritizing by SLA and severity.",
+ "Handled customer support tickets at a rate of around 200 per week from email and chat sources.",
+ "Resolved escalated billing disputes for 40 enterprise accounts, recovering $85K in disputed charges.",
+ "Onboarded 8 new support representatives with a 3-week training program that cut ramp time from 60 days to 28 days."
+ ]
+}
diff --git a/tests/fixtures/rewrite/reports/README.md b/tests/fixtures/rewrite/reports/README.md
new file mode 100644
index 00000000..3ab1c10a
--- /dev/null
+++ b/tests/fixtures/rewrite/reports/README.md
@@ -0,0 +1,39 @@
+# Rewrite eval reports
+
+Committed JSON + Markdown reports from local `npm run eval:rewrite` runs.
+
+Each run is one model, and produces two files named with the model
+slug + a UTC timestamp:
+
+```
+eval-rewrite--YYYY-MM-DDTHH-MM-SS-sssZ.json
+eval-rewrite--YYYY-MM-DDTHH-MM-SS-sssZ.md
+```
+
+To compare all three registry models, run the eval three times (one
+per tab) and commit all three pairs.
+
+Reports are append-only — **never overwrite** a prior run. A new commit
+adds a new pair; the historical record is what lets a future maintainer
+justify (or revisit) a `DEFAULT_MODEL_ID` change against the timeline of
+prompt + model changes.
+
+The JSON is machine-diffable across runs (per-criterion rates +
+per-cell records); the Markdown is the human-readable layer linked into
+PR descriptions.
+
+## Workflow
+
+```sh
+npm run eval:rewrite # opens /resumelint/eval-rewrite.html with WebGPU
+# in the browser: click "Run eval", wait, download both report files
+mv ~/Downloads/eval-rewrite-*.json tests/fixtures/rewrite/reports/
+mv ~/Downloads/eval-rewrite-*.md tests/fixtures/rewrite/reports/
+git add tests/fixtures/rewrite/reports/
+git commit -m "eval(rewrite): snapshot YYYY-MM-DD run"
+```
+
+A baked-in baseline report will land here in a follow-up PR once the
+first WebGPU run is captured on a maintainer machine; this directory is
+intentionally empty in the PR that introduces the harness so the
+artifact reflects a real run, not a synthetic placeholder.
diff --git a/tests/fixtures/rewrite/reports/eval-rewrite-gemma-2-2b-it-q4f16-1-mlc-2026-06-23T19-03-56-461Z.json b/tests/fixtures/rewrite/reports/eval-rewrite-gemma-2-2b-it-q4f16-1-mlc-2026-06-23T19-03-56-461Z.json
new file mode 100644
index 00000000..0ff57309
--- /dev/null
+++ b/tests/fixtures/rewrite/reports/eval-rewrite-gemma-2-2b-it-q4f16-1-mlc-2026-06-23T19-03-56-461Z.json
@@ -0,0 +1,844 @@
+{
+ "startedAt": "2026-06-23T19:01:41.578Z",
+ "appVersion": "8c12fb0",
+ "modelIds": [
+ "gemma-2-2b-it-q4f16_1-MLC"
+ ],
+ "variantIds": [
+ "baseline",
+ "terse",
+ "examples-led"
+ ],
+ "fixtureIds": [
+ "weak-marketing-generalist",
+ "strong-backend-engineer",
+ "numeric-growth-pm",
+ "redundant-support-lead"
+ ],
+ "judgeEnabled": false,
+ "records": [
+ {
+ "modelId": "gemma-2-2b-it-q4f16_1-MLC",
+ "variantId": "baseline",
+ "fixtureId": "weak-marketing-generalist",
+ "fixtureKind": "weak",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Oversaw marketing tasks, ensuring seamless team support and efficient daily operations.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Developed and executed social media campaigns that boosted engagement by 20% across multiple platforms.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Contributed to content creation, email blasts, and promotional materials, driving brand awareness and lead generation.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Supported senior team members in planning and coordinating marketing events, resulting in a 15% increase in attendance.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Provided strategic input and creative direction during weekly meetings, contributing to a 10% improvement in campaign effectiveness.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": [
+ "20%",
+ "15%",
+ "10%"
+ ]
+ },
+ "rewriteDurationMs": 6847,
+ "error": null
+ },
+ {
+ "modelId": "gemma-2-2b-it-q4f16_1-MLC",
+ "variantId": "baseline",
+ "fixtureId": "strong-backend-engineer",
+ "fixtureKind": "strong",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": true,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Led a Kafka-based migration that reduced median latency across 12 downstream services by 38% while achieving operational efficiency.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Designed a horizontally-sharded Postgres architecture that absorbed a 4x traffic surge during the 2024 product launch without requiring read-replica failover.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Mentored five engineers through code review and pairing sessions, contributing to the successful delivery of 23 production features across two quarters with zero rollback incidents.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Reduced cloud spending by $180,000 annually through rightsizing 60 underutilized instances and consolidating three staging environments into a single tenant-isolated environment.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Built a deterministic load generator that replicated 24 hours of production traffic in 90 minutes, enabling four teams to conduct regression checks with increased efficiency.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "5",
+ "$180K"
+ ],
+ "addedNumbers": [
+ "$180,000"
+ ]
+ },
+ "rewriteDurationMs": 9768,
+ "error": null
+ },
+ {
+ "modelId": "gemma-2-2b-it-q4f16_1-MLC",
+ "variantId": "baseline",
+ "fixtureId": "numeric-growth-pm",
+ "fixtureKind": "numeric",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Increased weekly active users by 1500% to 1.8 million users between 2022 and 2024, successfully launching 7 onboarding experiments.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Spearheaded a growth team that generated $4.2 million in incremental ARR through pricing-page experiments and a 2-tier paywall redesign.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Successfully reduced free-to-paid conversion friction by 23%, decreasing median time-to-purchase from 4.5 minutes to 90 seconds through a checkout overhaul.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Successfully achieved 100% activation targets and 85% acquisition targets across three quarterly OKRs in FY2023.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Secured a $250,000 co-marketing budget with two partner brands, generating 11.5 million impressions and 38,000 signups through a six-week campaign.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "120K",
+ "1.8M",
+ "14%",
+ "$4.2M",
+ "$250K",
+ "11.5M",
+ "38K"
+ ],
+ "addedNumbers": [
+ "1500%",
+ "1.8",
+ "$4.2",
+ "$250,000",
+ "11.5",
+ "38,000"
+ ]
+ },
+ "rewriteDurationMs": 11274,
+ "error": null
+ },
+ {
+ "modelId": "gemma-2-2b-it-q4f16_1-MLC",
+ "variantId": "baseline",
+ "fixtureId": "redundant-support-lead",
+ "fixtureKind": "redundant",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": false,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Triaged and resolved an average of 200+ inbound customer support tickets per week across email and chat channels.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Successfully managed a 200/week ticket queue, prioritizing customer support requests by SLA and severity.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Handled a consistent flow of 200+ customer support tickets per week from email and chat sources.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Successfully resolved escalated billing disputes for 40 enterprise accounts, recovering a total of $85,000 in disputed charges.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Developed and implemented a new onboarding program for new support representatives, reducing ramp time from 60 days to 28 days.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "$85K"
+ ],
+ "addedNumbers": [
+ "$85,000"
+ ]
+ },
+ "rewriteDurationMs": 7641,
+ "error": null
+ },
+ {
+ "modelId": "gemma-2-2b-it-q4f16_1-MLC",
+ "variantId": "terse",
+ "fixtureId": "weak-marketing-generalist",
+ "fixtureKind": "weak",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": true,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Spearheaded marketing tasks, providing essential support to the team and ensuring seamless daily operations.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Developed and executed social media campaigns that significantly boosted engagement across various channels.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Contributed to content creation, email blasts, and other promotional initiatives, effectively driving brand awareness and visibility.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Collaborated with senior team members to plan and coordinate marketing events and initiatives, ensuring successful execution and alignment with overall goals.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Actively participated in weekly meetings, offering strategic input and creative direction to optimize marketing efforts.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": []
+ },
+ "rewriteDurationMs": 6173,
+ "error": null
+ },
+ {
+ "modelId": "gemma-2-2b-it-q4f16_1-MLC",
+ "variantId": "terse",
+ "fixtureId": "strong-backend-engineer",
+ "fixtureKind": "strong",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": true,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Reduced latency across 12 downstream services by 38% through the migration of the order-processing pipeline to event-driven Kafka topics.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Designed and implemented a horizontally-sharded Postgres layout that absorbed a 4x traffic surge during the 2024 product launch, maintaining consistent performance without requiring read-replica failover.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Mentored five engineers through code review and pairing sessions, resulting in the successful delivery of 23 production features over two quarters with zero rollback incidents.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Optimized cloud infrastructure, reducing annual cloud spend by $180K through rightsizing 60 underutilized instances and consolidating three staging environments into a single tenant-isolated environment.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Developed a deterministic load generator that replicated 24 hours of production traffic in 90 minutes, enabling four teams to conduct regression checks with greater efficiency.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "5"
+ ],
+ "addedNumbers": []
+ },
+ "rewriteDurationMs": 10886,
+ "error": null
+ },
+ {
+ "modelId": "gemma-2-2b-it-q4f16_1-MLC",
+ "variantId": "terse",
+ "fixtureId": "numeric-growth-pm",
+ "fixtureKind": "numeric",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "**Increased** weekly active users by 1500% (120,000 to 1.8 million) by implementing seven onboarding experiments and driving a 14% improvement in day-7 retention.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "**Spearheaded** a growth team of six individuals, generating $4.2 million in incremental ARR through pricing-page experiments and a two-tier paywall redesign.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "**Streamlined** the checkout process, resulting in a 23% conversion rate increase and a 90% reduction in median time-to-purchase from 4.5 minutes to 90 seconds.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "**Successfully** owned and achieved 100% of activation targets and 85% of acquisition targets across three quarterly objectives in FY2023.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "**Negotiated** a $250,000 co-marketing budget with two partner brands, generating 11.5 million impressions and 38,000 signups through a six-week campaign.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "120K",
+ "1.8M",
+ "2022",
+ "2024",
+ "$4.2M",
+ "$250K",
+ "11.5M",
+ "38K"
+ ],
+ "addedNumbers": [
+ "1500%",
+ "120,000",
+ "1.8",
+ "$4.2",
+ "90%",
+ "$250,000",
+ "11.5",
+ "38,000"
+ ]
+ },
+ "rewriteDurationMs": 13653,
+ "error": null
+ },
+ {
+ "modelId": "gemma-2-2b-it-q4f16_1-MLC",
+ "variantId": "terse",
+ "fixtureId": "redundant-support-lead",
+ "fixtureKind": "redundant",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": false,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "**Triaged** an average of 200+ inbound customer support tickets per week across email and chat channels, efficiently routing them to appropriate channels and escalating critical issues.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "**Oversaw** a 200/week ticket queue, prioritizing customer support requests based on SLA and severity, ensuring timely resolution and customer satisfaction.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "**Successfully resolved** approximately 200 customer support tickets per week from email and chat sources, consistently exceeding customer expectations and minimizing support ticket backlog.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "**Successfully resolved** escalated billing disputes for 40 enterprise accounts, recovering a total of $85,000 in disputed charges.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "**Developed and implemented** a 3-week onboarding program for new support representatives, resulting in a 32-day reduction in ramp time from 60 days to 28 days.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "$85K"
+ ],
+ "addedNumbers": [
+ "$85,000"
+ ]
+ },
+ "rewriteDurationMs": 11664,
+ "error": null
+ },
+ {
+ "modelId": "gemma-2-2b-it-q4f16_1-MLC",
+ "variantId": "examples-led",
+ "fixtureId": "weak-marketing-generalist",
+ "fixtureKind": "weak",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Spearheaded social media campaigns that increased engagement by 25% and boosted website traffic by 15%.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Contributed to content creation, including blog posts and email newsletters, resulting in a 10% increase in email open rates.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Supported marketing team efforts by developing and executing promotional materials for product launches, driving a 12% lift in sales conversion.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Collaborated with senior team members to plan and coordinate marketing events, resulting in a 20% increase in attendance and a 15% improvement in lead generation.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Actively participated in weekly marketing meetings, providing strategic input and creative direction, and contributing to a 5% improvement in campaign performance.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": [
+ "25%",
+ "15%",
+ "10%",
+ "12%",
+ "20%",
+ "15%",
+ "5%"
+ ]
+ },
+ "rewriteDurationMs": 10743,
+ "error": null
+ },
+ {
+ "modelId": "gemma-2-2b-it-q4f16_1-MLC",
+ "variantId": "examples-led",
+ "fixtureId": "strong-backend-engineer",
+ "fixtureKind": "strong",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": true,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Led a Kafka-based migration of the order-processing pipeline, reducing median latency by 38% across 12 downstream services.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Designed a horizontally-sharded Postgres layout that absorbed a 4x traffic surge during the 2024 product launch without requiring read-replica failover.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Mentored five engineers through code review and pairing sessions, contributing to the successful delivery of 23 production features over two quarters with zero rollback incidents.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Optimized cloud infrastructure, reducing annual cloud spend by $180,000 through rightsizing 60 underused instances and consolidating three staging environments into a single tenant-isolated environment.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Developed a deterministic load generator that replays 24 hours of production traffic in 90 minutes, enabling four teams to conduct regression checks with greater efficiency.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "5",
+ "$180K"
+ ],
+ "addedNumbers": [
+ "$180,000"
+ ]
+ },
+ "rewriteDurationMs": 13974,
+ "error": null
+ },
+ {
+ "modelId": "gemma-2-2b-it-q4f16_1-MLC",
+ "variantId": "examples-led",
+ "fixtureId": "numeric-growth-pm",
+ "fixtureKind": "numeric",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Increased weekly active users by 1500% (120,000 to 1.8 million) between 2022 and 2024 by shipping seven onboarding experiments, boosting day-7 retention by 14%.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Led a six-person growth team that generated $4.2 million in incremental ARR through pricing-page experiments and a two-tier paywall redesign.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Optimized the checkout process, resulting in a 23% increase in free-to-paid conversion rates and a 90% reduction in median time-to-purchase from 4.5 minutes to 90 seconds.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Successfully achieved 100% activation targets and 85% acquisition targets across three quarterly objectives in FY2023.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Secured a $250,000 co-marketing budget with two partner brands, generating 11.5 million impressions and 38,000 signups through a six-week campaign.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "120K",
+ "1.8M",
+ "$4.2M",
+ "$250K",
+ "11.5M",
+ "38K"
+ ],
+ "addedNumbers": [
+ "1500%",
+ "120,000",
+ "1.8",
+ "$4.2",
+ "90%",
+ "$250,000",
+ "11.5",
+ "38,000"
+ ]
+ },
+ "rewriteDurationMs": 18374,
+ "error": null
+ },
+ {
+ "modelId": "gemma-2-2b-it-q4f16_1-MLC",
+ "variantId": "examples-led",
+ "fixtureId": "redundant-support-lead",
+ "fixtureKind": "redundant",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": false,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Triaged an average of 200+ customer support tickets per week across email and chat channels, ensuring timely resolution.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Optimized a 200/week ticket queue by prioritizing support requests based on SLA and severity, achieving a 10% reduction in average resolution time.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Handled an average of 200 customer support tickets per week, effectively resolving inquiries and addressing escalations from email and chat sources.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Successfully resolved escalated billing disputes for 40 enterprise accounts, recovering a total of $85,000 in disputed charges.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Developed and implemented a comprehensive onboarding program for 8 new support representatives, significantly reducing ramp time from 60 days to 28 days.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "$85K"
+ ],
+ "addedNumbers": [
+ "10%",
+ "$85,000"
+ ]
+ },
+ "rewriteDurationMs": 13875,
+ "error": null
+ }
+ ],
+ "aggregates": [
+ {
+ "modelId": "gemma-2-2b-it-q4f16_1-MLC",
+ "variantId": "baseline",
+ "scoredFixtures": 4,
+ "numbersPreservedRate": 0,
+ "oneLineRate": 1,
+ "actionVerbRate": 0.25,
+ "lengthSanityRate": 1,
+ "noPreambleLeakRate": 1,
+ "dedupEffectiveRate": 0,
+ "judgeMean": null,
+ "aggregateScore": 0.5416666666666666
+ },
+ {
+ "modelId": "gemma-2-2b-it-q4f16_1-MLC",
+ "variantId": "terse",
+ "scoredFixtures": 4,
+ "numbersPreservedRate": 0.25,
+ "oneLineRate": 1,
+ "actionVerbRate": 0.25,
+ "lengthSanityRate": 1,
+ "noPreambleLeakRate": 1,
+ "dedupEffectiveRate": 0,
+ "judgeMean": null,
+ "aggregateScore": 0.5833333333333334
+ },
+ {
+ "modelId": "gemma-2-2b-it-q4f16_1-MLC",
+ "variantId": "examples-led",
+ "scoredFixtures": 4,
+ "numbersPreservedRate": 0,
+ "oneLineRate": 1,
+ "actionVerbRate": 0.25,
+ "lengthSanityRate": 1,
+ "noPreambleLeakRate": 1,
+ "dedupEffectiveRate": 0,
+ "judgeMean": null,
+ "aggregateScore": 0.5416666666666666
+ }
+ ]
+}
diff --git a/tests/fixtures/rewrite/reports/eval-rewrite-gemma-2-2b-it-q4f16-1-mlc-2026-06-23T19-03-56-461Z.md b/tests/fixtures/rewrite/reports/eval-rewrite-gemma-2-2b-it-q4f16-1-mlc-2026-06-23T19-03-56-461Z.md
new file mode 100644
index 00000000..4e1791a9
--- /dev/null
+++ b/tests/fixtures/rewrite/reports/eval-rewrite-gemma-2-2b-it-q4f16-1-mlc-2026-06-23T19-03-56-461Z.md
@@ -0,0 +1,48 @@
+# Rewrite eval report
+
+- **Started:** 2026-06-23T19:01:41.578Z
+- **App version:** `8c12fb0`
+- **Models:** 1
+- **Prompt variants:** 3
+- **Fixtures:** 4
+- **LLM judge:** disabled (default)
+
+## Aggregate (per model × variant)
+
+| Model | Variant | Numbers | One-line | Verb | Length | No-preamble | Dedup | Judge | **Aggregate** |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| Gemma 2 (2B) | Baseline (shipped) | 0% | 100% | 25% | 100% | 100% | 0% | — | **54%** |
+| Gemma 2 (2B) | Terse (rules-only) | 25% | 100% | 25% | 100% | 100% | 0% | — | **58%** |
+| Gemma 2 (2B) | Examples-led (few-shot) | 0% | 100% | 25% | 100% | 100% | 0% | — | **54%** |
+
+## Per-cell records
+
+### Gemma 2 (2B)
+
+#### Baseline (shipped)
+
+| Fixture | Kind | In → Out | Numbers | Verb | Length | Preamble | Dedup | Error |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| weak-marketing-generalist | weak | 5 → 5 | fail | fail | PASS | PASS | — | |
+| strong-backend-engineer | strong | 5 → 5 | fail | PASS | PASS | PASS | — | |
+| numeric-growth-pm | numeric | 5 → 5 | fail | fail | PASS | PASS | — | |
+| redundant-support-lead | redundant | 5 → 5 | fail | fail | PASS | PASS | fail | |
+
+#### Terse (rules-only)
+
+| Fixture | Kind | In → Out | Numbers | Verb | Length | Preamble | Dedup | Error |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| weak-marketing-generalist | weak | 5 → 5 | PASS | fail | PASS | PASS | — | |
+| strong-backend-engineer | strong | 5 → 5 | fail | PASS | PASS | PASS | — | |
+| numeric-growth-pm | numeric | 5 → 5 | fail | fail | PASS | PASS | — | |
+| redundant-support-lead | redundant | 5 → 5 | fail | fail | PASS | PASS | fail | |
+
+#### Examples-led (few-shot)
+
+| Fixture | Kind | In → Out | Numbers | Verb | Length | Preamble | Dedup | Error |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| weak-marketing-generalist | weak | 5 → 5 | fail | fail | PASS | PASS | — | |
+| strong-backend-engineer | strong | 5 → 5 | fail | PASS | PASS | PASS | — | |
+| numeric-growth-pm | numeric | 5 → 5 | fail | fail | PASS | PASS | — | |
+| redundant-support-lead | redundant | 5 → 5 | fail | fail | PASS | PASS | fail | |
+
diff --git a/tests/fixtures/rewrite/reports/eval-rewrite-llama-3-2-3b-instruct-q4f16-1-mlc-2026-06-23T19-09-02-391Z.json b/tests/fixtures/rewrite/reports/eval-rewrite-llama-3-2-3b-instruct-q4f16-1-mlc-2026-06-23T19-09-02-391Z.json
new file mode 100644
index 00000000..6f266a35
--- /dev/null
+++ b/tests/fixtures/rewrite/reports/eval-rewrite-llama-3-2-3b-instruct-q4f16-1-mlc-2026-06-23T19-09-02-391Z.json
@@ -0,0 +1,865 @@
+{
+ "startedAt": "2026-06-23T19:06:32.946Z",
+ "appVersion": "8c12fb0",
+ "modelIds": [
+ "Llama-3.2-3B-Instruct-q4f16_1-MLC"
+ ],
+ "variantIds": [
+ "baseline",
+ "terse",
+ "examples-led"
+ ],
+ "fixtureIds": [
+ "weak-marketing-generalist",
+ "strong-backend-engineer",
+ "numeric-growth-pm",
+ "redundant-support-lead"
+ ],
+ "judgeEnabled": false,
+ "records": [
+ {
+ "modelId": "Llama-3.2-3B-Instruct-q4f16_1-MLC",
+ "variantId": "baseline",
+ "fixtureId": "weak-marketing-generalist",
+ "fixtureKind": "weak",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": true,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Developed and executed comprehensive marketing strategies to drive business growth and team productivity.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Led social media campaigns that increased engagement by 25% and expanded brand reach to 100,000 new followers.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Created high-quality content, including email blasts and promotional materials, that resulted in a 50% increase in sales leads.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Planned and coordinated marketing events, including trade shows and product launches, that exceeded revenue projections by 15%.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Facilitated strategic discussions and provided creative input at weekly team meetings, driving 90% alignment on marketing objectives.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": [
+ "25%",
+ "100,000",
+ "50%",
+ "15%",
+ "90%"
+ ]
+ },
+ "rewriteDurationMs": 7361,
+ "error": null
+ },
+ {
+ "modelId": "Llama-3.2-3B-Instruct-q4f16_1-MLC",
+ "variantId": "baseline",
+ "fixtureId": "strong-backend-engineer",
+ "fixtureKind": "strong",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": true,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Led the migration of the order-processing pipeline to event-driven Kafka topics, achieving a 38% reduction in median latency across 12 downstream services.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Designed a horizontally-sharded Postgres layout that successfully absorbed a 4x traffic surge during a 2024 product launch, without read-replica failover.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Mentored 5 engineers through code-review and pairing sessions, resulting in the delivery of 23 production features over 2 quarters, with zero rollback incidents.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Rightsized cloud instances, reducing annualized spend by $180K by consolidating 60 underused instances into a single tenant-isolated environment.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Developed a deterministic load generator that can replay 24 hours of production traffic in just 90 minutes, used by 4 teams for rigorous regression checks.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": []
+ },
+ "rewriteDurationMs": 10349,
+ "error": null
+ },
+ {
+ "modelId": "Llama-3.2-3B-Instruct-q4f16_1-MLC",
+ "variantId": "baseline",
+ "fixtureId": "numeric-growth-pm",
+ "fixtureKind": "numeric",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Expanded weekly active users to 1.8M between 2022 and 2024 through a series of 7 onboarding experiments, resulting in a 14% increase in day-7 retention.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Spearheaded a 6-person growth squad that drove $4.2M in incremental revenue through pricing-page experiments and a 2-tier paywall redesign.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Streamlined checkout processes, lifting conversion rates by 23% and reducing median time-to-purchase to 90 seconds.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Achieved 100% of activation targets and 85% of acquisition targets across 3 quarterly OKRs spanning acquisition, activation, and retention in FY2023.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Secured a $250K co-marketing budget with 2 partner brands, delivering 11.5M impressions and 38K signups across a 6-week campaign.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "120K",
+ "4.5"
+ ],
+ "addedNumbers": []
+ },
+ "rewriteDurationMs": 10896,
+ "error": null
+ },
+ {
+ "modelId": "Llama-3.2-3B-Instruct-q4f16_1-MLC",
+ "variantId": "baseline",
+ "fixtureId": "redundant-support-lead",
+ "fixtureKind": "redundant",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": false,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Triaged and resolved 200+ high-priority inbound customer support tickets per week, achieving a 95% first-response rate across email and chat channels.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Prioritized and managed a 200/week ticket queue, ensuring 99% of tickets were resolved within the SLA, and 95% were resolved within 2 hours.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Streamlined customer support operations, handling 200+ tickets per week from email and chat sources, with a 98% customer satisfaction rate.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Resolved escalated billing disputes for 40 enterprise accounts, recovering $85K in disputed charges and achieving a 100% customer satisfaction rate.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Developed and implemented a comprehensive onboarding program for 8 new support representatives, reducing ramp time from 60 days to 28 days and achieving a 90% onboarding success rate.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": [
+ "95%",
+ "99%",
+ "95%",
+ "98%",
+ "100%",
+ "90%"
+ ]
+ },
+ "rewriteDurationMs": 10396,
+ "error": null
+ },
+ {
+ "modelId": "Llama-3.2-3B-Instruct-q4f16_1-MLC",
+ "variantId": "terse",
+ "fixtureId": "weak-marketing-generalist",
+ "fixtureKind": "weak",
+ "inputBulletCount": 5,
+ "outputBulletCount": 6,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Here are the rewritten bullets:",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Spearheaded marketing task management, ensuring seamless execution of campaigns and daily team operations.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Developed and executed social media campaigns, resulting in 25% increase in engagement and 50% growth in followers.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Crafted high-quality content for email blasts, blog posts, and promotional materials, driving a 30% open rate and 20% conversion rate.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Collaborated with senior team members to plan and coordinate 5 marketing events, resulting in a 90% attendance rate and 25% sales boost.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 5,
+ "text": "Contributed to weekly strategy meetings, providing actionable insights that informed marketing decisions and drove a 20% increase in sales.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": [
+ "25%",
+ "50%",
+ "30%",
+ "20%",
+ "90%",
+ "25%",
+ "20%"
+ ]
+ },
+ "rewriteDurationMs": 8554,
+ "error": null
+ },
+ {
+ "modelId": "Llama-3.2-3B-Instruct-q4f16_1-MLC",
+ "variantId": "terse",
+ "fixtureId": "strong-backend-engineer",
+ "fixtureKind": "strong",
+ "inputBulletCount": 5,
+ "outputBulletCount": 6,
+ "rubric": {
+ "numbersPreserved": true,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Here are the rewritten bullets:",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Spearheaded the migration of the order-processing pipeline to event-driven Kafka topics, resulting in a 38% reduction in median latency across 12 downstream services.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Designed and implemented a horizontally-sharded Postgres layout that successfully absorbed a 4x traffic surge during a 2024 product launch, without any read-replica failover.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Mentored and guided 5 engineers through code-review and pairing sessions, driving the delivery of 23 production features over 2 quarters with zero rollback incidents.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Spearheaded a cloud cost optimization initiative, reducing annualized cloud spend by $180K by rightsizing 60 underutilized instances and consolidating 3 staging environments into a single tenant-isolated one.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 5,
+ "text": "Developed and deployed a deterministic load generator that can replay 24 hours of production traffic in just 90 minutes, used by 4 teams for rigorous regression checks.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": []
+ },
+ "rewriteDurationMs": 12100,
+ "error": null
+ },
+ {
+ "modelId": "Llama-3.2-3B-Instruct-q4f16_1-MLC",
+ "variantId": "terse",
+ "fixtureId": "numeric-growth-pm",
+ "fixtureKind": "numeric",
+ "inputBulletCount": 5,
+ "outputBulletCount": 6,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Here are the rewritten bullets:",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Spearheaded a 15-month experiment series, launching 7 onboarding experiments that collectively grew weekly active users from 120K to 1.8M by 2024.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Led a 6-person growth squad, driving $4.2M in incremental ARR through A/B testing and pricing-page redesigns that resulted in a 2-tier paywall.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Spearheaded a checkout overhaul, reducing free-to-paid conversion friction by 23% and slashing median time-to-purchase from 4.5 minutes to 90 seconds.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Owned and executed 3 quarterly OKRs, achieving 100% of activation targets and 85% of acquisition targets in FY2023, with a 25% increase in overall customer engagement.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 5,
+ "text": "Negotiated a $250K co-marketing budget with 2 partner brands, delivering 11.5M impressions and 38K signups across a 6-week campaign that outperformed projections by 15%.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "2022",
+ "14%"
+ ],
+ "addedNumbers": [
+ "25%",
+ "15%"
+ ]
+ },
+ "rewriteDurationMs": 14087,
+ "error": null
+ },
+ {
+ "modelId": "Llama-3.2-3B-Instruct-q4f16_1-MLC",
+ "variantId": "terse",
+ "fixtureId": "redundant-support-lead",
+ "fixtureKind": "redundant",
+ "inputBulletCount": 5,
+ "outputBulletCount": 6,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": false,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Here are the rewritten bullets:",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Spearheaded a high-volume inbound customer support operation, triaging over 200 tickets per week across multiple channels to ensure timely resolutions.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Spearheaded a 200-ticket-per-week ticket queue, expertly prioritizing by SLA and severity to minimize wait times and maximize customer satisfaction.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Spearheaded a high-volume email and chat support operation, handling over 200 tickets per week from these sources to deliver exceptional customer service.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Resolved complex escalated billing disputes for 40 enterprise accounts, recovering $85,000 in disputed charges and ensuring timely payment resolutions.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 5,
+ "text": "Designed and executed a 3-week training program for 8 new support representatives, significantly reducing ramp time from 60 days to just 28 days and ensuring seamless onboarding.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "$85K"
+ ],
+ "addedNumbers": [
+ "$85,000"
+ ]
+ },
+ "rewriteDurationMs": 11520,
+ "error": null
+ },
+ {
+ "modelId": "Llama-3.2-3B-Instruct-q4f16_1-MLC",
+ "variantId": "examples-led",
+ "fixtureId": "weak-marketing-generalist",
+ "fixtureKind": "weak",
+ "inputBulletCount": 5,
+ "outputBulletCount": 6,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Here are the rewritten bullets:",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Spearheaded a 30% increase in team productivity through efficient task delegation and prioritization.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Conceptualized and executed a 6-month social media campaign that drove a 25% boost in followers and a 15% increase in engagement.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Authored and edited 12 high-quality blog posts, resulting in a 50% increase in website traffic and a 20% boost in lead generation.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Collaborated with senior team members to plan and execute a 3-day brand awareness event, resulting in a 90% positive social media sentiment score.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 5,
+ "text": "Contributed to weekly strategy sessions, providing actionable insights that informed creative direction and drove a 12% increase in sales-qualified leads.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": [
+ "30%",
+ "25%",
+ "15%",
+ "50%",
+ "20%",
+ "90%",
+ "12%"
+ ]
+ },
+ "rewriteDurationMs": 12204,
+ "error": null
+ },
+ {
+ "modelId": "Llama-3.2-3B-Instruct-q4f16_1-MLC",
+ "variantId": "examples-led",
+ "fixtureId": "strong-backend-engineer",
+ "fixtureKind": "strong",
+ "inputBulletCount": 5,
+ "outputBulletCount": 6,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Here are the rewritten bullets:",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Spearheaded a 38% latency reduction by migrating the order-processing pipeline to event-driven Kafka topics, resulting in a 12% boost in downstream service performance.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Designed a highly available Postgres layout that withstood a 4x traffic surge during the 2024 product launch, without read-replica failover and minimal downtime.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Coached 5 engineers through rigorous code-review and pairing sessions, delivering 23 production features over 2 quarters with zero rollback incidents and significant team productivity gains.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Streamlined cloud spend by $180K annually through rightsizing 60 underutilized instances and consolidating 3 staging environments into a single, tenant-isolated environment.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 5,
+ "text": "Developed a deterministic load generator that can replay 24 hours of production traffic in just 90 minutes, enabling 4 teams to perform comprehensive regression checks.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": [
+ "12%"
+ ]
+ },
+ "rewriteDurationMs": 16311,
+ "error": null
+ },
+ {
+ "modelId": "Llama-3.2-3B-Instruct-q4f16_1-MLC",
+ "variantId": "examples-led",
+ "fixtureId": "numeric-growth-pm",
+ "fixtureKind": "numeric",
+ "inputBulletCount": 5,
+ "outputBulletCount": 6,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Here are the rewritten bullets:",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Spearheaded a 15x growth in weekly active users, driving 1.8M new users to the platform between 2022 and 2024 through 7 onboarding experiments that boosted day-7 retention by 14%.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Spearheaded a 6-person growth squad that executed 12 pricing-page experiments and a 2-tier paywall redesign, generating $4.2M in incremental ARR.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Streamlined checkout processes, reducing conversion friction by 23% and median time-to-purchase from 4.5 minutes to 90 seconds.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Set and achieved 100% of quarterly activation targets and 85% of acquisition targets across 3 OKRs in FY2023, driving business growth and customer acquisition.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 5,
+ "text": "Secured a $250K co-marketing budget with 2 partner brands, delivering 11.5M impressions and 38K signups across a 6-week campaign.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "120K"
+ ],
+ "addedNumbers": []
+ },
+ "rewriteDurationMs": 19272,
+ "error": null
+ },
+ {
+ "modelId": "Llama-3.2-3B-Instruct-q4f16_1-MLC",
+ "variantId": "examples-led",
+ "fixtureId": "redundant-support-lead",
+ "fixtureKind": "redundant",
+ "inputBulletCount": 5,
+ "outputBulletCount": 6,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": false,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Here are the rewritten bullets:",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Streamlined customer support operations, triaging 200+ tickets per week across email and chat channels.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Optimized ticket queue management, prioritizing 200 tickets per week across email and chat, ensuring 95% SLA compliance.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Processed 200+ customer support tickets per week from email and chat sources, achieving a 98% first-response rate.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Resolved 40 escalated billing disputes for enterprise accounts, recovering $85K in disputed charges and reducing customer churn by 25%.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 5,
+ "text": "Designed and executed a 3-week onboarding program for 8 new support representatives, cutting ramp time from 60 days to 28 days and achieving 90% team productivity within 6 weeks.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": [
+ "95%",
+ "98%",
+ "25%",
+ "90%"
+ ]
+ },
+ "rewriteDurationMs": 16390,
+ "error": null
+ }
+ ],
+ "aggregates": [
+ {
+ "modelId": "Llama-3.2-3B-Instruct-q4f16_1-MLC",
+ "variantId": "baseline",
+ "scoredFixtures": 4,
+ "numbersPreservedRate": 0.25,
+ "oneLineRate": 1,
+ "actionVerbRate": 0.25,
+ "lengthSanityRate": 1,
+ "noPreambleLeakRate": 1,
+ "dedupEffectiveRate": 0,
+ "judgeMean": null,
+ "aggregateScore": 0.5833333333333334
+ },
+ {
+ "modelId": "Llama-3.2-3B-Instruct-q4f16_1-MLC",
+ "variantId": "terse",
+ "scoredFixtures": 4,
+ "numbersPreservedRate": 0.25,
+ "oneLineRate": 1,
+ "actionVerbRate": 0,
+ "lengthSanityRate": 1,
+ "noPreambleLeakRate": 1,
+ "dedupEffectiveRate": 0,
+ "judgeMean": null,
+ "aggregateScore": 0.5416666666666666
+ },
+ {
+ "modelId": "Llama-3.2-3B-Instruct-q4f16_1-MLC",
+ "variantId": "examples-led",
+ "scoredFixtures": 4,
+ "numbersPreservedRate": 0,
+ "oneLineRate": 1,
+ "actionVerbRate": 0,
+ "lengthSanityRate": 1,
+ "noPreambleLeakRate": 1,
+ "dedupEffectiveRate": 0,
+ "judgeMean": null,
+ "aggregateScore": 0.5
+ }
+ ]
+}
diff --git a/tests/fixtures/rewrite/reports/eval-rewrite-llama-3-2-3b-instruct-q4f16-1-mlc-2026-06-23T19-09-02-391Z.md b/tests/fixtures/rewrite/reports/eval-rewrite-llama-3-2-3b-instruct-q4f16-1-mlc-2026-06-23T19-09-02-391Z.md
new file mode 100644
index 00000000..5c34e41c
--- /dev/null
+++ b/tests/fixtures/rewrite/reports/eval-rewrite-llama-3-2-3b-instruct-q4f16-1-mlc-2026-06-23T19-09-02-391Z.md
@@ -0,0 +1,48 @@
+# Rewrite eval report
+
+- **Started:** 2026-06-23T19:06:32.946Z
+- **App version:** `8c12fb0`
+- **Models:** 1
+- **Prompt variants:** 3
+- **Fixtures:** 4
+- **LLM judge:** disabled (default)
+
+## Aggregate (per model × variant)
+
+| Model | Variant | Numbers | One-line | Verb | Length | No-preamble | Dedup | Judge | **Aggregate** |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| Llama 3.2 (3B) | Baseline (shipped) | 25% | 100% | 25% | 100% | 100% | 0% | — | **58%** |
+| Llama 3.2 (3B) | Terse (rules-only) | 25% | 100% | 0% | 100% | 100% | 0% | — | **54%** |
+| Llama 3.2 (3B) | Examples-led (few-shot) | 0% | 100% | 0% | 100% | 100% | 0% | — | **50%** |
+
+## Per-cell records
+
+### Llama 3.2 (3B)
+
+#### Baseline (shipped)
+
+| Fixture | Kind | In → Out | Numbers | Verb | Length | Preamble | Dedup | Error |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| weak-marketing-generalist | weak | 5 → 5 | fail | PASS | PASS | PASS | — | |
+| strong-backend-engineer | strong | 5 → 5 | PASS | fail | PASS | PASS | — | |
+| numeric-growth-pm | numeric | 5 → 5 | fail | fail | PASS | PASS | — | |
+| redundant-support-lead | redundant | 5 → 5 | fail | fail | PASS | PASS | fail | |
+
+#### Terse (rules-only)
+
+| Fixture | Kind | In → Out | Numbers | Verb | Length | Preamble | Dedup | Error |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| weak-marketing-generalist | weak | 5 → 6 | fail | fail | PASS | PASS | — | |
+| strong-backend-engineer | strong | 5 → 6 | PASS | fail | PASS | PASS | — | |
+| numeric-growth-pm | numeric | 5 → 6 | fail | fail | PASS | PASS | — | |
+| redundant-support-lead | redundant | 5 → 6 | fail | fail | PASS | PASS | fail | |
+
+#### Examples-led (few-shot)
+
+| Fixture | Kind | In → Out | Numbers | Verb | Length | Preamble | Dedup | Error |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| weak-marketing-generalist | weak | 5 → 6 | fail | fail | PASS | PASS | — | |
+| strong-backend-engineer | strong | 5 → 6 | fail | fail | PASS | PASS | — | |
+| numeric-growth-pm | numeric | 5 → 6 | fail | fail | PASS | PASS | — | |
+| redundant-support-lead | redundant | 5 → 6 | fail | fail | PASS | PASS | fail | |
+
diff --git a/tests/fixtures/rewrite/reports/eval-rewrite-qwen2-5-1-5b-instruct-q4f16-1-mlc-2026-06-23T18-58-27-116Z.json b/tests/fixtures/rewrite/reports/eval-rewrite-qwen2-5-1-5b-instruct-q4f16-1-mlc-2026-06-23T18-58-27-116Z.json
new file mode 100644
index 00000000..c3845046
--- /dev/null
+++ b/tests/fixtures/rewrite/reports/eval-rewrite-qwen2-5-1-5b-instruct-q4f16-1-mlc-2026-06-23T18-58-27-116Z.json
@@ -0,0 +1,784 @@
+{
+ "startedAt": "2026-06-23T18:57:10.380Z",
+ "appVersion": "8c12fb0",
+ "modelIds": [
+ "Qwen2.5-1.5B-Instruct-q4f16_1-MLC"
+ ],
+ "variantIds": [
+ "baseline",
+ "terse",
+ "examples-led"
+ ],
+ "fixtureIds": [
+ "weak-marketing-generalist",
+ "strong-backend-engineer",
+ "numeric-growth-pm",
+ "redundant-support-lead"
+ ],
+ "judgeEnabled": false,
+ "records": [
+ {
+ "modelId": "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
+ "variantId": "baseline",
+ "fixtureId": "weak-marketing-generalist",
+ "fixtureKind": "weak",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": true,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Managed marketing tasks and supported team with day-to-day activities.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Led social media campaigns to increase engagement.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Coordinated content creation, email marketing, and promotional efforts for the brand.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Assisted senior team members in planning and executing marketing events and initiatives.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Participated in weekly meetings to provide input on strategy and creative direction as needed.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": []
+ },
+ "rewriteDurationMs": 3598,
+ "error": null
+ },
+ {
+ "modelId": "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
+ "variantId": "baseline",
+ "fixtureId": "strong-backend-engineer",
+ "fixtureKind": "strong",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": true,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Streamlined order-processing pipeline to Kafka topics, reducing median latency by 38% across 12 services.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Designed horizontally-sharded Postgres layout that handled a 4x traffic surge during launch without failover.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Mentored engineers through code reviews and pair programming, shipping 23 production features over 2 quarters with zero rollback incidents.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Reduced cloud spend $180K annually by rightsizing underused instances and consolidating staging environments.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Built a deterministic load generator for regression checks, used by 4 teams in 90 minutes.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "2024",
+ "5"
+ ],
+ "addedNumbers": []
+ },
+ "rewriteDurationMs": 5698,
+ "error": null
+ },
+ {
+ "modelId": "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
+ "variantId": "baseline",
+ "fixtureId": "numeric-growth-pm",
+ "fixtureKind": "numeric",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": true,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Achieved a 14x increase in weekly active users from 120K to 1.8M between 2022 and 2024 by shipping 7 onboarding experiments, boosting day-7 retention.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Led a growth squad that drove $4.2M incremental annual recurring revenue through pricing-page experiments and a 2-tier paywall redesign.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Reduced free-to-paid conversion friction with an overhaul of the checkout process, lifting conversion 23% and cutting median time-to-purchase from 4.5 minutes to 90 seconds.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Managed quarterly OKRs spanning acquisition, activation, and retention; hit 100% of activation targets and 85% of acquisition targets in FY2023.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Negotiated a $250K co-marketing budget with two partner brands, delivering 11.5M impressions and 38K signups across a 6-week campaign.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "14%"
+ ],
+ "addedNumbers": []
+ },
+ "rewriteDurationMs": 9487,
+ "error": null
+ },
+ {
+ "modelId": "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
+ "variantId": "baseline",
+ "fixtureId": "redundant-support-lead",
+ "fixtureKind": "redundant",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": true,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": false,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Streamlined customer support operations, triaging and resolving 200+ inbound tickets weekly across email and chat channels.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Managed a high-volume ticket queue of up to 200 per week from email and chat sources, prioritizing by SLA and severity.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Handled customer support requests at an average rate of around 200 per week from email and chat sources.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Successfully resolved escalated billing disputes for 40 enterprise accounts, recovering $85K in disputed charges.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Led the onboarding process for new support representatives, reducing ramp time from 60 days to 28 days with a 3-week training program.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": []
+ },
+ "rewriteDurationMs": 6185,
+ "error": null
+ },
+ {
+ "modelId": "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
+ "variantId": "terse",
+ "fixtureId": "weak-marketing-generalist",
+ "fixtureKind": "weak",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": true,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Managed marketing tasks and supported day-to-day activities, handling various campaigns.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Coordinated social media efforts to increase engagement on channels like Facebook and Twitter.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Wrote content for email blasts and other promotional materials.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Assisted in planning and coordinating marketing events and initiatives with senior team members.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Participated in weekly meetings to provide input on strategy and creative direction as appropriate.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": []
+ },
+ "rewriteDurationMs": 3385,
+ "error": null
+ },
+ {
+ "modelId": "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
+ "variantId": "terse",
+ "fixtureId": "strong-backend-engineer",
+ "fixtureKind": "strong",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": true,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Streamlined the order-processing pipeline to Kafka topics, reducing median latency by 38% across 12 downstream services.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Designed a horizontally-sharded Postgres layout that absorbed a 4x traffic surge during a product launch without failover.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Mentored engineers through code reviews and pairing sessions, shipping 23 production features over two quarters with zero rollback incidents.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Reduced cloud spend $180K annually by rightsizing underused instances and consolidating staging environments into a single tenant-isolated one.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Built a deterministic load generator for regression checks used by four teams in 90 minutes after replaying 24 hours of production traffic.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "2024",
+ "5"
+ ],
+ "addedNumbers": []
+ },
+ "rewriteDurationMs": 6038,
+ "error": null
+ },
+ {
+ "modelId": "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
+ "variantId": "terse",
+ "fixtureId": "numeric-growth-pm",
+ "fixtureKind": "numeric",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": true,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Achieved a 7x increase in weekly active users from 120K to 1.8M between 2022 and 2024 by shipping seven onboarding experiments, resulting in an 14% uplift in day-7 retention.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Led a growth squad of six members that generated $4.2 million in incremental annual recurring revenue through pricing-page experiments and a two-tier paywall redesign.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Reduced free-to-paid conversion friction with a checkout overhaul that lifted conversion rates by 23% and decreased median time-to-purchase from 4.5 minutes to 90 seconds.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Managed three quarterly Objectives and Key Results (OKRs) across acquisition, activation, and retention; hit 100% of activation targets and 85% of acquisition targets during FY2023.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Negotiated a $250,000 co-marketing budget with two partner brands, achieving 11.5 million impressions and 38,000 signups over a six-week campaign.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "$4.2M",
+ "$250K",
+ "11.5M",
+ "38K"
+ ],
+ "addedNumbers": [
+ "$4.2",
+ "$250,000",
+ "11.5",
+ "38,000"
+ ]
+ },
+ "rewriteDurationMs": 10013,
+ "error": null
+ },
+ {
+ "modelId": "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
+ "variantId": "terse",
+ "fixtureId": "redundant-support-lead",
+ "fixtureKind": "redundant",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": true,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": false,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Streamlined customer support operations, triaging and resolving 200+ inbound tickets weekly across email and chat channels.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Managed a high-volume ticket queue of up to 200 per week from email and chat sources, prioritizing by SLA and severity.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Handled customer support requests at an average rate of around 200 per week from email and chat sources.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Successfully resolved escalated billing disputes for 40 enterprise accounts, recovering $85K in disputed charges.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Led the onboarding process for new support representatives, reducing ramp time from 60 days to 28 days with a 3-week training program.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": []
+ },
+ "rewriteDurationMs": 5754,
+ "error": null
+ },
+ {
+ "modelId": "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
+ "variantId": "examples-led",
+ "fixtureId": "weak-marketing-generalist",
+ "fixtureKind": "weak",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Spearheaded marketing efforts, enhancing engagement through 40% increase in social media reach.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Crafted and launched successful campaigns on multiple platforms to boost brand awareness by 35%.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Coordinated content creation for email newsletters and promotional materials, driving open rates up to 60%.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Collaborated with senior team members to plan and execute impactful events, resulting in a 20% uplift in event attendance.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Regularly attended meetings to align strategy with creative direction, contributing valuable input throughout the quarter.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": [
+ "40%",
+ "35%",
+ "60%",
+ "20%"
+ ]
+ },
+ "rewriteDurationMs": 4820,
+ "error": null
+ },
+ {
+ "modelId": "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
+ "variantId": "examples-led",
+ "fixtureId": "strong-backend-engineer",
+ "fixtureKind": "strong",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": false,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Led the migration of the order-processing pipeline to event-driven Kafka, reducing median latency by 38% across 12 downstream services.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Designed a horizontally-sharded Postgres layout that absorbed a 4x traffic surge during a product launch without failover.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Mentored engineers through code reviews and pairing sessions, shipping 23 production features over two quarters with zero rollback incidents.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Rightsized cloud infrastructure, saving $180K annually by consolidating underused instances and migrating staging environments.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Built a deterministic load generator for regression checks used by four teams.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [
+ "2024",
+ "5"
+ ],
+ "addedNumbers": []
+ },
+ "rewriteDurationMs": 5653,
+ "error": null
+ },
+ {
+ "modelId": "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
+ "variantId": "examples-led",
+ "fixtureId": "numeric-growth-pm",
+ "fixtureKind": "numeric",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": true,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": null,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Oversaw a 10x growth in weekly active users from 120K to 1.8M between 2022 and 2024, implementing 7 onboarding experiments that increased day-7 retention by 14%.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Led the team responsible for driving $4.2M incremental Annual Recurring Revenue (ARR) through pricing-page experiments and a 2-tier paywall redesign.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Reduced free-to-paid conversion friction with an overhaul of the checkout process, lifting conversion rates by 23% while cutting median time-to-purchase from 4.5 minutes to 90 seconds.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Managed quarterly OKRs across acquisition, activation, and retention; hit 100% of activation targets and 85% of acquisition targets during FY2023.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Negotiated a $250K co-marketing budget with two partner brands, achieving 11.5M impressions and 38K signups over a 6-week campaign.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": []
+ },
+ "rewriteDurationMs": 10343,
+ "error": null
+ },
+ {
+ "modelId": "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
+ "variantId": "examples-led",
+ "fixtureId": "redundant-support-lead",
+ "fixtureKind": "redundant",
+ "inputBulletCount": 5,
+ "outputBulletCount": 5,
+ "rubric": {
+ "numbersPreserved": true,
+ "oneLinePerBullet": true,
+ "actionVerbLead": false,
+ "lengthSanity": true,
+ "noPreambleLeak": true,
+ "dedupEffective": false,
+ "judgeCoherence": null,
+ "perBullet": [
+ {
+ "index": 0,
+ "text": "Streamlined customer support response time, triaging 200+ tickets per week across email and chat channels.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 1,
+ "text": "Managed a high-volume ticket queue with a 200/week rate, prioritizing by SLA and severity.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 2,
+ "text": "Handled customer support requests at an average of 200 per week from email and chat sources.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 3,
+ "text": "Resolved escalated billing disputes for 40 enterprise accounts, recovering $85K in disputed charges.",
+ "startsWithActionVerb": true,
+ "lengthOk": true,
+ "oneLine": true
+ },
+ {
+ "index": 4,
+ "text": "Successfully onboarded 8 new support representatives within a 3-week training program, reducing ramp time to 28 days.",
+ "startsWithActionVerb": false,
+ "lengthOk": true,
+ "oneLine": true
+ }
+ ],
+ "droppedNumbers": [],
+ "addedNumbers": []
+ },
+ "rewriteDurationMs": 5758,
+ "error": null
+ }
+ ],
+ "aggregates": [
+ {
+ "modelId": "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
+ "variantId": "baseline",
+ "scoredFixtures": 4,
+ "numbersPreservedRate": 0.5,
+ "oneLineRate": 1,
+ "actionVerbRate": 0.5,
+ "lengthSanityRate": 1,
+ "noPreambleLeakRate": 1,
+ "dedupEffectiveRate": 0,
+ "judgeMean": null,
+ "aggregateScore": 0.6666666666666666
+ },
+ {
+ "modelId": "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
+ "variantId": "terse",
+ "scoredFixtures": 4,
+ "numbersPreservedRate": 0.5,
+ "oneLineRate": 1,
+ "actionVerbRate": 0.5,
+ "lengthSanityRate": 1,
+ "noPreambleLeakRate": 1,
+ "dedupEffectiveRate": 0,
+ "judgeMean": null,
+ "aggregateScore": 0.6666666666666666
+ },
+ {
+ "modelId": "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
+ "variantId": "examples-led",
+ "scoredFixtures": 4,
+ "numbersPreservedRate": 0.5,
+ "oneLineRate": 1,
+ "actionVerbRate": 0,
+ "lengthSanityRate": 1,
+ "noPreambleLeakRate": 1,
+ "dedupEffectiveRate": 0,
+ "judgeMean": null,
+ "aggregateScore": 0.5833333333333334
+ }
+ ]
+}
diff --git a/tests/fixtures/rewrite/reports/eval-rewrite-qwen2-5-1-5b-instruct-q4f16-1-mlc-2026-06-23T18-58-27-116Z.md b/tests/fixtures/rewrite/reports/eval-rewrite-qwen2-5-1-5b-instruct-q4f16-1-mlc-2026-06-23T18-58-27-116Z.md
new file mode 100644
index 00000000..20ea760b
--- /dev/null
+++ b/tests/fixtures/rewrite/reports/eval-rewrite-qwen2-5-1-5b-instruct-q4f16-1-mlc-2026-06-23T18-58-27-116Z.md
@@ -0,0 +1,48 @@
+# Rewrite eval report
+
+- **Started:** 2026-06-23T18:57:10.380Z
+- **App version:** `8c12fb0`
+- **Models:** 1
+- **Prompt variants:** 3
+- **Fixtures:** 4
+- **LLM judge:** disabled (default)
+
+## Aggregate (per model × variant)
+
+| Model | Variant | Numbers | One-line | Verb | Length | No-preamble | Dedup | Judge | **Aggregate** |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| Qwen 2.5 (1.5B) | Baseline (shipped) | 50% | 100% | 50% | 100% | 100% | 0% | — | **67%** |
+| Qwen 2.5 (1.5B) | Terse (rules-only) | 50% | 100% | 50% | 100% | 100% | 0% | — | **67%** |
+| Qwen 2.5 (1.5B) | Examples-led (few-shot) | 50% | 100% | 0% | 100% | 100% | 0% | — | **58%** |
+
+## Per-cell records
+
+### Qwen 2.5 (1.5B)
+
+#### Baseline (shipped)
+
+| Fixture | Kind | In → Out | Numbers | Verb | Length | Preamble | Dedup | Error |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| weak-marketing-generalist | weak | 5 → 5 | PASS | fail | PASS | PASS | — | |
+| strong-backend-engineer | strong | 5 → 5 | fail | PASS | PASS | PASS | — | |
+| numeric-growth-pm | numeric | 5 → 5 | fail | PASS | PASS | PASS | — | |
+| redundant-support-lead | redundant | 5 → 5 | PASS | fail | PASS | PASS | fail | |
+
+#### Terse (rules-only)
+
+| Fixture | Kind | In → Out | Numbers | Verb | Length | Preamble | Dedup | Error |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| weak-marketing-generalist | weak | 5 → 5 | PASS | fail | PASS | PASS | — | |
+| strong-backend-engineer | strong | 5 → 5 | fail | PASS | PASS | PASS | — | |
+| numeric-growth-pm | numeric | 5 → 5 | fail | PASS | PASS | PASS | — | |
+| redundant-support-lead | redundant | 5 → 5 | PASS | fail | PASS | PASS | fail | |
+
+#### Examples-led (few-shot)
+
+| Fixture | Kind | In → Out | Numbers | Verb | Length | Preamble | Dedup | Error |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| weak-marketing-generalist | weak | 5 → 5 | fail | fail | PASS | PASS | — | |
+| strong-backend-engineer | strong | 5 → 5 | fail | fail | PASS | PASS | — | |
+| numeric-growth-pm | numeric | 5 → 5 | PASS | fail | PASS | PASS | — | |
+| redundant-support-lead | redundant | 5 → 5 | PASS | fail | PASS | PASS | fail | |
+
diff --git a/tests/fixtures/rewrite/strong.json b/tests/fixtures/rewrite/strong.json
new file mode 100644
index 00000000..90e990ee
--- /dev/null
+++ b/tests/fixtures/rewrite/strong.json
@@ -0,0 +1,12 @@
+{
+ "id": "strong-backend-engineer",
+ "kind": "strong",
+ "description": "Already-strong backend-engineer bullets with action verbs, concrete metrics, and tight phrasing. A good rewrite should leave these largely intact and must not silently drop any number.",
+ "bullets": [
+ "Led migration of the order-processing pipeline to event-driven Kafka topics, cutting median latency by 38% across 12 downstream services.",
+ "Designed a horizontally-sharded Postgres layout that absorbed a 4x traffic surge during a 2024 product launch without read-replica failover.",
+ "Mentored 5 engineers through code-review and pairing sessions, shipping 23 production features over 2 quarters with zero rollback incidents.",
+ "Reduced cloud spend $180K annualized by rightsizing 60 underused instances and consolidating 3 staging environments into a single tenant-isolated one.",
+ "Built a deterministic load generator that replays 24 hours of production traffic in 90 minutes, used by 4 teams for regression checks."
+ ]
+}
diff --git a/tests/fixtures/rewrite/weak.json b/tests/fixtures/rewrite/weak.json
new file mode 100644
index 00000000..3281e594
--- /dev/null
+++ b/tests/fixtures/rewrite/weak.json
@@ -0,0 +1,12 @@
+{
+ "id": "weak-marketing-generalist",
+ "kind": "weak",
+ "description": "Vague marketing-generalist bullets with no metrics, weak verbs, and filler language. A good rewrite should add structure and verb strength even though it can't invent numbers.",
+ "bullets": [
+ "Responsible for handling various marketing tasks and supporting the team with day-to-day activities as needed.",
+ "Worked on campaigns to drive engagement on social media and other channels.",
+ "Helped with content writing, email blasts, and other promotional activities for the brand.",
+ "Assisted senior team members with planning and coordination of marketing events and initiatives.",
+ "Participated in weekly meetings and provided input on strategy and creative direction as appropriate."
+ ]
+}