Add isolated quick A/A and A/B performance workflow

packages/performance-tests/README.md

@@ -18,7 +18,7 @@ The quick infrastructure tests are separate from both performance suites:
 
 See [ARCHITECTURE.md](./ARCHITECTURE.md) for the quick suite's component model,
 and [COMPARISON_STATISTICS.md](./COMPARISON_STATISTICS.md) for the calibrated
-paired-comparison estimator, evidence levels, and execution identity contract.
+paired-comparison estimator, evidence levels, and execution identity contract,
 provider lifecycles, execution diagram, timing boundaries, and report format.
 
 ## Quick performance terminology
@@ -142,6 +142,72 @@ gh workflow run quick-performance.yml --ref <branch> \
 GitHub can dispatch the workflow only after the workflow file exists on the
 repository's default branch. The caller must have repository write access.
 
+### Quick A/A calibration and A/B comparison
+
+`quick-performance-comparison.yml` is a separate manual-only workflow for
+`changeset-scanning`. It does not use the weekly Hub-backed regression suite and
+never creates a merge-blocking performance gate. Each independent job produces
+exactly one `PairObservation`: the declared `AB` or `BA` order runs one isolated
+process per arm, each process performs one excluded warm-up plus three measured
+scenario executions, and each arm's three measurements are median-collapsed
+before computing one log-ratio. This is eight scenario executions per job; the
+within-process samples are never treated as three independent pairs.
+
+Calibration mode builds one selected `calibration_ref` for both labeled arms and
+runs three independent jobs in `AB`, `BA`, `AB` order. The aggregation job
+rejects fixture, recipe, workload, environment, execution-policy, build, or
+fingerprint mismatches. Three matching observations from three jobs establish
+the initial pool. An absolute A/A band at or below 5% is target quality, between
+5% and 10% is marginal, and 10% or greater is unresolvable. All three remain
+informational; the practical effect and equivalence threshold is 10%.
+
+```sh
+gh workflow run quick-performance-comparison.yml --ref main \
+  -f mode=calibrate-a-a \
+  -f scenario=changeset-scanning \
+  -f calibration_ref=main
+```
+
+Comparison mode checks out and builds baseline and candidate separately. The
+top-layer harness loads `ChangedInstanceIds` from each explicit built package in
+its own child process, so the baseline ref does not need to contain the
+comparison harness. Supply the prior calibration workflow run ID and artifact;
+leave `candidate_ref` blank to use the workflow ref selected by `--ref`.
+
+```sh
+gh workflow run quick-performance-comparison.yml --ref main \
+  -f mode=compare-a-b \
+  -f scenario=changeset-scanning \
+  -f baseline_ref=main \
+  -f candidate_ref=my-candidate \
+  -f pair_order=AB \
+  -f calibration_run_id=123456789 \
+  -f calibration_artifact=QuickPerformanceCalibration \
+  -f calibration_file=calibration.json
+```
+
+The workflow publishes raw arm samples, the pair observation (including discard
+reasons), the calibration pool and band, summary JSON, and Markdown used for the
+job summary. A/B rejects a missing calibration or any calibration fingerprint
+or hosted-runner class mismatch instead of borrowing a nearby result.
+
+The comparison commands run from explicit native ESM output under
+`test/quick/runtime/.compiled`, produced by `pnpm build:quick-cli`; unit coverage
+runs under the package's serialized Vitest configuration with
+`pnpm test:comparison`. Neither the harness nor an arm checkout requires Mocha,
+Chai, or ts-node.
+
+GitHub can dispatch this workflow only after
+`quick-performance-comparison.yml` reaches the repository's default branch.
+Before then, local smoke runs can exercise process orchestration but cannot
+create hosted calibration evidence. The CLI's `prepare-fixture --smoke true`
+uses an explicitly labeled reduced fixture; its different identity prevents it
+from being accepted as calibration for the full fixture.
+
+`pnpm quick:build-fixture` writes the canonical recipe manifest.
+`pnpm quick:verify-fixture` performs two fresh reconstructions (warm-up plus one
+measured sample), checks their semantic digests, and writes a diagnostic report.
+
 ## Adding quick performance coverage
 
 ### Add a scenario

packages/performance-tests/package.json

@@ -11,11 +11,13 @@
     "test": "vitest run",
     "format": "prettier \"./test/**/*.ts\" --write",
     "test:quick-unit": "vitest run --config vitest.quick.config.ts tests/unit",
+    "test:comparison": "vitest run --config vitest.quick.config.ts tests/unit/comparison.test.ts tests/unit/ComparisonRunner.test.ts",
     "test:quick-integration": "vitest run --config vitest.quick.config.ts tests/integration",
     "test:quick-harness": "vitest run --config vitest.quick.config.ts tests",
     "test:quick": "vitest run --config vitest.quick.config.ts QuickPerformance.test.ts",
     "quick:build-fixture": "pnpm build:quick-cli && node test/quick/runtime/.compiled/quick/src/cli/fixtureCli.js build-fixture",
     "quick:verify-fixture": "pnpm build:quick-cli && node test/quick/runtime/.compiled/quick/src/cli/fixtureCli.js verify-fixture",
+    "quick:compare": "pnpm build:quick-cli && node test/quick/runtime/.compiled/quick/src/cli/comparisonCli.js",
     "process-reports": "node scripts/process-reports"
   },
   "repository": {},

packages/performance-tests/test/quick/src/cli/armProcessCli.ts

@@ -0,0 +1,43 @@
+/*---------------------------------------------------------------------------------------------
+ * Copyright (c) Bentley Systems, Incorporated. All rights reserved.
+ * See LICENSE.md in the project root for license terms and full copyright notice.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+import {
+  aliasHarnessCoreBackendToArm,
+  ArmSpec,
+  resolveArmSpec,
+} from "../comparison/ArmModule.js";
+
+interface ArmRequestFile {
+  readonly arm: ArmSpec;
+}
+
+function argument(name: string): string {
+  const index = process.argv.indexOf(`--${name}`);
+  const value = index < 0 ? undefined : process.argv[index + 1];
+  if (!value) throw new Error(`Missing required argument --${name}`);
+  return value;
+}
+
+async function main(): Promise<void> {
+  const requestFile = argument("request");
+  const outputFile = argument("output");
+  const request = JSON.parse(
+    fs.readFileSync(requestFile, "utf8")
+  ) as ArmRequestFile;
+  aliasHarnessCoreBackendToArm(resolveArmSpec(request.arm));
+  const { runArm } = await import("../comparison/ComparisonRunner.js");
+  const result = await runArm(request as Parameters<typeof runArm>[0]);
+  fs.mkdirSync(path.dirname(outputFile), { recursive: true });
+  fs.writeFileSync(outputFile, `${JSON.stringify(result, undefined, 2)}\n`);
+}
+
+void main().catch((error) => {
+  process.stderr.write(
+    `${error instanceof Error ? (error.stack ?? error.message) : String(error)}\n`
+  );
+  process.exitCode = 1;
+});