Add focused changeset scanning benchmark

.github/workflows/quick-performance.yml

@@ -0,0 +1,102 @@
+name: Quick performance
+
+on:
+  workflow_dispatch:
+    inputs:
+      scenario:
+        description: Quick performance scenario
+        required: true
+        default: incremental-synchronization
+        type: choice
+        options:
+          - incremental-synchronization
+          - changeset-scanning
+      fixture:
+        description: Fixture id (blank uses the scenario's default)
+        required: false
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  quick-performance:
+    name: ${{ inputs.scenario || 'incremental-synchronization' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      NODE_OPTIONS: --disable-warning=MODULE_TYPELESS_PACKAGE_JSON
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          persist-credentials: false
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093
+
+      - name: Use Node.js 24
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e
+        with:
+          node-version: 24.18.0
+          cache: pnpm
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Build transformer
+        run: pnpm --dir packages/transformer build:cjs
+
+      - name: Validate quick harness
+        run: |
+          pnpm --dir packages/performance-tests test:quick-unit \
+            2>&1 | sed \
+              -e "s#${GITHUB_WORKSPACE}#<workspace>#g" \
+              -e "s#${RUNNER_TEMP}#<runner-temp>#g"
+
+      - name: Run quick performance suite
+        run: |
+          pnpm --dir packages/performance-tests test:quick \
+            2>&1 | sed \
+              -e "s#${GITHUB_WORKSPACE}#<workspace>#g" \
+              -e "s#${RUNNER_TEMP}#<runner-temp>#g"
+        env:
+          QUICK_PERF_OUTPUT: ${{ runner.temp }}/quick-performance
+          QUICK_PERF_SCENARIO: ${{ inputs.scenario || 'incremental-synchronization' }}
+          QUICK_PERF_FIXTURE: ${{ inputs.fixture }}
+
+      - name: Report variance reliability
+        if: always()
+        shell: bash
+        run: |
+          node <<'NODE'
+          const fs = require("fs");
+          const path = require("path");
+          const report = path.join(process.env.RUNNER_TEMP, "quick-performance", "summary.json");
+          if (!fs.existsSync(report))
+            process.exit(0);
+          const summary = JSON.parse(fs.readFileSync(report, "utf8"));
+          fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, [
+            "### Quick performance reliability",
+            `Scenario: **${summary.scenarioId}**`,
+            `Status: **${summary.varianceStatus}**`,
+            `Transformer: ${summary.versions?.transformer} / core-backend: ${summary.versions?.coreBackend} / node: ${summary.versions?.node}`,
+            `CV: ${summary.wallMilliseconds.coefficientOfVariation}`,
+            `Normalized MAD: ${summary.wallMilliseconds.normalizedMad}`,
+            `Unstable samples: ${summary.unstableSamples.join(", ")}`,
+            "",
+          ].join("\n"));
+          if (summary.varianceStatus !== "stable")
+            console.log("::warning title=Unstable quick performance variance::Do not use this run as regression evidence.");
+          NODE
+
+      - name: Publish report
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
+        with:
+          name: QuickPerformanceReport
+          path: |
+            ${{ runner.temp }}/quick-performance/samples.jsonl
+            ${{ runner.temp }}/quick-performance/summary.json
+            ${{ runner.temp }}/quick-performance/summary.csv

.vscode/launch.json

@@ -11,8 +11,7 @@
         "runtimeExecutable": "npm",
         "cwd": "${workspaceFolder}/packages/performance-tests/",
         "runtimeArgs": [
-          "run",
-          "test-mocha"
+          "test"
         ],
         "skipFiles": [
           "<node_internals>/**"
@@ -31,17 +30,6 @@
         "skipFiles": [
           "<node_internals>/**"
         ]
-      },
-      {
-        "type": "node",
-        "request": "launch",
-        "name": "Performance tests",
-        "runtimeExecutable": "npm",
-        "cwd": "${workspaceFolder}/packages/performance-tests/test",
-        "runtimeArgs": [
-          "run",
-          "test"
-        ],
       }
     ]
-}
+}

packages/performance-tests/.gitignore

@@ -1,2 +1,5 @@
 .env
 **/.output
+.quick-cache/
+.quick-output/
+**/*.bim

packages/performance-tests/ARCHITECTURE.md

@@ -0,0 +1,118 @@
+# Performance Test Architecture
+
+This package contains the transformer's performance regression tests and the
+unit tests for their supporting infrastructure. Both use Vitest, but they have
+different runtime requirements.
+
+## Test categories
+
+| Category | Location | Purpose | External requirements |
+| --- | --- | --- | --- |
+| Weekly regression | `test/TransformerRegression.test.ts` | Measure transformer implementations against selected Hub iModels and a generated local iModel | Hub credentials, OIDC configuration, and network access |
+| Infrastructure unit | `test/unit/**/*.test.ts` | Verify registration and cleanup behavior | None |
+
+The unit tests do not measure transformer performance. They validate code used
+to construct and tear down the weekly regression suite.
+
+## Vitest configuration
+
+`vitest.config.ts` includes both categories and uses one forked worker with file
+parallelism disabled. The weekly tests share process-wide resources such as
+`IModelHost`, authentication, downloaded briefcases, and the CSV reporter, so
+they must run serially.
+
+The weekly regression file opts out of test and hook timeouts because individual
+transformations and downloads can run for hours. Infrastructure unit tests keep
+Vitest's default timeouts so a broken unit test fails instead of hanging the
+worker.
+
+Environment loading is scoped to `TransformerRegression.test.ts`. Unit tests can
+therefore run locally without a `.env` file by running
+`pnpm exec vitest run test/unit`.
+
+The repository's root `pnpm test` excludes this package. Run its tests explicitly
+from `packages/performance-tests`.
+
+## Weekly regression lifecycle
+
+The weekly suite has two phases.
+
+### Collection
+
+Before running any tests, Vitest imports `TransformerRegression.test.ts` to
+discover them. During this setup step, the module:
+
+1. Load environment configuration and authenticate.
+2. Start `IModelHost` with Hub access.
+3. Discover and filter the configured Hub iModels.
+4. Add the generated local iModel.
+5. Load the built-in and optional comparison transformer modules.
+6. Build the supported test-case/module combinations.
+
+If collection fails after authentication or host startup begins, the worker
+lifecycle attempts to shut down every initialized resource before preserving
+and rethrowing the original error.
+
+### Execution
+
+For each selected iModel, the suite:
+
+1. Download or generate a local `.bim` source database and record its report
+   metadata.
+2. Opens a fresh read-only source database for each test.
+3. Runs every supported test-case/module combination.
+4. Closes the source database after each test.
+
+The raw-insert comparison runs after the per-iModel transform cases. Once all
+tests finish, the suite exports `test/.output/report.csv`, shuts down
+`IModelHost`, and signs out. Cleanup tasks are all attempted even if an earlier
+task fails.
+
+## Test registration
+
+`RegressionTestRegistration.ts` creates the execution matrix; it does not run
+tests or store results.
+
+Each test case names the factory function it requires from
+`TestTransformerModule`. Each loaded transformer module is paired only with the
+test cases it supports. `TransformerRegression.test.ts` consumes those
+definitions and registers the corresponding Vitest tests.
+
+The module name is the human-readable identifier used in test names and report
+entries. The module object contains the implementation that the test executes.
+Additional implementations can be loaded through `EXTRA_TRANSFORMERS`.
+
+## Inputs and authentication
+
+`template.env` documents the weekly suite's environment variables. The important
+input controls are:
+
+- `ITWIN_IDS`: iTwins from which test iModels are discovered.
+- `IMODEL_IDS`: specific iModels to include, or `*` for every iModel in the
+  configured iTwins.
+- `EXTRA_TRANSFORMERS`: optional module paths for comparison implementations.
+- `LOG_LEVEL`: verbosity for the iTwin logger.
+
+CI uses headless authentication. Local weekly runs use the CLI authorization
+client and still require the OIDC and Hub configuration described in
+`template.env`. Never commit a populated `.env` file.
+
+## Reporting
+
+Test cases report measurements through the callback in `TestCaseContext`.
+`@itwin/perf-tools` combines those measurements with iModel and branch metadata,
+then writes `test/.output/report.csv`.
+
+That path and CSV format are the artifact contract consumed by the hosted weekly
+performance pipeline. Changes to either require coordinated pipeline validation.
+
+## Extending the package
+
+- Put credential-free tests of support code under `test/unit`.
+- Add a weekly performance case under `test/cases`, declare its required
+  transformer factory in `TestTransformerModule`, and add it to `testCasesMap`.
+- Add a built-in transformer implementation under `test/transformers`, or load a
+  comparison implementation with `EXTRA_TRANSFORMERS`.
+- Give future performance-test categories their own entry file, setup, timeout,
+  and reporting policy. Do not make Hub credentials or unlimited timeouts global
+  merely because the weekly suite needs them.

packages/performance-tests/README.md

@@ -1,22 +1,92 @@
-# Presentation Performance Tests
+# Transformer Performance Tests
 
 A package containing performance tests for the [`@itwin/imodel-transformer` library](../../README.md).
 
+See [ARCHITECTURE.md](./ARCHITECTURE.md) for the test categories, weekly suite
+lifecycle, registration model, and extension guidance.
+
 ## Tests
 
+### Quick incremental performance
+
+The quick suite is independent of the cloud-backed weekly regression suite. It
+reconstructs a fresh local HubMock from the versioned
+`balanced-incremental` recipe for every sample, establishes target provenance,
+pushes eight real source changesets, and then times only
+`IModelTransformer.process()` with `argsForProcessChanges`.
+
+Reconstruction, verification, and reporting are outside benchmark timing but
+are reported and count against the 15-minute end-to-end budget. The suite does
+not use iModelHub credentials or download QA iModels.
+
+Build the transformer package before running the TypeScript suite:
+
+```sh
+pnpm --dir ../transformer build:cjs
+pnpm test:quick-unit
+pnpm test:quick
+```
+
+The default scenario is `incremental-synchronization`, selected with
+`QUICK_PERF_SCENARIO`. Unknown scenario names fail before fixture
+reconstruction. The default run is sample 0 as a warm-up plus eight measured
+samples. Each sample is a fresh reconstruction, one timed scenario execution,
+and untimed verification and cleanup. The warm-up follows the same lifecycle
+but is excluded from summary timing statistics. Set `QUICK_PERF_SAMPLES` only
+for local diagnostics. Reports are written under `test/quick/.quick-output/`
+unless `QUICK_PERF_OUTPUT` is set and include `samples.jsonl`, `summary.json`,
+and `summary.csv`. Every sample and report includes the scenario ID, and the
+reporter rejects mixed-scenario sample sets.
+
+The calibrated fixture contains 6,000 base elements, 12,000 aspects, 3,000
+relationships, and 3,000 geometry-bearing elements. Its eight changesets apply
+600 element inserts/updates/deletes, 600 aspect inserts/updates, 1,200 aspect
+deletes, 300 relationship inserts/updates, 825 relationship deletes, and 150
+geometry updates. This is 25 deterministic repetitions of one balanced content
+unit, preserving the original scenario ratios.
+
+`varianceStatus` requires coefficient of variation (standard deviation divided
+by mean) and normalized MAD at or below 5%. CV can exceed the target when host
+scheduling or background load delays one sample. An unstable manual run emits a
+workflow warning and must not be used as regression evidence, but correctness
+and the 15-minute budget remain hard gates. Variance does not currently fail the
+manual workflow: six local calibration suites informed the final scale, and
+three ratio-correct final suites on a shared workstation produced 1.38-1.48
+second medians but only two met the CV threshold. A hard gate would therefore
+have false-failed one of three final runs. The GitHub workflow now targets
+`ubuntu-latest`, but no Linux reliability evidence exists until that workflow
+runs. Revisit the failure policy after repeated measurements on the hosted
+Linux runner or a dedicated performance agent.
+
+The GitHub Actions workflow is manual-only. Select the branch with GitHub's
+native workflow ref and the scenario with its dispatch input:
+
+```sh
+gh workflow run quick-performance.yml --ref <branch> \
+  -f scenario=incremental-synchronization
+```
+
+GitHub can dispatch this workflow only after `quick-performance.yml` exists on
+the repository's default branch. The `--ref` selects which committed branch
+revision runs after that requirement is met; there is intentionally no custom
+branch input and no automatic pull-request or push trigger.
+
+`pnpm quick:build-fixture` writes the canonical recipe manifest.
+`pnpm quick:verify-fixture` performs two fresh reconstructions (warm-up plus one
+measured sample), checks their semantic digests, and writes a diagnostic report.
+
 Here are tests we need but don't have:
 
-- *Identity Transform*
+- _Identity Transform_
   transform the entire contents of the iModel to an empty iModel seed
-- *JSON Geometry Editing Transform*
+- _JSON Geometry Editing Transform_
   transform the iModel, editing geometry as we go using the json format
-- *Binary Geometry Editing Transform*
+- _Binary Geometry Editing Transform_
   transform the iModel, editing geometry as we go using elementGeometryBuilderParams
-- *Optimistically Locking Remote Target*
-- *Pessimistically Locking Remote Target*
-- *Processing Changes*
-- *More Branching Stuff*
-
+- _Optimistically Locking Remote Target_
+- _Pessimistically Locking Remote Target_
+- _Processing Changes_
+- _More Branching Stuff_
 
 ## Usage
 
@@ -30,16 +100,11 @@ Here are tests we need but don't have:
 
 3. Create `.env` file using `template.env` template.
 
-5. Run:
+4. Run the serialized Vitest suite:
 
    ```sh
    pnpm test
    ```
 
-<!-- FIXME: output csv -->
-6. Review results like:
-
-```sh
-pnpm exec process-results < report.jsonl
-```
-
+5. Review `test/.output/report.csv`. This path is also the artifact contract used by
+   the weekly Azure pipeline.