Migrate weekly performance tests to Vitest

.vscode/launch.json

@@ -11,8 +11,7 @@
         "runtimeExecutable": "npm",
         "cwd": "${workspaceFolder}/packages/performance-tests/",
         "runtimeArgs": [
-          "run",
-          "test-mocha"
+          "test"
         ],
         "skipFiles": [
           "<node_internals>/**"
@@ -31,17 +30,6 @@
         "skipFiles": [
           "<node_internals>/**"
         ]
-      },
-      {
-        "type": "node",
-        "request": "launch",
-        "name": "Performance tests",
-        "runtimeExecutable": "npm",
-        "cwd": "${workspaceFolder}/packages/performance-tests/test",
-        "runtimeArgs": [
-          "run",
-          "test"
-        ],
       }
     ]
-}
+}

packages/performance-tests/ARCHITECTURE.md

@@ -0,0 +1,118 @@
+# Performance Test Architecture
+
+This package contains the transformer's performance regression tests and the
+unit tests for their supporting infrastructure. Both use Vitest, but they have
+different runtime requirements.
+
+## Test categories
+
+| Category | Location | Purpose | External requirements |
+| --- | --- | --- | --- |
+| Weekly regression | `test/TransformerRegression.test.ts` | Measure transformer implementations against selected Hub iModels and a generated local iModel | Hub credentials, OIDC configuration, and network access |
+| Infrastructure unit | `test/unit/**/*.test.ts` | Verify registration and cleanup behavior | None |
+
+The unit tests do not measure transformer performance. They validate code used
+to construct and tear down the weekly regression suite.
+
+## Vitest configuration
+
+`vitest.config.ts` includes both categories and uses one forked worker with file
+parallelism disabled. The weekly tests share process-wide resources such as
+`IModelHost`, authentication, downloaded briefcases, and the CSV reporter, so
+they must run serially.
+
+The weekly regression file opts out of test and hook timeouts because individual
+transformations and downloads can run for hours. Infrastructure unit tests keep
+Vitest's default timeouts so a broken unit test fails instead of hanging the
+worker.
+
+Environment loading is scoped to `TransformerRegression.test.ts`. Unit tests can
+therefore run locally without a `.env` file by running
+`pnpm exec vitest run test/unit`.
+
+The repository's root `pnpm test` excludes this package. Run its tests explicitly
+from `packages/performance-tests`.
+
+## Weekly regression lifecycle
+
+The weekly suite has two phases.
+
+### Collection
+
+Before running any tests, Vitest imports `TransformerRegression.test.ts` to
+discover them. During this setup step, the module:
+
+1. Load environment configuration and authenticate.
+2. Start `IModelHost` with Hub access.
+3. Discover and filter the configured Hub iModels.
+4. Add the generated local iModel.
+5. Load the built-in and optional comparison transformer modules.
+6. Build the supported test-case/module combinations.
+
+If collection fails after authentication or host startup begins, the worker
+lifecycle attempts to shut down every initialized resource before preserving
+and rethrowing the original error.
+
+### Execution
+
+For each selected iModel, the suite:
+
+1. Download or generate a local `.bim` source database and record its report
+   metadata.
+2. Opens a fresh read-only source database for each test.
+3. Runs every supported test-case/module combination.
+4. Closes the source database after each test.
+
+The raw-insert comparison runs after the per-iModel transform cases. Once all
+tests finish, the suite exports `test/.output/report.csv`, shuts down
+`IModelHost`, and signs out. Cleanup tasks are all attempted even if an earlier
+task fails.
+
+## Test registration
+
+`RegressionTestRegistration.ts` creates the execution matrix; it does not run
+tests or store results.
+
+Each test case names the factory function it requires from
+`TestTransformerModule`. Each loaded transformer module is paired only with the
+test cases it supports. `TransformerRegression.test.ts` consumes those
+definitions and registers the corresponding Vitest tests.
+
+The module name is the human-readable identifier used in test names and report
+entries. The module object contains the implementation that the test executes.
+Additional implementations can be loaded through `EXTRA_TRANSFORMERS`.
+
+## Inputs and authentication
+
+`template.env` documents the weekly suite's environment variables. The important
+input controls are:
+
+- `ITWIN_IDS`: iTwins from which test iModels are discovered.
+- `IMODEL_IDS`: specific iModels to include, or `*` for every iModel in the
+  configured iTwins.
+- `EXTRA_TRANSFORMERS`: optional module paths for comparison implementations.
+- `LOG_LEVEL`: verbosity for the iTwin logger.
+
+CI uses headless authentication. Local weekly runs use the CLI authorization
+client and still require the OIDC and Hub configuration described in
+`template.env`. Never commit a populated `.env` file.
+
+## Reporting
+
+Test cases report measurements through the callback in `TestCaseContext`.
+`@itwin/perf-tools` combines those measurements with iModel and branch metadata,
+then writes `test/.output/report.csv`.
+
+That path and CSV format are the artifact contract consumed by the hosted weekly
+performance pipeline. Changes to either require coordinated pipeline validation.
+
+## Extending the package
+
+- Put credential-free tests of support code under `test/unit`.
+- Add a weekly performance case under `test/cases`, declare its required
+  transformer factory in `TestTransformerModule`, and add it to `testCasesMap`.
+- Add a built-in transformer implementation under `test/transformers`, or load a
+  comparison implementation with `EXTRA_TRANSFORMERS`.
+- Give future performance-test categories their own entry file, setup, timeout,
+  and reporting policy. Do not make Hub credentials or unlimited timeouts global
+  merely because the weekly suite needs them.

packages/performance-tests/README.md

@@ -1,7 +1,10 @@
-# Presentation Performance Tests
+# Transformer Performance Tests
 
 A package containing performance tests for the [`@itwin/imodel-transformer` library](../../README.md).
 
+See [ARCHITECTURE.md](./ARCHITECTURE.md) for the test categories, weekly suite
+lifecycle, registration model, and extension guidance.
+
 ## Tests
 
 Here are tests we need but don't have:
@@ -17,7 +20,6 @@ Here are tests we need but don't have:
 - *Processing Changes*
 - *More Branching Stuff*
 
-
 ## Usage
 
 1. Clone the repository.
@@ -30,16 +32,11 @@ Here are tests we need but don't have:
 
 3. Create `.env` file using `template.env` template.
 
-5. Run:
+4. Run the serialized Vitest suite:
 
    ```sh
    pnpm test
    ```
 
-<!-- FIXME: output csv -->
-6. Review results like:
-
-```sh
-pnpm exec process-results < report.jsonl
-```
-
+5. Review `test/.output/report.csv`. This path is also the artifact contract used by
+   the weekly Azure pipeline.

packages/performance-tests/package.json

@@ -4,12 +4,11 @@
   "license": "MIT",
   "version": "0.1.0",
   "scripts": {
-    "build": "tsc 1>&2",
+    "build": "tsc --noEmit --incremental false 1>&2",
     "clean": "rimraf lib",
     "lint": "eslint \"./test/**/*.ts\" 1>&2",
-    "test": "mocha --delay --timeout 300000 --require ts-node/register test/**/*.test.ts",
+    "test": "vitest run",
     "format": "prettier \"./test/**/*.ts\" --write",
-    "test-mocha": "mocha --delay --timeout 300000 \"./lib/**/TransformerRegression.test.js\"",
     "process-reports": "node scripts/process-reports"
   },
   "repository": {},
@@ -37,19 +36,15 @@
     "@itwin/eslint-plugin": "^5.2.1",
     "@itwin/itwins-client": "^1.6.1",
     "@itwin/oidc-signin-tool": "^6.0.0",
-    "@types/chai": "^4.1.4",
     "@types/fs-extra": "^4.0.7",
-    "@types/mocha": "^8.2.2",
     "@types/node": "^22",
     "@types/yargs": "^12.0.5",
-    "chai": "^4.3.6",
     "eslint": "^9.11.1",
     "eslint-config-prettier": "^9.1.0",
-    "mocha": "^10.0.0",
     "prettier": "^3.1.1",
     "rimraf": "^3.0.2",
-    "ts-node": "^10.7.0",
-    "typescript": "~5.6.2"
+    "typescript": "~5.6.2",
+    "vitest": "4.1.10"
   },
   "eslintConfig": {
     "plugins": [

packages/performance-tests/test/Cleanup.ts

@@ -0,0 +1,73 @@
+/*---------------------------------------------------------------------------------------------
+ * Copyright (c) Bentley Systems, Incorporated. All rights reserved.
+ * See LICENSE.md in the project root for license terms and full copyright notice.
+ *--------------------------------------------------------------------------------------------*/
+
+export interface CleanupTask {
+  name: string;
+  run(): void | Promise<void>;
+}
+
+interface NamedCleanupError {
+  name: string;
+  error: unknown;
+}
+
+function toError({ name, error }: NamedCleanupError): Error {
+  return new Error(`Cleanup failed: ${name}`, { cause: error });
+}
+
+async function collectCleanupErrors(
+  cleanupTasks: CleanupTask[]
+): Promise<Error[]> {
+  const errors: NamedCleanupError[] = [];
+  for (const cleanupTask of cleanupTasks) {
+    try {
+      await cleanupTask.run();
+    } catch (error) {
+      errors.push({ name: cleanupTask.name, error });
+    }
+  }
+  return errors.map(toError);
+}
+
+export async function runCleanupTasks(
+  cleanupTasks: CleanupTask[]
+): Promise<void> {
+  const errors = await collectCleanupErrors(cleanupTasks);
+  if (errors.length === 1) {
+    throw errors[0];
+  }
+  if (errors.length > 1) {
+    throw new AggregateError(errors, "Multiple cleanup tasks failed");
+  }
+}
+
+export async function throwAfterCleanup(
+  primaryError: unknown,
+  cleanupTasks: CleanupTask[]
+): Promise<never> {
+  const cleanupErrors = await collectCleanupErrors(cleanupTasks);
+  if (cleanupErrors.length === 0) {
+    throw primaryError;
+  }
+  throw new AggregateError(
+    [primaryError, ...cleanupErrors],
+    "Operation and cleanup both failed",
+    { cause: primaryError }
+  );
+}
+
+export async function runWithCleanup<T>(
+  operation: () => Promise<T>,
+  cleanupTasks: CleanupTask[]
+): Promise<T> {
+  let result!: T;
+  try {
+    result = await operation();
+  } catch (error) {
+    await throwAfterCleanup(error, cleanupTasks);
+  }
+  await runCleanupTasks(cleanupTasks);
+  return result;
+}

packages/performance-tests/test/RegressionTestRegistration.ts

@@ -0,0 +1,38 @@
+/*---------------------------------------------------------------------------------------------
+ * Copyright (c) Bentley Systems, Incorporated. All rights reserved.
+ * See LICENSE.md in the project root for license terms and full copyright notice.
+ *--------------------------------------------------------------------------------------------*/
+
+import { TestTransformerModule } from "./TestTransformerModule";
+
+export interface RegressionTestCase<T> {
+  testCase: T;
+  functionNameToValidate: keyof TestTransformerModule;
+}
+
+export interface RegressionTestDefinition<T> {
+  testCaseName: string;
+  testCase: T;
+  moduleName: string;
+  transformerModule: TestTransformerModule;
+}
+
+export function getRegressionTestDefinitions<T>(
+  testCases: ReadonlyMap<string, RegressionTestCase<T>>,
+  transformerModules: ReadonlyMap<string, TestTransformerModule>
+): RegressionTestDefinition<T>[] {
+  const definitions: RegressionTestDefinition<T>[] = [];
+  for (const [testCaseName, testCaseDefinition] of testCases) {
+    for (const [moduleName, transformerModule] of transformerModules) {
+      if (transformerModule[testCaseDefinition.functionNameToValidate]) {
+        definitions.push({
+          testCaseName,
+          testCase: testCaseDefinition.testCase,
+          moduleName,
+          transformerModule,
+        });
+      }
+    }
+  }
+  return definitions;
+}

packages/performance-tests/test/TestUtils.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 import * as fs from "node:fs";
 import * as path from "node:path";
-import { assert } from "chai";
+import { assert } from "vitest";
 import { IModelDb } from "@itwin/core-backend";
 import { DbResult, StopWatch } from "@itwin/core-bentley";
 import { GeometryStreamBuilder, GeometryStreamProps } from "@itwin/core-common";
@@ -83,7 +83,7 @@ export function timed<R extends any | Promise<any>>(
   }
 }
 
-// Mocha tests must know the test cases ahead time, so we collect the the Imodels first before beginning the tests
+// Vitest must know the test cases during collection, so collect the iModels first.
 export async function preFetchAsyncIterator<T>(
   iter: AsyncGenerator<T>
 ): Promise<T[]> {