Migrate weekly performance tests to Vitest

.vscode/launch.json

@@ -11,8 +11,7 @@
         "runtimeExecutable": "npm",
         "cwd": "${workspaceFolder}/packages/performance-tests/",
         "runtimeArgs": [
-          "run",
-          "test-mocha"
+          "test"
         ],
         "skipFiles": [
           "<node_internals>/**"
@@ -31,17 +30,6 @@
         "skipFiles": [
           "<node_internals>/**"
         ]
-      },
-      {
-        "type": "node",
-        "request": "launch",
-        "name": "Performance tests",
-        "runtimeExecutable": "npm",
-        "cwd": "${workspaceFolder}/packages/performance-tests/test",
-        "runtimeArgs": [
-          "run",
-          "test"
-        ],
       }
     ]
-}
+}

packages/performance-tests/README.md

@@ -1,4 +1,4 @@
-# Presentation Performance Tests
+# Transformer Performance Tests
 
 A package containing performance tests for the [`@itwin/imodel-transformer` library](../../README.md).
 
@@ -17,7 +17,6 @@ Here are tests we need but don't have:
 - *Processing Changes*
 - *More Branching Stuff*
 
-
 ## Usage
 
 1. Clone the repository.
@@ -30,16 +29,11 @@ Here are tests we need but don't have:
 
 3. Create `.env` file using `template.env` template.
 
-5. Run:
+4. Run the serialized Vitest suite:
 
    ```sh
    pnpm test
    ```
 
-<!-- FIXME: output csv -->
-6. Review results like:
-
-```sh
-pnpm exec process-results < report.jsonl
-```
-
+5. Review `test/.output/report.csv`. This path is also the artifact contract used by
+   the weekly Azure pipeline.

packages/performance-tests/package.json

@@ -4,12 +4,11 @@
   "license": "MIT",
   "version": "0.1.0",
   "scripts": {
-    "build": "tsc 1>&2",
+    "build": "tsc --noEmit --incremental false 1>&2",
     "clean": "rimraf lib",
     "lint": "eslint \"./test/**/*.ts\" 1>&2",
-    "test": "mocha --delay --timeout 300000 --require ts-node/register test/**/*.test.ts",
+    "test": "vitest run",
     "format": "prettier \"./test/**/*.ts\" --write",
-    "test-mocha": "mocha --delay --timeout 300000 \"./lib/**/TransformerRegression.test.js\"",
     "process-reports": "node scripts/process-reports"
   },
   "repository": {},
@@ -37,19 +36,15 @@
     "@itwin/eslint-plugin": "^5.2.1",
     "@itwin/itwins-client": "^1.6.1",
     "@itwin/oidc-signin-tool": "^6.0.0",
-    "@types/chai": "^4.1.4",
     "@types/fs-extra": "^4.0.7",
-    "@types/mocha": "^8.2.2",
     "@types/node": "^22",
     "@types/yargs": "^12.0.5",
-    "chai": "^4.3.6",
     "eslint": "^9.11.1",
     "eslint-config-prettier": "^9.1.0",
-    "mocha": "^10.0.0",
     "prettier": "^3.1.1",
     "rimraf": "^3.0.2",
-    "ts-node": "^10.7.0",
-    "typescript": "~5.6.2"
+    "typescript": "~5.6.2",
+    "vitest": "4.1.10"
   },
   "eslintConfig": {
     "plugins": [

packages/performance-tests/test/Cleanup.ts

@@ -0,0 +1,73 @@
+/*---------------------------------------------------------------------------------------------
+ * Copyright (c) Bentley Systems, Incorporated. All rights reserved.
+ * See LICENSE.md in the project root for license terms and full copyright notice.
+ *--------------------------------------------------------------------------------------------*/
+
+export interface CleanupTask {
+  name: string;
+  run(): void | Promise<void>;
+}
+
+interface NamedCleanupError {
+  name: string;
+  error: unknown;
+}
+
+function toError({ name, error }: NamedCleanupError): Error {
+  return new Error(`Cleanup failed: ${name}`, { cause: error });
+}
+
+async function collectCleanupErrors(
+  cleanupTasks: CleanupTask[]
+): Promise<Error[]> {
+  const errors: NamedCleanupError[] = [];
+  for (const cleanupTask of cleanupTasks) {
+    try {
+      await cleanupTask.run();
+    } catch (error) {
+      errors.push({ name: cleanupTask.name, error });
+    }
+  }
+  return errors.map(toError);
+}
+
+export async function runCleanupTasks(
+  cleanupTasks: CleanupTask[]
+): Promise<void> {
+  const errors = await collectCleanupErrors(cleanupTasks);
+  if (errors.length === 1) {
+    throw errors[0];
+  }
+  if (errors.length > 1) {
+    throw new AggregateError(errors, "Multiple cleanup tasks failed");
+  }
+}
+
+export async function throwAfterCleanup(
+  primaryError: unknown,
+  cleanupTasks: CleanupTask[]
+): Promise<never> {
+  const cleanupErrors = await collectCleanupErrors(cleanupTasks);
+  if (cleanupErrors.length === 0) {
+    throw primaryError;
+  }
+  throw new AggregateError(
+    [primaryError, ...cleanupErrors],
+    "Operation and cleanup both failed",
+    { cause: primaryError }
+  );
+}
+
+export async function runWithCleanup<T>(
+  operation: () => Promise<T>,
+  cleanupTasks: CleanupTask[]
+): Promise<T> {
+  let result!: T;
+  try {
+    result = await operation();
+  } catch (error) {
+    await throwAfterCleanup(error, cleanupTasks);
+  }
+  await runCleanupTasks(cleanupTasks);
+  return result;
+}

packages/performance-tests/test/RegressionTestRegistration.ts

@@ -0,0 +1,38 @@
+/*---------------------------------------------------------------------------------------------
+ * Copyright (c) Bentley Systems, Incorporated. All rights reserved.
+ * See LICENSE.md in the project root for license terms and full copyright notice.
+ *--------------------------------------------------------------------------------------------*/
+
+import { TestTransformerModule } from "./TestTransformerModule";
+
+export interface RegressionTestCase<T> {
+  testCase: T;
+  functionNameToValidate: keyof TestTransformerModule;
+}
+
+export interface RegressionTestDefinition<T> {
+  testCaseName: string;
+  testCase: T;
+  moduleName: string;
+  transformerModule: TestTransformerModule;
+}
+
+export function getRegressionTestDefinitions<T>(
+  testCases: ReadonlyMap<string, RegressionTestCase<T>>,
+  transformerModules: ReadonlyMap<string, TestTransformerModule>
+): RegressionTestDefinition<T>[] {
+  const definitions: RegressionTestDefinition<T>[] = [];
+  for (const [testCaseName, testCaseDefinition] of testCases) {
+    for (const [moduleName, transformerModule] of transformerModules) {
+      if (transformerModule[testCaseDefinition.functionNameToValidate]) {
+        definitions.push({
+          testCaseName,
+          testCase: testCaseDefinition.testCase,
+          moduleName,
+          transformerModule,
+        });
+      }
+    }
+  }
+  return definitions;
+}

packages/performance-tests/test/TestUtils.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 import * as fs from "node:fs";
 import * as path from "node:path";
-import { assert } from "chai";
+import { assert } from "vitest";
 import { IModelDb } from "@itwin/core-backend";
 import { DbResult, StopWatch } from "@itwin/core-bentley";
 import { GeometryStreamBuilder, GeometryStreamProps } from "@itwin/core-common";
@@ -83,7 +83,7 @@ export function timed<R extends any | Promise<any>>(
   }
 }
 
-// Mocha tests must know the test cases ahead time, so we collect the the Imodels first before beginning the tests
+// Vitest must know the test cases during collection, so collect the iModels first.
 export async function preFetchAsyncIterator<T>(
   iter: AsyncGenerator<T>
 ): Promise<T[]> {