From 09f4a4f246275d82d54da283e46e00c4e303eb04 Mon Sep 17 00:00:00 2001 From: Nam Le <50554904+hl662@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:06:38 -0400 Subject: [PATCH 1/5] Migrate weekly performance tests to Vitest Replace Mocha delayed registration with serialized Vitest collection and explicit worker lifecycle cleanup while preserving the weekly report and authentication contracts. Validated transformer build, performance test type-check, lint, and focused cleanup/registration tests. The credentialed suite has not been executed locally; live Bentley Hub runtime behavior remains to be validated by the weekly ADO pipeline. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .vscode/launch.json | 16 +- packages/performance-tests/README.md | 14 +- packages/performance-tests/package.json | 13 +- packages/performance-tests/test/Cleanup.ts | 73 +++++ .../test/RegressionTestRegistration.ts | 38 +++ packages/performance-tests/test/TestUtils.ts | 4 +- .../test/TransformerRegression.test.ts | 297 +++++++++++------- packages/performance-tests/test/rawInserts.ts | 216 +++++++------ .../test/unit/Cleanup.test.ts | 55 ++++ .../unit/RegressionTestRegistration.test.ts | 56 ++++ packages/performance-tests/test/vitest.d.ts | 8 + packages/performance-tests/tsconfig.json | 4 +- packages/performance-tests/vitest.config.ts | 21 ++ pnpm-lock.yaml | 190 +---------- 14 files changed, 567 insertions(+), 438 deletions(-) create mode 100644 packages/performance-tests/test/Cleanup.ts create mode 100644 packages/performance-tests/test/RegressionTestRegistration.ts create mode 100644 packages/performance-tests/test/unit/Cleanup.test.ts create mode 100644 packages/performance-tests/test/unit/RegressionTestRegistration.test.ts create mode 100644 packages/performance-tests/test/vitest.d.ts create mode 100644 packages/performance-tests/vitest.config.ts diff --git a/.vscode/launch.json b/.vscode/launch.json index 1d5c91ce3..f91b11e63 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,8 +11,7 @@ "runtimeExecutable": "npm", "cwd": "${workspaceFolder}/packages/performance-tests/", "runtimeArgs": [ - "run", - "test-mocha" + "test" ], "skipFiles": [ "/**" @@ -31,17 +30,6 @@ "skipFiles": [ "/**" ] - }, - { - "type": "node", - "request": "launch", - "name": "Performance tests", - "runtimeExecutable": "npm", - "cwd": "${workspaceFolder}/packages/performance-tests/test", - "runtimeArgs": [ - "run", - "test" - ], } ] -} \ No newline at end of file +} diff --git a/packages/performance-tests/README.md b/packages/performance-tests/README.md index b9dc054f4..b520154e2 100644 --- a/packages/performance-tests/README.md +++ b/packages/performance-tests/README.md @@ -1,4 +1,4 @@ -# Presentation Performance Tests +# Transformer Performance Tests A package containing performance tests for the [`@itwin/imodel-transformer` library](../../README.md). @@ -17,7 +17,6 @@ Here are tests we need but don't have: - *Processing Changes* - *More Branching Stuff* - ## Usage 1. Clone the repository. @@ -30,16 +29,11 @@ Here are tests we need but don't have: 3. Create `.env` file using `template.env` template. -5. Run: +4. Run the serialized Vitest suite: ```sh pnpm test ``` - -6. Review results like: - -```sh -pnpm exec process-results < report.jsonl -``` - +5. Review `test/.output/report.csv`. This path is also the artifact contract used by + the weekly Azure pipeline. diff --git a/packages/performance-tests/package.json b/packages/performance-tests/package.json index 09a07f57e..2b9885e29 100644 --- a/packages/performance-tests/package.json +++ b/packages/performance-tests/package.json @@ -4,12 +4,11 @@ "license": "MIT", "version": "0.1.0", "scripts": { - "build": "tsc 1>&2", + "build": "tsc --noEmit --incremental false 1>&2", "clean": "rimraf lib", "lint": "eslint \"./test/**/*.ts\" 1>&2", - "test": "mocha --delay --timeout 300000 --require ts-node/register test/**/*.test.ts", + "test": "vitest run", "format": "prettier \"./test/**/*.ts\" --write", - "test-mocha": "mocha --delay --timeout 300000 \"./lib/**/TransformerRegression.test.js\"", "process-reports": "node scripts/process-reports" }, "repository": {}, @@ -37,19 +36,15 @@ "@itwin/eslint-plugin": "^5.2.1", "@itwin/itwins-client": "^1.6.1", "@itwin/oidc-signin-tool": "^6.0.0", - "@types/chai": "^4.1.4", "@types/fs-extra": "^4.0.7", - "@types/mocha": "^8.2.2", "@types/node": "^22", "@types/yargs": "^12.0.5", - "chai": "^4.3.6", "eslint": "^9.11.1", "eslint-config-prettier": "^9.1.0", - "mocha": "^10.0.0", "prettier": "^3.1.1", "rimraf": "^3.0.2", - "ts-node": "^10.7.0", - "typescript": "~5.6.2" + "typescript": "~5.6.2", + "vitest": "4.1.10" }, "eslintConfig": { "plugins": [ diff --git a/packages/performance-tests/test/Cleanup.ts b/packages/performance-tests/test/Cleanup.ts new file mode 100644 index 000000000..d014ee946 --- /dev/null +++ b/packages/performance-tests/test/Cleanup.ts @@ -0,0 +1,73 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +export interface CleanupTask { + name: string; + run(): void | Promise; +} + +interface NamedCleanupError { + name: string; + error: unknown; +} + +function toError({ name, error }: NamedCleanupError): Error { + return new Error(`Cleanup failed: ${name}`, { cause: error }); +} + +async function collectCleanupErrors( + cleanupTasks: CleanupTask[] +): Promise { + const errors: NamedCleanupError[] = []; + for (const cleanupTask of cleanupTasks) { + try { + await cleanupTask.run(); + } catch (error) { + errors.push({ name: cleanupTask.name, error }); + } + } + return errors.map(toError); +} + +export async function runCleanupTasks( + cleanupTasks: CleanupTask[] +): Promise { + const errors = await collectCleanupErrors(cleanupTasks); + if (errors.length === 1) { + throw errors[0]; + } + if (errors.length > 1) { + throw new AggregateError(errors, "Multiple cleanup tasks failed"); + } +} + +export async function throwAfterCleanup( + primaryError: unknown, + cleanupTasks: CleanupTask[] +): Promise { + const cleanupErrors = await collectCleanupErrors(cleanupTasks); + if (cleanupErrors.length === 0) { + throw primaryError; + } + throw new AggregateError( + [primaryError, ...cleanupErrors], + "Operation and cleanup both failed", + { cause: primaryError } + ); +} + +export async function runWithCleanup( + operation: () => Promise, + cleanupTasks: CleanupTask[] +): Promise { + let result!: T; + try { + result = await operation(); + } catch (error) { + await throwAfterCleanup(error, cleanupTasks); + } + await runCleanupTasks(cleanupTasks); + return result; +} diff --git a/packages/performance-tests/test/RegressionTestRegistration.ts b/packages/performance-tests/test/RegressionTestRegistration.ts new file mode 100644 index 000000000..80cac5e74 --- /dev/null +++ b/packages/performance-tests/test/RegressionTestRegistration.ts @@ -0,0 +1,38 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { TestTransformerModule } from "./TestTransformerModule"; + +export interface RegressionTestCase { + testCase: T; + functionNameToValidate: keyof TestTransformerModule; +} + +export interface RegressionTestDefinition { + testCaseName: string; + testCase: T; + moduleName: string; + transformerModule: TestTransformerModule; +} + +export function getRegressionTestDefinitions( + testCases: ReadonlyMap>, + transformerModules: ReadonlyMap +): RegressionTestDefinition[] { + const definitions: RegressionTestDefinition[] = []; + for (const [testCaseName, testCaseDefinition] of testCases) { + for (const [moduleName, transformerModule] of transformerModules) { + if (transformerModule[testCaseDefinition.functionNameToValidate]) { + definitions.push({ + testCaseName, + testCase: testCaseDefinition.testCase, + moduleName, + transformerModule, + }); + } + } + } + return definitions; +} diff --git a/packages/performance-tests/test/TestUtils.ts b/packages/performance-tests/test/TestUtils.ts index 13d82f621..fb38d835d 100644 --- a/packages/performance-tests/test/TestUtils.ts +++ b/packages/performance-tests/test/TestUtils.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import * as fs from "node:fs"; import * as path from "node:path"; -import { assert } from "chai"; +import { assert } from "vitest"; import { IModelDb } from "@itwin/core-backend"; import { DbResult, StopWatch } from "@itwin/core-bentley"; import { GeometryStreamBuilder, GeometryStreamProps } from "@itwin/core-common"; @@ -83,7 +83,7 @@ export function timed>( } } -// Mocha tests must know the test cases ahead time, so we collect the the Imodels first before beginning the tests +// Vitest must know the test cases during collection, so collect the iModels first. export async function preFetchAsyncIterator( iter: AsyncGenerator ): Promise { diff --git a/packages/performance-tests/test/TransformerRegression.test.ts b/packages/performance-tests/test/TransformerRegression.test.ts index 25e330905..799550459 100644 --- a/packages/performance-tests/test/TransformerRegression.test.ts +++ b/packages/performance-tests/test/TransformerRegression.test.ts @@ -3,49 +3,57 @@ * See LICENSE.md in the project root for license terms and full copyright notice. *--------------------------------------------------------------------------------------------*/ -/* - * Tests where we perform "identity" transforms, that is just rebuilding an entire identical iModel (minus IDs) - * through the transformation process. - */ - import "./setup"; import * as fs from "node:fs"; import * as path from "node:path"; -import { BackendIModelsAccess } from "@itwin/imodels-access-backend"; +import assert from "node:assert"; import { BriefcaseDb, IModelHost, IModelHostConfiguration, } from "@itwin/core-backend"; import { Logger, LogLevel } from "@itwin/core-bentley"; +import { TransformerLoggerCategory } from "@itwin/imodel-transformer"; +import { BackendIModelsAccess } from "@itwin/imodels-access-backend"; import { IModelsClient } from "@itwin/imodels-client-authoring"; import { NodeCliAuthorizationClient } from "@itwin/node-cli-authorization"; -import { Reporter } from "@itwin/perf-tools"; -import { ReporterInfo } from "./ReporterUtils"; -import { TestBrowserAuthorizationClient } from "@itwin/oidc-signin-tool"; -import { TestTransformerModule } from "./TestTransformerModule"; -import { TransformerLoggerCategory } from "@itwin/imodel-transformer"; import { AzureClientStorage, BlockBlobClientWrapperFactory, } from "@itwin/object-storage-azure"; +import { Reporter } from "@itwin/perf-tools"; +import { TestBrowserAuthorizationClient } from "@itwin/oidc-signin-tool"; +import { + CleanupTask, + runCleanupTasks, + runWithCleanup, + throwAfterCleanup, +} from "./Cleanup"; +import { getBranchName } from "./GitUtils"; +import { + getRegressionTestDefinitions, + RegressionTestCase, +} from "./RegressionTestRegistration"; +import { ReporterInfo } from "./ReporterUtils"; +import { getTestIModels, TestIModel } from "./TestContext"; +import { TestTransformerModule } from "./TestTransformerModule"; import { filterIModels, initOutputFile, preFetchAsyncIterator, } from "./TestUtils"; -import { getBranchName } from "./GitUtils"; -import { getTestIModels } from "./TestContext"; -import assert from "node:assert"; +import identityTransformer from "./cases/identity-transformer"; +import prepareFork from "./cases/prepare-fork"; +import rawInserts from "./rawInserts"; import nativeTransformerTestModule from "./transformers/NativeTransformer"; import rawForkOperationsTestModule from "./transformers/RawForkOperations"; -import rawInserts from "./rawInserts"; -// cases -import identityTransformer from "./cases/identity-transformer"; -import prepareFork from "./cases/prepare-fork"; +type AuthorizationClient = + | NodeCliAuthorizationClient + | TestBrowserAuthorizationClient; +type PerformanceTestCase = typeof identityTransformer; -const testCasesMap = new Map([ +const testCasesMap = new Map>([ [ "identity transform (provenance)", { @@ -65,27 +73,65 @@ const testCasesMap = new Map([ const loggerCategory = "Transformer Performance Regression Tests"; const outputDir = path.join(__dirname, ".output"); -const loadTransformers = async () => { +class WorkerLifecycle { + private _authClient?: AuthorizationClient; + private _hostStarted = false; + + public setAuthClient(authClient: AuthorizationClient): void { + this._authClient = authClient; + } + + public markHostStarted(): void { + this._hostStarted = true; + } + + public async shutdown(): Promise { + const hostStarted = this._hostStarted; + const authClient = this._authClient; + this._hostStarted = false; + this._authClient = undefined; + + const cleanupTasks: CleanupTask[] = []; + if (hostStarted) { + cleanupTasks.push({ + name: "IModelHost shutdown", + run: async () => IModelHost.shutdown(), + }); + } + if (authClient) { + cleanupTasks.push({ + name: "authorization sign out", + run: async () => authClient.signOut(), + }); + } + await runCleanupTasks(cleanupTasks); + } +} + +async function loadTransformers(): Promise> { const modulePaths = process.env.EXTRA_TRANSFORMERS?.split(",") .map((name) => name.trim()) .filter(Boolean) ?? []; - const envSpecifiedExtraTransformerCases = (await Promise.all( - modulePaths.map(async (m) => [m, (await import(m)).default]) + const extraTransformerCases = (await Promise.all( + modulePaths.map(async (modulePath) => [ + modulePath, + (await import(modulePath)).default, + ]) )) as [string, TestTransformerModule][]; - const transformerModules = new Map([ + return new Map([ ["NativeTransformer", nativeTransformerTestModule], ["RawForkOperations", rawForkOperationsTestModule], - ...envSpecifiedExtraTransformerCases, + ...extraTransformerCases, ]); - return transformerModules; -}; +} -const setupTestData = async () => { +async function setupTestData( + workerLifecycle: WorkerLifecycle +): Promise { const logLevel = process.env.LOG_LEVEL ? Number(process.env.LOG_LEVEL) : LogLevel.Error; - assert(LogLevel[logLevel] !== undefined, "unknown log level"); Logger.initializeToConsole(); @@ -109,10 +155,7 @@ const setupTestData = async () => { assert(usrEmail, "user name was not configured"); assert(usrPass, "user password was not configured"); - const user = { - email: usrEmail, - password: usrPass, - }; + const user = { email: usrEmail, password: usrPass }; assert(process.env.OIDC_CLIENT_ID, "OIDC_CLIENT_ID not set"); assert(process.env.OIDC_REDIRECT, "OIDC_REDIRECT not set"); @@ -135,7 +178,7 @@ const setupTestData = async () => { redirectUri: process.env.OIDC_REDIRECT, scope: process.env.OIDC_SCOPES, }); - + workerLifecycle.setAuthClient(authClient); await authClient.signIn(); const hostConfig = new IModelHostConfiguration(); @@ -148,37 +191,62 @@ const setupTestData = async () => { }); hostConfig.hubAccess = new BackendIModelsAccess(hubClient); await IModelHost.startup(hostConfig); + workerLifecycle.markHostStarted(); return preFetchAsyncIterator(getTestIModels(filterIModels)); -}; +} -async function runRegressionTests() { - const testIModels = await setupTestData(); +async function collectRegressionInputs(workerLifecycle: WorkerLifecycle) { + const testIModels = await setupTestData(workerLifecycle); const transformerModules = await loadTransformers(); const reporter = new Reporter(); const reportPath = initOutputFile("report.csv", outputDir); const branchName = await getBranchName(); + return { + branchName, + reportPath, + reporter, + testIModels, + transformerModules, + }; +} + +const lifecycle = new WorkerLifecycle(); +const { + branchName: currentBranchName, + reportPath: csvReportPath, + reporter: performanceReporter, + testIModels: collectedIModels, + transformerModules: loadedTransformerModules, +} = await collectRegressionInputs(lifecycle).catch(async (error: unknown) => + throwAfterCleanup(error, [ + { name: "worker lifecycle", run: async () => lifecycle.shutdown() }, + ]) +); +const regressionTestDefinitions = getRegressionTestDefinitions( + testCasesMap, + loadedTransformerModules +); - describe("Transformer Regression Tests", function () { - testIModels.forEach(async (iModel) => { - let sourceDb: BriefcaseDb; +describe("Transformer Regression Tests", () => { + for (const iModel of collectedIModels) { + describe(`Transforms of ${iModel.name}`, () => { + let sourceDb: BriefcaseDb | undefined; let reportInfo: ReporterInfo; let sourceFileName: string; - describe(`Transforms of ${iModel.name}`, async () => { - before(async () => { - Logger.logInfo( - loggerCategory, - `processing iModel '${ - iModel.name - }' of size '${iModel.tShirtSize.toUpperCase()}'` - ); - sourceFileName = await iModel.getFileName(); - sourceDb = await BriefcaseDb.open({ - fileName: sourceFileName, - readonly: true, - }); - const fedGuidReader = sourceDb.createQueryReader( + beforeAll(async () => { + Logger.logInfo( + loggerCategory, + `processing iModel '${iModel.name}' of size '${iModel.tShirtSize.toUpperCase()}'` + ); + sourceFileName = await iModel.getFileName(); + const metadataDb = await BriefcaseDb.open({ + fileName: sourceFileName, + readonly: true, + }); + await runWithCleanup(async () => { + const fedGuidReader = metadataDb.createQueryReader( "SELECT CAST(SUM(IIF(FederationGuid IS NOT NULL, 1, 0)) AS DOUBLE)/COUNT(*) AS ratio FROM bis.Element", undefined, { usePrimaryConn: true } @@ -190,80 +258,81 @@ async function runRegressionTests() { loggerCategory, `Federation Guid Saturation '${fedGuidSaturation}'` ); - const toGb = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(2)}Gb`; - const sizeInGb = toGb(fs.statSync(sourceDb.pathName).size); + const sizeInGb = `${( + fs.statSync(metadataDb.pathName).size / + 1024 ** 3 + ).toFixed(2)}Gb`; Logger.logInfo(loggerCategory, `loaded (${sizeInGb})'`); reportInfo = { Id: iModel.iModelId, "T-shirt size": iModel.tShirtSize, "Gb size": sizeInGb, - "Branch Name": branchName, + "Branch Name": currentBranchName, "Federation Guid Saturation 0-1": fedGuidSaturation, }; - sourceDb.close(); - }); + }, [ + { + name: `${iModel.name} metadata briefcase`, + run: () => metadataDb.close(), + }, + ]); + }); - beforeEach(async () => { - sourceDb = await BriefcaseDb.open({ - fileName: sourceFileName, - readonly: true, - }); + beforeEach(async () => { + sourceDb = await BriefcaseDb.open({ + fileName: sourceFileName, + readonly: true, }); + }); - afterEach(async () => { - sourceDb.close(); // closing to ensure connection cache reusage doesn't affect results - }); + afterEach(() => { + const dbToClose = sourceDb; + sourceDb = undefined; + dbToClose?.close(); + }); - testCasesMap.forEach( - async ({ testCase, functionNameToValidate }, testCaseName) => { - transformerModules.forEach( - ( - transformerModule: TestTransformerModule, - moduleName: string - ) => { - const moduleFunc = - transformerModule[ - functionNameToValidate as keyof TestTransformerModule - ]; - if (moduleFunc) { - it(`${testCaseName} on ${moduleName}`, async () => { - const addReport = ( - iModelName: string, - valDescription: string, - value: number - ) => { - reporter.addEntry( - `${testCaseName} ${moduleName}`, - iModelName, - valDescription, - value, - reportInfo - ); - }; - await testCase({ sourceDb, transformerModule, addReport }); - // eslint-disable-next-line no-console - console.log("Finished the test"); - }).timeout(0); - } - } + for (const definition of regressionTestDefinitions) { + test(`${definition.testCaseName} on ${definition.moduleName}`, async () => { + assert(sourceDb, "source briefcase was not opened"); + const addReport = ( + iModelName: string, + valDescription: string, + value: number + ) => { + performanceReporter.addEntry( + `${definition.testCaseName} ${definition.moduleName}`, + iModelName, + valDescription, + value, + reportInfo ); - } - ); - }); + }; + await definition.testCase({ + sourceDb, + transformerModule: definition.transformerModule, + addReport, + }); + // eslint-disable-next-line no-console + console.log("Finished the test"); + }); + } }); + } - const _15minutes = 15 * 60 * 1000; - - it("Transform vs raw inserts", async () => { - return rawInserts(reporter, branchName); - }).timeout(0); - }); - - after(async () => { - reporter.exportCSV(reportPath); + test("Transform vs raw inserts", async () => { + await rawInserts(performanceReporter, currentBranchName); }); +}); - run(); -} - -void runRegressionTests(); +afterAll(async () => { + await runCleanupTasks([ + { + name: "report export", + run: () => performanceReporter.exportCSV(csvReportPath), + }, + { + name: "worker lifecycle", + run: async () => lifecycle.shutdown(), + }, + ]); +}); diff --git a/packages/performance-tests/test/rawInserts.ts b/packages/performance-tests/test/rawInserts.ts index 02373e219..03ee6d1a5 100644 --- a/packages/performance-tests/test/rawInserts.ts +++ b/packages/performance-tests/test/rawInserts.ts @@ -18,6 +18,7 @@ import { generateTestIModel } from "./iModelUtils"; import { count, initOutputFile, timed } from "./TestUtils"; import assert from "node:assert"; import path from "node:path"; +import { runWithCleanup } from "./Cleanup"; const loggerCategory = "Raw Inserts"; const outputDir = path.join(__dirname, ".output"); @@ -31,108 +32,121 @@ export default async function rawInserts( reporter: Reporter, branchName: string ) { - Logger.logInfo(loggerCategory, "starting 150k entity inserts"); - - let testIModel: TestIModel | undefined; - const [insertsTimer] = timed(() => { - testIModel = generateTestIModel({ - numElements: 100_000, - fedGuids: true, - fileName: "RawInserts-source.bim", + let sourceDb: StandaloneDb | undefined; + let targetDb: SnapshotDb | undefined; + let targetNoProvDb: SnapshotDb | undefined; + await runWithCleanup(async () => { + Logger.logInfo(loggerCategory, "starting 150k entity inserts"); + + let testIModel: TestIModel | undefined; + const [insertsTimer] = timed(() => { + testIModel = generateTestIModel({ + numElements: 100_000, + fedGuids: true, + fileName: "RawInserts-source.bim", + }); }); - }); - - if (testIModel === undefined) - throw Error("Generated iModel not correctly defined"); // needed because TS does not know that timer will run before insertsTimer - const fileName = await testIModel.getFileName(); - const sourceDb = StandaloneDb.openFile(fileName, OpenMode.ReadWrite); - - reporter.addEntry( - "populate by insert", - iModelName, - "time elapsed (seconds)", - insertsTimer?.elapsedSeconds ?? -1, - { - "Element Count": count(sourceDb, Element.classFullName), - "Relationship Count": count(sourceDb, ElementGroupsMembers.classFullName), - "Branch Name": branchName, - } - ); - - Logger.logInfo( - loggerCategory, - "Done. Starting with-provenance transformation of same content" - ); - - const targetPath = initOutputFile("RawInserts-Target.bim", outputDir); - const targetDb = SnapshotDb.createEmpty(targetPath, { - rootSubject: { name: "RawInsertsTarget" }, - }); - const withProvEditTxn = new EditTxn(targetDb, "IModelTransformer"); - withProvEditTxn.start(); - const transformerWithProv = new IModelTransformer( - { source: sourceDb, target: withProvEditTxn }, - { - noProvenance: false, - } - ); - - const [transformWithProvTimer] = await timed(async () => { - await transformerWithProv.process(); - }); - withProvEditTxn.end(); - - reporter.addEntry( - "populate by transform (adding provenance)", - iModelName, - "time elapsed (seconds)", - transformWithProvTimer?.elapsedSeconds ?? -1, - { - "Element Count": count(sourceDb, Element.classFullName), - "Relationship Count": count(sourceDb, ElementGroupsMembers.classFullName), - "Branch Name": branchName, - } - ); - Logger.logInfo( - loggerCategory, - "Done. Starting without-provenance transformation of same content" - ); - - const targetNoProvPath = initOutputFile( - "RawInserts-TargetNoProv.bim", - outputDir - ); - const targetNoProvDb = SnapshotDb.createEmpty(targetNoProvPath, { - rootSubject: { name: "RawInsertsTarget" }, - }); - const noProvEditTxn = new EditTxn(targetNoProvDb, "IModelTransformer"); - noProvEditTxn.start(); - const transformerNoProv = new IModelTransformer( - { source: sourceDb, target: noProvEditTxn }, - { - noProvenance: true, - } - ); - - const [transformNoProvTimer] = await timed(async () => { - await transformerNoProv.process(); - }); - noProvEditTxn.end(); - - reporter.addEntry( - "populate by transform", - iModelName, - "time elapsed (seconds)", - transformNoProvTimer?.elapsedSeconds ?? -1, + if (testIModel === undefined) + throw Error("Generated iModel not correctly defined"); + const fileName = await testIModel.getFileName(); + sourceDb = StandaloneDb.openFile(fileName, OpenMode.ReadWrite); + + reporter.addEntry( + "populate by insert", + iModelName, + "time elapsed (seconds)", + insertsTimer?.elapsedSeconds ?? -1, + { + "Element Count": count(sourceDb, Element.classFullName), + "Relationship Count": count( + sourceDb, + ElementGroupsMembers.classFullName + ), + "Branch Name": branchName, + } + ); + + Logger.logInfo( + loggerCategory, + "Done. Starting with-provenance transformation of same content" + ); + + const targetPath = initOutputFile("RawInserts-Target.bim", outputDir); + targetDb = SnapshotDb.createEmpty(targetPath, { + rootSubject: { name: "RawInsertsTarget" }, + }); + const withProvEditTxn = new EditTxn(targetDb, "IModelTransformer"); + withProvEditTxn.start(); + const transformerWithProv = new IModelTransformer( + { source: sourceDb, target: withProvEditTxn }, + { noProvenance: false } + ); + + const [transformWithProvTimer] = await timed(async () => { + await transformerWithProv.process(); + }); + withProvEditTxn.end(); + + reporter.addEntry( + "populate by transform (adding provenance)", + iModelName, + "time elapsed (seconds)", + transformWithProvTimer?.elapsedSeconds ?? -1, + { + "Element Count": count(sourceDb, Element.classFullName), + "Relationship Count": count( + sourceDb, + ElementGroupsMembers.classFullName + ), + "Branch Name": branchName, + } + ); + + Logger.logInfo( + loggerCategory, + "Done. Starting without-provenance transformation of same content" + ); + + const targetNoProvPath = initOutputFile( + "RawInserts-TargetNoProv.bim", + outputDir + ); + targetNoProvDb = SnapshotDb.createEmpty(targetNoProvPath, { + rootSubject: { name: "RawInsertsTarget" }, + }); + const noProvEditTxn = new EditTxn(targetNoProvDb, "IModelTransformer"); + noProvEditTxn.start(); + const transformerNoProv = new IModelTransformer( + { source: sourceDb, target: noProvEditTxn }, + { noProvenance: true } + ); + + const [transformNoProvTimer] = await timed(async () => { + await transformerNoProv.process(); + }); + noProvEditTxn.end(); + + reporter.addEntry( + "populate by transform", + iModelName, + "time elapsed (seconds)", + transformNoProvTimer?.elapsedSeconds ?? -1, + { + "Element Count": count(sourceDb, Element.classFullName), + "Relationship Count": count( + sourceDb, + ElementGroupsMembers.classFullName + ), + "Branch Name": branchName, + } + ); + }, [ { - "Element Count": count(sourceDb, Element.classFullName), - "Relationship Count": count(sourceDb, ElementGroupsMembers.classFullName), - "Branch Name": branchName, - } - ); - - sourceDb.close(); - targetDb.close(); - targetNoProvDb.close(); + name: "raw inserts no-provenance target", + run: () => targetNoProvDb?.close(), + }, + { name: "raw inserts provenance target", run: () => targetDb?.close() }, + { name: "raw inserts source", run: () => sourceDb?.close() }, + ]); } diff --git a/packages/performance-tests/test/unit/Cleanup.test.ts b/packages/performance-tests/test/unit/Cleanup.test.ts new file mode 100644 index 000000000..192ff0149 --- /dev/null +++ b/packages/performance-tests/test/unit/Cleanup.test.ts @@ -0,0 +1,55 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { describe, expect, it, vi } from "vitest"; +import { runCleanupTasks, runWithCleanup, throwAfterCleanup } from "../Cleanup"; + +describe("cleanup", () => { + it("runs every cleanup task when one fails", async () => { + const first = vi.fn(() => { + throw new Error("first cleanup"); + }); + const second = vi.fn(); + + await expect( + runCleanupTasks([ + { name: "first", run: first }, + { name: "second", run: second }, + ]) + ).rejects.toThrow("Cleanup failed: first"); + expect(first).toHaveBeenCalledOnce(); + expect(second).toHaveBeenCalledOnce(); + }); + + it("retains the primary failure when cleanup also fails", async () => { + const primaryError = new Error("primary"); + const cleanupError = new Error("cleanup"); + + const error = await throwAfterCleanup(primaryError, [ + { + name: "resource", + run: () => { + throw cleanupError; + }, + }, + ]).catch((caughtError: unknown) => caughtError); + + expect(error).toBeInstanceOf(AggregateError); + expect((error as AggregateError).errors[0]).toBe(primaryError); + expect((error as Error).cause).toBe(primaryError); + expect((error as AggregateError).errors[1]).toMatchObject({ + cause: cleanupError, + }); + }); + + it("cleans up after a successful operation", async () => { + const cleanup = vi.fn(); + + await expect( + runWithCleanup(async () => 42, [{ name: "resource", run: cleanup }]) + ).resolves.toBe(42); + expect(cleanup).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/performance-tests/test/unit/RegressionTestRegistration.test.ts b/packages/performance-tests/test/unit/RegressionTestRegistration.test.ts new file mode 100644 index 000000000..e25bdbffa --- /dev/null +++ b/packages/performance-tests/test/unit/RegressionTestRegistration.test.ts @@ -0,0 +1,56 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { describe, expect, it } from "vitest"; +import { getRegressionTestDefinitions } from "../RegressionTestRegistration"; +import { TestTransformerModule } from "../TestTransformerModule"; + +describe("getRegressionTestDefinitions", () => { + it("registers each supported case and skips unsupported combinations", () => { + const identityCase = () => "identity"; + const forkCase = () => "fork"; + const allOperations: TestTransformerModule = { + createIdentityTransform: async () => ({ run: async () => {} }), + createForkInitTransform: async () => ({ run: async () => {} }), + }; + const identityOnly: TestTransformerModule = { + createIdentityTransform: async () => ({ run: async () => {} }), + }; + + const definitions = getRegressionTestDefinitions( + new Map([ + [ + "identity", + { + testCase: identityCase, + functionNameToValidate: "createIdentityTransform", + }, + ], + [ + "fork", + { + testCase: forkCase, + functionNameToValidate: "createForkInitTransform", + }, + ], + ]), + new Map([ + ["all", allOperations], + ["identity-only", identityOnly], + ]) + ); + + expect( + definitions.map(({ testCaseName, moduleName }) => [ + testCaseName, + moduleName, + ]) + ).toEqual([ + ["identity", "all"], + ["identity", "identity-only"], + ["fork", "all"], + ]); + }); +}); diff --git a/packages/performance-tests/test/vitest.d.ts b/packages/performance-tests/test/vitest.d.ts new file mode 100644 index 000000000..03a558a91 --- /dev/null +++ b/packages/performance-tests/test/vitest.d.ts @@ -0,0 +1,8 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +/// + +export {}; diff --git a/packages/performance-tests/tsconfig.json b/packages/performance-tests/tsconfig.json index 2942eed50..30b1b5890 100644 --- a/packages/performance-tests/tsconfig.json +++ b/packages/performance-tests/tsconfig.json @@ -2,8 +2,8 @@ "extends": "./node_modules/@itwin/build-tools/tsconfig-base.json", "compilerOptions": { "skipLibCheck": true, - "outDir": "./lib", - "esModuleInterop": true + "esModuleInterop": true, + "module": "ESNext" }, "include": ["./test/**/*.ts"] } diff --git a/packages/performance-tests/vitest.config.ts b/packages/performance-tests/vitest.config.ts new file mode 100644 index 000000000..38c500a75 --- /dev/null +++ b/packages/performance-tests/vitest.config.ts @@ -0,0 +1,21 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + globals: true, + environment: "node", + include: ["test/TransformerRegression.test.ts", "test/unit/**/*.test.ts"], + setupFiles: ["./test/setup.ts"], + // Transformations and worker-owned startup/teardown may legitimately run for hours. + testTimeout: 0, + hookTimeout: 0, + pool: "forks", + maxWorkers: 1, + fileParallelism: false, + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7764f6a6d..f03ea5473 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,45 +111,33 @@ importers: '@itwin/oidc-signin-tool': specifier: ^6.0.0 version: 6.0.0(@itwin/core-bentley@5.10.3)(@itwin/service-authorization@2.1.0(@itwin/core-bentley@5.10.3)) - '@types/chai': - specifier: ^4.1.4 - version: 4.3.20 '@types/fs-extra': specifier: ^4.0.7 version: 4.0.15 - '@types/mocha': - specifier: ^8.2.2 - version: 8.2.3 '@types/node': specifier: ^22 version: 22.18.12 '@types/yargs': specifier: ^12.0.5 version: 12.0.20 - chai: - specifier: ^4.3.6 - version: 4.5.0 eslint: specifier: ^9.11.1 version: 9.38.0(supports-color@8.1.1) eslint-config-prettier: specifier: ^9.1.0 version: 9.1.2(eslint@9.38.0(supports-color@8.1.1)) - mocha: - specifier: ^10.0.0 - version: 10.8.2 prettier: specifier: ^3.1.1 version: 3.6.2 rimraf: specifier: ^3.0.2 version: 3.0.2 - ts-node: - specifier: ^10.7.0 - version: 10.9.2(@types/node@22.18.12)(typescript@5.6.3) typescript: specifier: ~5.6.2 version: 5.6.3 + vitest: + specifier: 4.1.10 + version: 4.1.10(@types/node@22.18.12)(@vitest/coverage-v8@4.1.10)(vite@8.1.5(@types/node@22.18.12)(yaml@2.9.0)) packages/test-app: dependencies: @@ -437,10 +425,6 @@ packages: '@bentley/imodeljs-native@5.10.35': resolution: {integrity: sha512-jK26ydqHHCCrTFEuGy4CzJVHAEFlSvp4zBipWr1q/7o6kb2xDhkbmD/LNpwjS98ur3cZh38tCJkzVdArwzy7Kg==} - '@cspotcode/source-map-support@0.8.1': - resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} - engines: {node: '>=12'} - '@emnapi/core@1.11.1': resolution: {integrity: sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==} @@ -738,9 +722,6 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} - '@jridgewell/trace-mapping@0.3.9': - resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} - '@js-sdsl/ordered-map@4.4.2': resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} @@ -981,18 +962,6 @@ packages: resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==} engines: {node: '>= 10'} - '@tsconfig/node10@1.0.11': - resolution: {integrity: sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==} - - '@tsconfig/node12@1.0.11': - resolution: {integrity: sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==} - - '@tsconfig/node14@1.0.3': - resolution: {integrity: sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==} - - '@tsconfig/node16@1.0.4': - resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} - '@tybys/wasm-util@0.10.3': resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==} @@ -1009,9 +978,6 @@ packages: '@types/chai@4.3.1': resolution: {integrity: sha512-/zPMqDkzSZ8t3VtxOa4KPq7uzzW978M9Tvh+j7GHKuo6k6GTLxPJ4J5gE5cjfJ26pnXst0N5Hax8Sr0T2Mi9zQ==} - '@types/chai@4.3.20': - resolution: {integrity: sha512-/pC9HAB5I/xMlc5FP77qjCnI16ChlJfW0tGa0IUcFn38VJrTV6DeZ60NU5KZBtaOZqjdpwTWohz5HU1RrhiYxQ==} - '@types/chai@5.2.3': resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} @@ -1214,10 +1180,6 @@ packages: peerDependencies: acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 - acorn-walk@8.3.4: - resolution: {integrity: sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==} - engines: {node: '>=0.4.0'} - acorn@8.15.0: resolution: {integrity: sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==} engines: {node: '>=0.4.0'} @@ -1296,9 +1258,6 @@ packages: resolution: {integrity: sha512-ixiS0nLNNG5jNQzgZJNoUpBKdo9yTYZMGJ+QgT2jmjR7G7+QHRCc4v6LQ3NgE7EBJq+o0ams3waJwkrlBom8Ig==} engines: {node: '>=14'} - arg@4.1.3: - resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} - argparse@1.0.10: resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} @@ -1352,9 +1311,6 @@ packages: resolution: {integrity: sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==} engines: {node: '>=8'} - assertion-error@1.1.0: - resolution: {integrity: sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==} - assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} @@ -1488,10 +1444,6 @@ packages: canonical-path@1.0.0: resolution: {integrity: sha512-feylzsbDxi1gPZ1IjystzIQZagYYLvfKrSuygUCgf7z6x790VEzze5QEkdSV1U58RA7Hi0+v6fv4K54atOzATg==} - chai@4.5.0: - resolution: {integrity: sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==} - engines: {node: '>=4'} - chai@6.2.2: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} @@ -1511,9 +1463,6 @@ packages: charenc@0.0.2: resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==} - check-error@1.0.3: - resolution: {integrity: sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==} - chokidar@3.6.0: resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==} engines: {node: '>= 8.10.0'} @@ -1600,9 +1549,6 @@ packages: engines: {node: ^20.0.0 || >=22.0.0, npm: '>=10'} hasBin: true - create-require@1.1.1: - resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} - cross-env@5.2.1: resolution: {integrity: sha512-1yHhtcfAd1r4nwQgknowuUNfIT9E8dOMMspC36g45dN+iD1blloi7xp8X/xAIDnjHWyt1uQ8PHk2fkNaym7soQ==} engines: {node: '>=4.0'} @@ -1675,10 +1621,6 @@ packages: resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} engines: {node: '>=10'} - deep-eql@4.1.4: - resolution: {integrity: sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg==} - engines: {node: '>=6'} - deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} @@ -1727,10 +1669,6 @@ packages: engines: {node: '>= 4.2.1'} hasBin: true - diff@4.0.2: - resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==} - engines: {node: '>=0.3.1'} - diff@5.2.0: resolution: {integrity: sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==} engines: {node: '>=0.3.1'} @@ -2216,9 +2154,6 @@ packages: resolution: {integrity: sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==} engines: {node: '>=18'} - get-func-name@2.0.2: - resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==} - get-intrinsic@1.3.0: resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} engines: {node: '>= 0.4'} @@ -2917,9 +2852,6 @@ packages: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true - loupe@2.3.7: - resolution: {integrity: sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==} - lowercase-keys@3.0.0: resolution: {integrity: sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -2951,9 +2883,6 @@ packages: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} engines: {node: '>=10'} - make-error@1.3.6: - resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==} - markdown-it@14.2.0: resolution: {integrity: sha512-1TGiQiJVRQ3NPmZH6sx5Cfnmg6GQm9jvC1ch4TK511NjSJvjzKLzn5pPfZRNZkRPZP0HqCioSndqH8v2nRaWVQ==} hasBin: true @@ -3338,9 +3267,6 @@ packages: pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} - pathval@1.1.1: - resolution: {integrity: sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==} - picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -3890,20 +3816,6 @@ packages: peerDependencies: typescript: '>=4.8.4' - ts-node@10.9.2: - resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==} - hasBin: true - peerDependencies: - '@swc/core': '>=1.2.50' - '@swc/wasm': '>=1.2.50' - '@types/node': '*' - typescript: '>=2.7' - peerDependenciesMeta: - '@swc/core': - optional: true - '@swc/wasm': - optional: true - tsconfig-paths@3.15.0: resolution: {integrity: sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg==} @@ -3914,10 +3826,6 @@ packages: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} - type-detect@4.1.0: - resolution: {integrity: sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==} - engines: {node: '>=4'} - type-is@1.6.18: resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} engines: {node: '>= 0.6'} @@ -4013,9 +3921,6 @@ packages: deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). hasBin: true - v8-compile-cache-lib@3.0.1: - resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} - validate-npm-package-license@3.0.4: resolution: {integrity: sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==} @@ -4247,10 +4152,6 @@ packages: resolution: {integrity: sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==} engines: {node: '>=12'} - yn@3.1.1: - resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} - engines: {node: '>=6'} - yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -4465,10 +4366,6 @@ snapshots: '@bentley/imodeljs-native@5.10.35': {} - '@cspotcode/source-map-support@0.8.1': - dependencies: - '@jridgewell/trace-mapping': 0.3.9 - '@emnapi/core@1.11.1': dependencies: '@emnapi/wasi-threads': 1.2.2 @@ -4904,11 +4801,6 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 - '@jridgewell/trace-mapping@0.3.9': - dependencies: - '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.5 - '@js-sdsl/ordered-map@4.4.2': {} '@microsoft/api-extractor-model@7.33.8(@types/node@22.18.12)': @@ -5129,14 +5021,6 @@ snapshots: '@tootallnate/once@2.0.0': {} - '@tsconfig/node10@1.0.11': {} - - '@tsconfig/node12@1.0.11': {} - - '@tsconfig/node14@1.0.3': {} - - '@tsconfig/node16@1.0.4': {} - '@tybys/wasm-util@0.10.3': dependencies: tslib: 2.8.1 @@ -5152,8 +5036,6 @@ snapshots: '@types/chai@4.3.1': {} - '@types/chai@4.3.20': {} - '@types/chai@5.2.3': dependencies: '@types/deep-eql': 4.0.2 @@ -5494,10 +5376,6 @@ snapshots: dependencies: acorn: 8.15.0 - acorn-walk@8.3.4: - dependencies: - acorn: 8.15.0 - acorn@8.15.0: {} address@1.2.2: {} @@ -5561,8 +5439,6 @@ snapshots: are-docs-informative@0.0.2: {} - arg@4.1.3: {} - argparse@1.0.10: dependencies: sprintf-js: 1.0.3 @@ -5644,8 +5520,6 @@ snapshots: arrify@2.0.1: {} - assertion-error@1.1.0: {} - assertion-error@2.0.1: {} ast-types-flow@0.0.8: {} @@ -5802,16 +5676,6 @@ snapshots: canonical-path@1.0.0: {} - chai@4.5.0: - dependencies: - assertion-error: 1.1.0 - check-error: 1.0.3 - deep-eql: 4.1.4 - get-func-name: 2.0.2 - loupe: 2.3.7 - pathval: 1.1.1 - type-detect: 4.1.0 - chai@6.2.2: {} chalk@2.4.2: @@ -5832,10 +5696,6 @@ snapshots: charenc@0.0.2: {} - check-error@1.0.3: - dependencies: - get-func-name: 2.0.2 - chokidar@3.6.0: dependencies: anymatch: 3.1.3 @@ -5930,8 +5790,6 @@ snapshots: shell-quote: 1.9.0 subarg: 1.0.0 - create-require@1.1.1: {} - cross-env@5.2.1: dependencies: cross-spawn: 6.0.6 @@ -5996,10 +5854,6 @@ snapshots: dependencies: mimic-response: 3.1.0 - deep-eql@4.1.4: - dependencies: - type-detect: 4.1.0 - deep-is@0.1.4: {} default-browser-id@5.0.0: {} @@ -6040,8 +5894,6 @@ snapshots: transitivePeerDependencies: - supports-color - diff@4.0.2: {} - diff@5.2.0: {} diff@7.0.0: {} @@ -6841,8 +6693,6 @@ snapshots: get-east-asian-width@1.6.0: {} - get-func-name@2.0.2: {} - get-intrinsic@1.3.0: dependencies: call-bind-apply-helpers: 1.0.2 @@ -7587,10 +7437,6 @@ snapshots: dependencies: js-tokens: 4.0.0 - loupe@2.3.7: - dependencies: - get-func-name: 2.0.2 - lowercase-keys@3.0.0: {} lru-cache@10.4.3: {} @@ -7622,8 +7468,6 @@ snapshots: dependencies: semver: 7.8.5 - make-error@1.3.6: {} - markdown-it@14.2.0: dependencies: argparse: 2.0.1 @@ -8014,8 +7858,6 @@ snapshots: pathe@2.0.3: {} - pathval@1.1.1: {} - picocolors@1.1.1: {} picomatch@2.3.2: {} @@ -8628,24 +8470,6 @@ snapshots: dependencies: typescript: 5.9.3 - ts-node@10.9.2(@types/node@22.18.12)(typescript@5.6.3): - dependencies: - '@cspotcode/source-map-support': 0.8.1 - '@tsconfig/node10': 1.0.11 - '@tsconfig/node12': 1.0.11 - '@tsconfig/node14': 1.0.3 - '@tsconfig/node16': 1.0.4 - '@types/node': 22.18.12 - acorn: 8.15.0 - acorn-walk: 8.3.4 - arg: 4.1.3 - create-require: 1.1.1 - diff: 4.0.2 - make-error: 1.3.6 - typescript: 5.6.3 - v8-compile-cache-lib: 3.0.1 - yn: 3.1.1 - tsconfig-paths@3.15.0: dependencies: '@types/json5': 0.0.29 @@ -8659,8 +8483,6 @@ snapshots: dependencies: prelude-ls: 1.2.1 - type-detect@4.1.0: {} - type-is@1.6.18: dependencies: media-typer: 0.3.0 @@ -8759,8 +8581,6 @@ snapshots: uuid@9.0.1: {} - v8-compile-cache-lib@3.0.1: {} - validate-npm-package-license@3.0.4: dependencies: spdx-correct: 3.2.0 @@ -8794,7 +8614,7 @@ snapshots: magic-string: 0.30.21 obug: 2.1.4 pathe: 2.0.3 - picomatch: 4.0.4 + picomatch: 4.0.5 std-env: 4.2.0 tinybench: 2.9.0 tinyexec: 1.1.1 @@ -8977,6 +8797,4 @@ snapshots: y18n: 5.0.8 yargs-parser: 21.1.1 - yn@3.1.1: {} - yocto-queue@0.1.0: {} From 6d539dbd12fb188f4e328d052686dbe3ae95e503 Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Date: Wed, 29 Jul 2026 15:39:15 -0400 Subject: [PATCH 2/5] Scope environment setup to weekly tests Keep credential loading in the Hub-backed regression suite so local helper unit tests run without CI or a .env file. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- packages/performance-tests/vitest.config.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/performance-tests/vitest.config.ts b/packages/performance-tests/vitest.config.ts index 38c500a75..4c42d9f66 100644 --- a/packages/performance-tests/vitest.config.ts +++ b/packages/performance-tests/vitest.config.ts @@ -10,7 +10,6 @@ export default defineConfig({ globals: true, environment: "node", include: ["test/TransformerRegression.test.ts", "test/unit/**/*.test.ts"], - setupFiles: ["./test/setup.ts"], // Transformations and worker-owned startup/teardown may legitimately run for hours. testTimeout: 0, hookTimeout: 0, From 479dc31a426b62841264f743a3e70671aca6ba5e Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Date: Wed, 29 Jul 2026 15:44:39 -0400 Subject: [PATCH 3/5] Scope weekly performance lifecycle settings Use IModelHost validity for partial-startup cleanup and keep unlimited timeouts local to the credentialed weekly regression file. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../test/TransformerRegression.test.ts | 12 +++--------- packages/performance-tests/vitest.config.ts | 3 --- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/packages/performance-tests/test/TransformerRegression.test.ts b/packages/performance-tests/test/TransformerRegression.test.ts index 799550459..66be0c493 100644 --- a/packages/performance-tests/test/TransformerRegression.test.ts +++ b/packages/performance-tests/test/TransformerRegression.test.ts @@ -73,26 +73,21 @@ const testCasesMap = new Map>([ const loggerCategory = "Transformer Performance Regression Tests"; const outputDir = path.join(__dirname, ".output"); +vi.setConfig({ testTimeout: 0, hookTimeout: 0 }); + class WorkerLifecycle { private _authClient?: AuthorizationClient; - private _hostStarted = false; public setAuthClient(authClient: AuthorizationClient): void { this._authClient = authClient; } - public markHostStarted(): void { - this._hostStarted = true; - } - public async shutdown(): Promise { - const hostStarted = this._hostStarted; const authClient = this._authClient; - this._hostStarted = false; this._authClient = undefined; const cleanupTasks: CleanupTask[] = []; - if (hostStarted) { + if (IModelHost.isValid) { cleanupTasks.push({ name: "IModelHost shutdown", run: async () => IModelHost.shutdown(), @@ -191,7 +186,6 @@ async function setupTestData( }); hostConfig.hubAccess = new BackendIModelsAccess(hubClient); await IModelHost.startup(hostConfig); - workerLifecycle.markHostStarted(); return preFetchAsyncIterator(getTestIModels(filterIModels)); } diff --git a/packages/performance-tests/vitest.config.ts b/packages/performance-tests/vitest.config.ts index 4c42d9f66..b42dd75d1 100644 --- a/packages/performance-tests/vitest.config.ts +++ b/packages/performance-tests/vitest.config.ts @@ -10,9 +10,6 @@ export default defineConfig({ globals: true, environment: "node", include: ["test/TransformerRegression.test.ts", "test/unit/**/*.test.ts"], - // Transformations and worker-owned startup/teardown may legitimately run for hours. - testTimeout: 0, - hookTimeout: 0, pool: "forks", maxWorkers: 1, fileParallelism: false, From 96c981d20985bbc5ad7229f2a01832d6903d967d Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Date: Wed, 29 Jul 2026 15:55:45 -0400 Subject: [PATCH 4/5] Remove redundant performance test log Rely on Vitest's per-test reporting instead of emitting an unstructured completion message. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- packages/performance-tests/test/TransformerRegression.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/performance-tests/test/TransformerRegression.test.ts b/packages/performance-tests/test/TransformerRegression.test.ts index 66be0c493..32b71a720 100644 --- a/packages/performance-tests/test/TransformerRegression.test.ts +++ b/packages/performance-tests/test/TransformerRegression.test.ts @@ -306,8 +306,6 @@ describe("Transformer Regression Tests", () => { transformerModule: definition.transformerModule, addReport, }); - // eslint-disable-next-line no-console - console.log("Finished the test"); }); } }); From 33bf668915c30b7a8a871165da5cf0836aa8284b Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Date: Wed, 29 Jul 2026 16:11:11 -0400 Subject: [PATCH 5/5] Document performance test architecture Explain the weekly regression lifecycle, helper unit tests, registration matrix, cleanup, reporting contract, and extension boundaries. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- packages/performance-tests/ARCHITECTURE.md | 118 +++++++++++++++++++++ packages/performance-tests/README.md | 3 + 2 files changed, 121 insertions(+) create mode 100644 packages/performance-tests/ARCHITECTURE.md diff --git a/packages/performance-tests/ARCHITECTURE.md b/packages/performance-tests/ARCHITECTURE.md new file mode 100644 index 000000000..9b59e0b58 --- /dev/null +++ b/packages/performance-tests/ARCHITECTURE.md @@ -0,0 +1,118 @@ +# Performance Test Architecture + +This package contains the transformer's performance regression tests and the +unit tests for their supporting infrastructure. Both use Vitest, but they have +different runtime requirements. + +## Test categories + +| Category | Location | Purpose | External requirements | +| --- | --- | --- | --- | +| Weekly regression | `test/TransformerRegression.test.ts` | Measure transformer implementations against selected Hub iModels and a generated local iModel | Hub credentials, OIDC configuration, and network access | +| Infrastructure unit | `test/unit/**/*.test.ts` | Verify registration and cleanup behavior | None | + +The unit tests do not measure transformer performance. They validate code used +to construct and tear down the weekly regression suite. + +## Vitest configuration + +`vitest.config.ts` includes both categories and uses one forked worker with file +parallelism disabled. The weekly tests share process-wide resources such as +`IModelHost`, authentication, downloaded briefcases, and the CSV reporter, so +they must run serially. + +The weekly regression file opts out of test and hook timeouts because individual +transformations and downloads can run for hours. Infrastructure unit tests keep +Vitest's default timeouts so a broken unit test fails instead of hanging the +worker. + +Environment loading is scoped to `TransformerRegression.test.ts`. Unit tests can +therefore run locally without a `.env` file by running +`pnpm exec vitest run test/unit`. + +The repository's root `pnpm test` excludes this package. Run its tests explicitly +from `packages/performance-tests`. + +## Weekly regression lifecycle + +The weekly suite has two phases. + +### Collection + +Before running any tests, Vitest imports `TransformerRegression.test.ts` to +discover them. During this setup step, the module: + +1. Load environment configuration and authenticate. +2. Start `IModelHost` with Hub access. +3. Discover and filter the configured Hub iModels. +4. Add the generated local iModel. +5. Load the built-in and optional comparison transformer modules. +6. Build the supported test-case/module combinations. + +If collection fails after authentication or host startup begins, the worker +lifecycle attempts to shut down every initialized resource before preserving +and rethrowing the original error. + +### Execution + +For each selected iModel, the suite: + +1. Download or generate a local `.bim` source database and record its report + metadata. +2. Opens a fresh read-only source database for each test. +3. Runs every supported test-case/module combination. +4. Closes the source database after each test. + +The raw-insert comparison runs after the per-iModel transform cases. Once all +tests finish, the suite exports `test/.output/report.csv`, shuts down +`IModelHost`, and signs out. Cleanup tasks are all attempted even if an earlier +task fails. + +## Test registration + +`RegressionTestRegistration.ts` creates the execution matrix; it does not run +tests or store results. + +Each test case names the factory function it requires from +`TestTransformerModule`. Each loaded transformer module is paired only with the +test cases it supports. `TransformerRegression.test.ts` consumes those +definitions and registers the corresponding Vitest tests. + +The module name is the human-readable identifier used in test names and report +entries. The module object contains the implementation that the test executes. +Additional implementations can be loaded through `EXTRA_TRANSFORMERS`. + +## Inputs and authentication + +`template.env` documents the weekly suite's environment variables. The important +input controls are: + +- `ITWIN_IDS`: iTwins from which test iModels are discovered. +- `IMODEL_IDS`: specific iModels to include, or `*` for every iModel in the + configured iTwins. +- `EXTRA_TRANSFORMERS`: optional module paths for comparison implementations. +- `LOG_LEVEL`: verbosity for the iTwin logger. + +CI uses headless authentication. Local weekly runs use the CLI authorization +client and still require the OIDC and Hub configuration described in +`template.env`. Never commit a populated `.env` file. + +## Reporting + +Test cases report measurements through the callback in `TestCaseContext`. +`@itwin/perf-tools` combines those measurements with iModel and branch metadata, +then writes `test/.output/report.csv`. + +That path and CSV format are the artifact contract consumed by the hosted weekly +performance pipeline. Changes to either require coordinated pipeline validation. + +## Extending the package + +- Put credential-free tests of support code under `test/unit`. +- Add a weekly performance case under `test/cases`, declare its required + transformer factory in `TestTransformerModule`, and add it to `testCasesMap`. +- Add a built-in transformer implementation under `test/transformers`, or load a + comparison implementation with `EXTRA_TRANSFORMERS`. +- Give future performance-test categories their own entry file, setup, timeout, + and reporting policy. Do not make Hub credentials or unlimited timeouts global + merely because the weekly suite needs them. diff --git a/packages/performance-tests/README.md b/packages/performance-tests/README.md index b520154e2..b971251a1 100644 --- a/packages/performance-tests/README.md +++ b/packages/performance-tests/README.md @@ -2,6 +2,9 @@ A package containing performance tests for the [`@itwin/imodel-transformer` library](../../README.md). +See [ARCHITECTURE.md](./ARCHITECTURE.md) for the test categories, weekly suite +lifecycle, registration model, and extension guidance. + ## Tests Here are tests we need but don't have: