From 2a8942d421a90af07967c7e569be8627a08f31d8 Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Date: Tue, 14 Jul 2026 13:36:06 -0400 Subject: [PATCH 1/4] Add quick performance benchmark tests Create lightweight perf benchmarks in the performance-tests package that: - Generate synthetic iModels at runtime (no auth/hub/.env needed) - Import 15 custom ECSchemas + insert 10k PhysicalObject elements - Measure processSchemas() and process() with detailed breakdowns - Use BenchmarkTransformer/BenchmarkImporter subclasses to capture per-method timing (export elements, import elements, per-schema cost) - Include ECReferenceTypesCache initialization benchmark - Output structured results to console Run with: pnpm test:quick-perf (from packages/performance-tests) Closes #328 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- packages/performance-tests/package.json | 1 + .../test/QuickBenchmark.test.ts | 226 ++++++++++++++++++ .../test/benchmarking/BenchmarkImporter.ts | 47 ++++ .../test/benchmarking/BenchmarkStats.ts | 117 +++++++++ .../test/benchmarking/BenchmarkTransformer.ts | 82 +++++++ .../test/benchmarking/index.ts | 12 + 6 files changed, 485 insertions(+) create mode 100644 packages/performance-tests/test/QuickBenchmark.test.ts create mode 100644 packages/performance-tests/test/benchmarking/BenchmarkImporter.ts create mode 100644 packages/performance-tests/test/benchmarking/BenchmarkStats.ts create mode 100644 packages/performance-tests/test/benchmarking/BenchmarkTransformer.ts create mode 100644 packages/performance-tests/test/benchmarking/index.ts diff --git a/packages/performance-tests/package.json b/packages/performance-tests/package.json index 09a07f57e..259fc62b8 100644 --- a/packages/performance-tests/package.json +++ b/packages/performance-tests/package.json @@ -8,6 +8,7 @@ "clean": "rimraf lib", "lint": "eslint \"./test/**/*.ts\" 1>&2", "test": "mocha --delay --timeout 300000 --require ts-node/register test/**/*.test.ts", + "test:quick-perf": "mocha --timeout 120000 --require ts-node/register test/QuickBenchmark.test.ts", "format": "prettier \"./test/**/*.ts\" --write", "test-mocha": "mocha --delay --timeout 300000 \"./lib/**/TransformerRegression.test.js\"", "process-reports": "node scripts/process-reports" diff --git a/packages/performance-tests/test/QuickBenchmark.test.ts b/packages/performance-tests/test/QuickBenchmark.test.ts new file mode 100644 index 000000000..b008d0450 --- /dev/null +++ b/packages/performance-tests/test/QuickBenchmark.test.ts @@ -0,0 +1,226 @@ +/* eslint-disable no-console */ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +/** + * Quick performance benchmarks for imodel-transformer. + * + * These tests generate a synthetic iModel at runtime (no auth, no hub, no .env required) + * with multiple custom schemas and 10k physical elements, then run a full identity + * transformation while measuring schema processing and element processing times. + * + * Run with: pnpm test:quick-perf + */ + +import { + EditTxn, + IModelDb, + IModelHost, + IModelHostOptions, + PhysicalModel, + PhysicalObject, + SnapshotDb, + SpatialCategory, + StandaloneDb, + withEditTxn, +} from "@itwin/core-backend"; +import { + Code, + ColorDef, + IModel, + PhysicalElementProps, +} from "@itwin/core-common"; +import { Point3d, YawPitchRollAngles } from "@itwin/core-geometry"; +import { Logger, LogLevel } from "@itwin/core-bentley"; +import { IModelTransformerTestUtils } from "@itwin/imodel-transformer/lib/cjs/test/IModelTransformerUtils"; +import { ECReferenceTypesCache } from "@itwin/imodel-transformer/lib/cjs/ECReferenceTypesCache"; +import { + BenchmarkImporter, + BenchmarkTransformer, + createEmptyStats, + printBenchmarkStats, +} from "./benchmarking"; +import * as path from "path"; +import * as fs from "fs"; + +const NUM_ELEMENTS = 10_000; +const NUM_CUSTOM_SCHEMAS = 15; +const outputDir = path.join(__dirname, ".output"); + +function ensureOutputDir(): void { + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } +} + +function initOutputFile(filename: string): string { + ensureOutputDir(); + const filePath = path.join(outputDir, filename); + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + } + return filePath; +} + +/** + * Generate synthetic ECSchema XML strings that reference BisCore. + * Each schema has a couple of entity classes and properties to give + * processSchemas() meaningful work. + */ +function generateSchemaStrings(count: number): string[] { + return Array.from({ length: count }, (_, i) => { + const schemaName = `PerfTestDomain${i}`; + const alias = `ptd${i}`; + return ` + + + + + bis:PhysicalElement + + + + + + bis:InformationRecordElement + + + +`; + }); +} + +/** + * Create a source iModel with custom schemas and elements. + */ +async function createSourceIModel(): Promise { + const sourceFileName = initOutputFile("quick_perf_source.bim"); + const sourceDb = StandaloneDb.createEmpty(sourceFileName, { + rootSubject: { name: "QuickPerfBenchmark Source" }, + }); + + // Import custom schemas + const schemas = generateSchemaStrings(NUM_CUSTOM_SCHEMAS); + await sourceDb.importSchemaStrings(schemas); + + // Insert elements + const geom = IModelTransformerTestUtils.createBox(Point3d.create(1, 1, 1)); + withEditTxn(sourceDb, "insert benchmark elements", (txn) => { + const categoryId = SpatialCategory.insert( + txn, + IModel.dictionaryId, + "BenchmarkCategory", + { color: ColorDef.blue.toJSON() } + ); + const modelId = PhysicalModel.insert( + txn, + IModel.rootSubjectId, + "BenchmarkPhysicalModel" + ); + + for (let i = 0; i < NUM_ELEMENTS; i++) { + const elementProps: PhysicalElementProps = { + classFullName: PhysicalObject.classFullName, + model: modelId, + category: categoryId, + code: Code.createEmpty(), + userLabel: `BenchElem_${i}`, + geom, + placement: { + origin: new Point3d(i % 100, Math.floor(i / 100), 0), + angles: YawPitchRollAngles.createDegrees(0, 0, 0), + }, + }; + txn.insertElement(elementProps); + } + }); + + return sourceDb; +} + +before(async () => { + Logger.initializeToConsole(); + Logger.setLevelDefault(LogLevel.Error); + const cfg: IModelHostOptions = {}; + cfg.cacheDir = path.join(__dirname, ".cache"); + await IModelHost.startup(cfg); +}); + +after(async () => { + await IModelHost.shutdown(); +}); + +describe("Quick Performance Benchmarks", function () { + this.timeout(120_000); + + it("should benchmark identity transform (10k elements, 15+ schemas)", async () => { + // Generate source iModel + console.log( + "Generating source iModel with %d elements and %d custom schemas...", + NUM_ELEMENTS, + NUM_CUSTOM_SCHEMAS + ); + const sourceDb = await createSourceIModel(); + console.log("Source iModel created: %s", sourceDb.pathName); + + // Create empty target + const targetFileName = initOutputFile("quick_perf_target.bim"); + const targetDb = SnapshotDb.createEmpty(targetFileName, { + rootSubject: { name: "QuickPerfBenchmark Target" }, + }); + + // Set up benchmarked transformer with a BenchmarkImporter + const stats = createEmptyStats(); + const editTxn = new EditTxn(targetDb, "BenchmarkTransformer"); + editTxn.start(); + + const benchmarkImporter = new BenchmarkImporter(editTxn, stats); + const transformer = new BenchmarkTransformer( + { source: sourceDb, target: benchmarkImporter }, + { loadSourceGeometry: true, noProvenance: true } + ); + + // Run the full transformation + console.log("Running processSchemas()..."); + await transformer.processSchemas(); + + console.log("Running process()..."); + await transformer.process(); + + editTxn.end(); + + // Print results + printBenchmarkStats(stats); + + // Cleanup + transformer.dispose(); + sourceDb.close(); + targetDb.close(); + }); + + it("should benchmark ECReferenceTypesCache initialization", async () => { + // Generate a source iModel with schemas for cache init benchmarking + console.log("Benchmarking ECReferenceTypesCache.initAllSchemasInIModel..."); + const sourceFileName = initOutputFile("quick_perf_cache_source.bim"); + const sourceDb = StandaloneDb.createEmpty(sourceFileName, { + rootSubject: { name: "CacheBenchmark Source" }, + }); + const schemas = generateSchemaStrings(NUM_CUSTOM_SCHEMAS); + await sourceDb.importSchemaStrings(schemas); + + const cache = new ECReferenceTypesCache(); + const start = performance.now(); + await cache.initAllSchemasInIModel(sourceDb); + const elapsed = performance.now() - start; + + console.log("\n ECReferenceTypesCache Init"); + console.log(" ─────────────────────────────────────"); + console.log(" Total init time: %s ms", elapsed.toFixed(2)); + console.log(" ─────────────────────────────────────\n"); + + sourceDb.close(); + }); +}); diff --git a/packages/performance-tests/test/benchmarking/BenchmarkImporter.ts b/packages/performance-tests/test/benchmarking/BenchmarkImporter.ts new file mode 100644 index 000000000..ec252d25f --- /dev/null +++ b/packages/performance-tests/test/benchmarking/BenchmarkImporter.ts @@ -0,0 +1,47 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { EditTxn, RelationshipProps } from "@itwin/core-backend"; +import { ElementProps } from "@itwin/core-common"; +import { Id64String } from "@itwin/core-bentley"; +import { IModelImporter, IModelImportOptions } from "@itwin/imodel-transformer"; +import { BenchmarkStats } from "./BenchmarkStats"; + +/** + * An IModelImporter subclass that captures timing data for import operations. + * Measures time spent inserting elements and relationships into the target iModel. + */ +export class BenchmarkImporter extends IModelImporter { + private readonly _stats: BenchmarkStats; + + public constructor( + editTxn: EditTxn, + stats: BenchmarkStats, + options?: IModelImportOptions + ) { + super(editTxn, options); + this._stats = stats; + } + + protected override async onInsertElement( + elementProps: ElementProps + ): Promise { + const start = performance.now(); + const result = await super.onInsertElement(elementProps); + this._stats.importInsertElementMs += performance.now() - start; + this._stats.importInsertElementCount++; + return result; + } + + protected override async onInsertRelationship( + relationshipProps: RelationshipProps + ): Promise { + const start = performance.now(); + const result = await super.onInsertRelationship(relationshipProps); + this._stats.importInsertRelationshipMs += performance.now() - start; + this._stats.importInsertRelationshipCount++; + return result; + } +} diff --git a/packages/performance-tests/test/benchmarking/BenchmarkStats.ts b/packages/performance-tests/test/benchmarking/BenchmarkStats.ts new file mode 100644 index 000000000..e676ab75b --- /dev/null +++ b/packages/performance-tests/test/benchmarking/BenchmarkStats.ts @@ -0,0 +1,117 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +/** Collected performance statistics from a benchmarked transformation run. */ +export interface BenchmarkStats { + /** Total wall-clock time for processSchemas() in ms */ + schemaTotalMs: number; + /** Number of schemas exported during processSchemas() */ + schemaCount: number; + /** Per-schema export times in ms (indexed by schema name) */ + schemaExportTimes: Map; + /** Total wall-clock time for process() in ms */ + processTotalMs: number; + /** Time spent in onExportElement calls in ms */ + exportElementMs: number; + /** Number of elements exported */ + exportElementCount: number; + /** Time spent in onExportRelationship calls in ms */ + exportRelationshipMs: number; + /** Number of relationships exported */ + exportRelationshipCount: number; + /** Time spent in onInsertElement calls in ms */ + importInsertElementMs: number; + /** Number of elements inserted */ + importInsertElementCount: number; + /** Time spent in onInsertRelationship calls in ms */ + importInsertRelationshipMs: number; + /** Number of relationships inserted */ + importInsertRelationshipCount: number; +} + +export function createEmptyStats(): BenchmarkStats { + return { + schemaTotalMs: 0, + schemaCount: 0, + schemaExportTimes: new Map(), + processTotalMs: 0, + exportElementMs: 0, + exportElementCount: 0, + exportRelationshipMs: 0, + exportRelationshipCount: 0, + importInsertElementMs: 0, + importInsertElementCount: 0, + importInsertRelationshipMs: 0, + importInsertRelationshipCount: 0, + }; +} + +export function printBenchmarkStats(stats: BenchmarkStats): void { + const separator = "═".repeat(60); + const line = "─".repeat(60); + + // eslint-disable-next-line no-console + const log = console.log.bind(console); + + log(`\n╔${separator}╗`); + log(`║ Quick Perf Benchmark Results${" ".repeat(30)}║`); + log(`╠${separator}╣`); + + log(`║ Schema Processing${" ".repeat(41)}║`); + log(`║${line}║`); + log( + `║ Total time: ${stats.schemaTotalMs.toFixed(2).padStart(10)} ms${" ".repeat(21)}║` + ); + log( + `║ Schemas exported: ${String(stats.schemaCount).padStart(10)}${" ".repeat(24)}║` + ); + if (stats.schemaCount > 0) { + const avgPerSchema = stats.schemaTotalMs / stats.schemaCount; + log( + `║ Avg per schema: ${avgPerSchema.toFixed(2).padStart(10)} ms${" ".repeat(21)}║` + ); + } + log(`║${line}║`); + + log(`║ Element/Relationship Processing${" ".repeat(27)}║`); + log(`║${line}║`); + log( + `║ process() total: ${stats.processTotalMs.toFixed(2).padStart(10)} ms${" ".repeat(21)}║` + ); + log( + `║ Export elements: ${stats.exportElementMs.toFixed(2).padStart(10)} ms (${stats.exportElementCount} items)${" ".repeat(Math.max(0, 10 - String(stats.exportElementCount).length))}║` + ); + log( + `║ Export relationships:${stats.exportRelationshipMs.toFixed(2).padStart(10)} ms (${stats.exportRelationshipCount} items)${" ".repeat(Math.max(0, 10 - String(stats.exportRelationshipCount).length))}║` + ); + log( + `║ Import elements: ${stats.importInsertElementMs.toFixed(2).padStart(10)} ms (${stats.importInsertElementCount} items)${" ".repeat(Math.max(0, 10 - String(stats.importInsertElementCount).length))}║` + ); + log( + `║ Import relationships:${stats.importInsertRelationshipMs.toFixed(2).padStart(10)} ms (${stats.importInsertRelationshipCount} items)${" ".repeat(Math.max(0, 10 - String(stats.importInsertRelationshipCount).length))}║` + ); + if (stats.exportElementCount > 0) { + const avgPerElement = stats.processTotalMs / stats.exportElementCount; + log(`║${line}║`); + log( + `║ Avg per element: ${avgPerElement.toFixed(4).padStart(10)} ms${" ".repeat(21)}║` + ); + } + + log(`╚${separator}╝\n`); + + // Print per-schema breakdown if available + if (stats.schemaExportTimes.size > 0) { + log(` Per-schema export times:`); + log(` ${"─".repeat(50)}`); + const sorted = [...stats.schemaExportTimes.entries()].sort( + (a, b) => b[1] - a[1] + ); + for (const [name, ms] of sorted) { + log(` ${name.padEnd(35)} ${ms.toFixed(2).padStart(8)} ms`); + } + log(` ${"─".repeat(50)}\n`); + } +} diff --git a/packages/performance-tests/test/benchmarking/BenchmarkTransformer.ts b/packages/performance-tests/test/benchmarking/BenchmarkTransformer.ts new file mode 100644 index 000000000..f2d00eb74 --- /dev/null +++ b/packages/performance-tests/test/benchmarking/BenchmarkTransformer.ts @@ -0,0 +1,82 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { Element, Relationship } from "@itwin/core-backend"; +import { + IModelTransformer, + IModelTransformArgs, + IModelTransformOptions, + ExportSchemaResult, +} from "@itwin/imodel-transformer"; +import { Schema } from "@itwin/ecschema-metadata"; +import { BenchmarkStats, createEmptyStats } from "./BenchmarkStats"; +import { BenchmarkImporter } from "./BenchmarkImporter"; + +/** + * An IModelTransformer subclass that captures detailed timing data for all + * phases of a transformation: schema export, element export, and the overall + * processSchemas/process calls. + * + * Uses a BenchmarkImporter for capturing import-side timing. + */ +export class BenchmarkTransformer extends IModelTransformer { + private readonly _stats: BenchmarkStats; + + public get stats(): BenchmarkStats { + return this._stats; + } + + public constructor( + args: IModelTransformArgs, + options?: IModelTransformOptions + ) { + super(args, options); + // If a BenchmarkImporter was passed, reuse its stats object + if (args.target instanceof BenchmarkImporter) { + this._stats = (args.target as any)._stats; + } else { + this._stats = createEmptyStats(); + } + } + + public override async processSchemas(): Promise { + const start = performance.now(); + await super.processSchemas(); + this._stats.schemaTotalMs = performance.now() - start; + } + + public override async onExportSchema( + schema: Schema + ): Promise { + const start = performance.now(); + const result = await super.onExportSchema(schema); + const elapsed = performance.now() - start; + this._stats.schemaExportTimes.set(schema.name, elapsed); + this._stats.schemaCount++; + return result; + } + + public override async process(): Promise { + const start = performance.now(); + await super.process(); + this._stats.processTotalMs = performance.now() - start; + } + + public override async onExportElement(sourceElement: Element): Promise { + const start = performance.now(); + await super.onExportElement(sourceElement); + this._stats.exportElementMs += performance.now() - start; + this._stats.exportElementCount++; + } + + public override async onExportRelationship( + sourceRelationship: Relationship + ): Promise { + const start = performance.now(); + await super.onExportRelationship(sourceRelationship); + this._stats.exportRelationshipMs += performance.now() - start; + this._stats.exportRelationshipCount++; + } +} diff --git a/packages/performance-tests/test/benchmarking/index.ts b/packages/performance-tests/test/benchmarking/index.ts new file mode 100644 index 000000000..662dae51f --- /dev/null +++ b/packages/performance-tests/test/benchmarking/index.ts @@ -0,0 +1,12 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +export { BenchmarkImporter } from "./BenchmarkImporter"; +export { BenchmarkTransformer } from "./BenchmarkTransformer"; +export { + BenchmarkStats, + createEmptyStats, + printBenchmarkStats, +} from "./BenchmarkStats"; From 8c1a10b694dea1b4ca9d8bb1ce07bb0e12433178 Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Date: Tue, 14 Jul 2026 13:45:34 -0400 Subject: [PATCH 2/4] Add .cache to performance-tests .gitignore IModelHost.startup() generates .bcv and .itwindb files in the cache directory during test runs. These should not be tracked. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- packages/performance-tests/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/performance-tests/.gitignore b/packages/performance-tests/.gitignore index 7600a8a06..6ec115e78 100644 --- a/packages/performance-tests/.gitignore +++ b/packages/performance-tests/.gitignore @@ -1,2 +1,3 @@ .env **/.output +**/.cache From 1a57a201502e4ad89c88c9dc74d8fa8bfb93593d Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Date: Tue, 14 Jul 2026 14:00:55 -0400 Subject: [PATCH 3/4] Simplify benchmark instrumentation to phase-level timing only Remove per-call performance.now() from element/relationship/schema overrides to eliminate measurement overhead. Keep only: - processSchemas() and process() wall-clock timing - Element/relationship/schema counts - Avg per element derived from process() / element count Remove BenchmarkImporter (no longer needed without per-call timing). This brings process() times within ~7% of TransformerPerf.test.ts. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../test/QuickBenchmark.test.ts | 15 +--- .../test/benchmarking/BenchmarkImporter.ts | 47 ---------- .../test/benchmarking/BenchmarkStats.ts | 90 +++---------------- .../test/benchmarking/BenchmarkTransformer.ts | 30 ++----- .../test/benchmarking/index.ts | 1 - 5 files changed, 20 insertions(+), 163 deletions(-) delete mode 100644 packages/performance-tests/test/benchmarking/BenchmarkImporter.ts diff --git a/packages/performance-tests/test/QuickBenchmark.test.ts b/packages/performance-tests/test/QuickBenchmark.test.ts index b008d0450..eb577d5dc 100644 --- a/packages/performance-tests/test/QuickBenchmark.test.ts +++ b/packages/performance-tests/test/QuickBenchmark.test.ts @@ -36,12 +36,7 @@ import { Point3d, YawPitchRollAngles } from "@itwin/core-geometry"; import { Logger, LogLevel } from "@itwin/core-bentley"; import { IModelTransformerTestUtils } from "@itwin/imodel-transformer/lib/cjs/test/IModelTransformerUtils"; import { ECReferenceTypesCache } from "@itwin/imodel-transformer/lib/cjs/ECReferenceTypesCache"; -import { - BenchmarkImporter, - BenchmarkTransformer, - createEmptyStats, - printBenchmarkStats, -} from "./benchmarking"; +import { BenchmarkTransformer, printBenchmarkStats } from "./benchmarking"; import * as path from "path"; import * as fs from "fs"; @@ -172,14 +167,12 @@ describe("Quick Performance Benchmarks", function () { rootSubject: { name: "QuickPerfBenchmark Target" }, }); - // Set up benchmarked transformer with a BenchmarkImporter - const stats = createEmptyStats(); + // Set up benchmarked transformer const editTxn = new EditTxn(targetDb, "BenchmarkTransformer"); editTxn.start(); - const benchmarkImporter = new BenchmarkImporter(editTxn, stats); const transformer = new BenchmarkTransformer( - { source: sourceDb, target: benchmarkImporter }, + { source: sourceDb, target: editTxn }, { loadSourceGeometry: true, noProvenance: true } ); @@ -193,7 +186,7 @@ describe("Quick Performance Benchmarks", function () { editTxn.end(); // Print results - printBenchmarkStats(stats); + printBenchmarkStats(transformer.stats); // Cleanup transformer.dispose(); diff --git a/packages/performance-tests/test/benchmarking/BenchmarkImporter.ts b/packages/performance-tests/test/benchmarking/BenchmarkImporter.ts deleted file mode 100644 index ec252d25f..000000000 --- a/packages/performance-tests/test/benchmarking/BenchmarkImporter.ts +++ /dev/null @@ -1,47 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Bentley Systems, Incorporated. All rights reserved. - * See LICENSE.md in the project root for license terms and full copyright notice. - *--------------------------------------------------------------------------------------------*/ - -import { EditTxn, RelationshipProps } from "@itwin/core-backend"; -import { ElementProps } from "@itwin/core-common"; -import { Id64String } from "@itwin/core-bentley"; -import { IModelImporter, IModelImportOptions } from "@itwin/imodel-transformer"; -import { BenchmarkStats } from "./BenchmarkStats"; - -/** - * An IModelImporter subclass that captures timing data for import operations. - * Measures time spent inserting elements and relationships into the target iModel. - */ -export class BenchmarkImporter extends IModelImporter { - private readonly _stats: BenchmarkStats; - - public constructor( - editTxn: EditTxn, - stats: BenchmarkStats, - options?: IModelImportOptions - ) { - super(editTxn, options); - this._stats = stats; - } - - protected override async onInsertElement( - elementProps: ElementProps - ): Promise { - const start = performance.now(); - const result = await super.onInsertElement(elementProps); - this._stats.importInsertElementMs += performance.now() - start; - this._stats.importInsertElementCount++; - return result; - } - - protected override async onInsertRelationship( - relationshipProps: RelationshipProps - ): Promise { - const start = performance.now(); - const result = await super.onInsertRelationship(relationshipProps); - this._stats.importInsertRelationshipMs += performance.now() - start; - this._stats.importInsertRelationshipCount++; - return result; - } -} diff --git a/packages/performance-tests/test/benchmarking/BenchmarkStats.ts b/packages/performance-tests/test/benchmarking/BenchmarkStats.ts index e676ab75b..dca32bdd8 100644 --- a/packages/performance-tests/test/benchmarking/BenchmarkStats.ts +++ b/packages/performance-tests/test/benchmarking/BenchmarkStats.ts @@ -9,109 +9,41 @@ export interface BenchmarkStats { schemaTotalMs: number; /** Number of schemas exported during processSchemas() */ schemaCount: number; - /** Per-schema export times in ms (indexed by schema name) */ - schemaExportTimes: Map; /** Total wall-clock time for process() in ms */ processTotalMs: number; - /** Time spent in onExportElement calls in ms */ - exportElementMs: number; /** Number of elements exported */ exportElementCount: number; - /** Time spent in onExportRelationship calls in ms */ - exportRelationshipMs: number; /** Number of relationships exported */ exportRelationshipCount: number; - /** Time spent in onInsertElement calls in ms */ - importInsertElementMs: number; - /** Number of elements inserted */ - importInsertElementCount: number; - /** Time spent in onInsertRelationship calls in ms */ - importInsertRelationshipMs: number; - /** Number of relationships inserted */ - importInsertRelationshipCount: number; } export function createEmptyStats(): BenchmarkStats { return { schemaTotalMs: 0, schemaCount: 0, - schemaExportTimes: new Map(), processTotalMs: 0, - exportElementMs: 0, exportElementCount: 0, - exportRelationshipMs: 0, exportRelationshipCount: 0, - importInsertElementMs: 0, - importInsertElementCount: 0, - importInsertRelationshipMs: 0, - importInsertRelationshipCount: 0, }; } export function printBenchmarkStats(stats: BenchmarkStats): void { - const separator = "═".repeat(60); - const line = "─".repeat(60); + const line = "─".repeat(50); // eslint-disable-next-line no-console const log = console.log.bind(console); - log(`\n╔${separator}╗`); - log(`║ Quick Perf Benchmark Results${" ".repeat(30)}║`); - log(`╠${separator}╣`); - - log(`║ Schema Processing${" ".repeat(41)}║`); - log(`║${line}║`); - log( - `║ Total time: ${stats.schemaTotalMs.toFixed(2).padStart(10)} ms${" ".repeat(21)}║` - ); - log( - `║ Schemas exported: ${String(stats.schemaCount).padStart(10)}${" ".repeat(24)}║` - ); - if (stats.schemaCount > 0) { - const avgPerSchema = stats.schemaTotalMs / stats.schemaCount; - log( - `║ Avg per schema: ${avgPerSchema.toFixed(2).padStart(10)} ms${" ".repeat(21)}║` - ); - } - log(`║${line}║`); - - log(`║ Element/Relationship Processing${" ".repeat(27)}║`); - log(`║${line}║`); - log( - `║ process() total: ${stats.processTotalMs.toFixed(2).padStart(10)} ms${" ".repeat(21)}║` - ); - log( - `║ Export elements: ${stats.exportElementMs.toFixed(2).padStart(10)} ms (${stats.exportElementCount} items)${" ".repeat(Math.max(0, 10 - String(stats.exportElementCount).length))}║` - ); - log( - `║ Export relationships:${stats.exportRelationshipMs.toFixed(2).padStart(10)} ms (${stats.exportRelationshipCount} items)${" ".repeat(Math.max(0, 10 - String(stats.exportRelationshipCount).length))}║` - ); - log( - `║ Import elements: ${stats.importInsertElementMs.toFixed(2).padStart(10)} ms (${stats.importInsertElementCount} items)${" ".repeat(Math.max(0, 10 - String(stats.importInsertElementCount).length))}║` - ); - log( - `║ Import relationships:${stats.importInsertRelationshipMs.toFixed(2).padStart(10)} ms (${stats.importInsertRelationshipCount} items)${" ".repeat(Math.max(0, 10 - String(stats.importInsertRelationshipCount).length))}║` - ); + log(`\n Quick Perf Benchmark Results`); + log(` ${line}`); + log(` Schemas exported: ${stats.schemaCount}`); + log(` processSchemas(): ${stats.schemaTotalMs.toFixed(2)} ms`); + log(` ${line}`); + log(` Elements exported: ${stats.exportElementCount}`); + log(` Relationships exported: ${stats.exportRelationshipCount}`); + log(` process(): ${stats.processTotalMs.toFixed(2)} ms`); if (stats.exportElementCount > 0) { const avgPerElement = stats.processTotalMs / stats.exportElementCount; - log(`║${line}║`); - log( - `║ Avg per element: ${avgPerElement.toFixed(4).padStart(10)} ms${" ".repeat(21)}║` - ); - } - - log(`╚${separator}╝\n`); - - // Print per-schema breakdown if available - if (stats.schemaExportTimes.size > 0) { - log(` Per-schema export times:`); - log(` ${"─".repeat(50)}`); - const sorted = [...stats.schemaExportTimes.entries()].sort( - (a, b) => b[1] - a[1] - ); - for (const [name, ms] of sorted) { - log(` ${name.padEnd(35)} ${ms.toFixed(2).padStart(8)} ms`); - } - log(` ${"─".repeat(50)}\n`); + log(` Avg per element: ${avgPerElement.toFixed(4)} ms`); } + log(` ${line}\n`); } diff --git a/packages/performance-tests/test/benchmarking/BenchmarkTransformer.ts b/packages/performance-tests/test/benchmarking/BenchmarkTransformer.ts index f2d00eb74..8643e4916 100644 --- a/packages/performance-tests/test/benchmarking/BenchmarkTransformer.ts +++ b/packages/performance-tests/test/benchmarking/BenchmarkTransformer.ts @@ -8,18 +8,13 @@ import { IModelTransformer, IModelTransformArgs, IModelTransformOptions, - ExportSchemaResult, } from "@itwin/imodel-transformer"; import { Schema } from "@itwin/ecschema-metadata"; import { BenchmarkStats, createEmptyStats } from "./BenchmarkStats"; -import { BenchmarkImporter } from "./BenchmarkImporter"; /** - * An IModelTransformer subclass that captures detailed timing data for all - * phases of a transformation: schema export, element export, and the overall - * processSchemas/process calls. - * - * Uses a BenchmarkImporter for capturing import-side timing. + * An IModelTransformer subclass that captures timing data for the major + * transformation phases (processSchemas, process) and counts of exported entities. */ export class BenchmarkTransformer extends IModelTransformer { private readonly _stats: BenchmarkStats; @@ -33,12 +28,7 @@ export class BenchmarkTransformer extends IModelTransformer { options?: IModelTransformOptions ) { super(args, options); - // If a BenchmarkImporter was passed, reuse its stats object - if (args.target instanceof BenchmarkImporter) { - this._stats = (args.target as any)._stats; - } else { - this._stats = createEmptyStats(); - } + this._stats = createEmptyStats(); } public override async processSchemas(): Promise { @@ -47,15 +37,9 @@ export class BenchmarkTransformer extends IModelTransformer { this._stats.schemaTotalMs = performance.now() - start; } - public override async onExportSchema( - schema: Schema - ): Promise { - const start = performance.now(); - const result = await super.onExportSchema(schema); - const elapsed = performance.now() - start; - this._stats.schemaExportTimes.set(schema.name, elapsed); + public override async onExportSchema(schema: Schema): Promise { + await super.onExportSchema(schema); this._stats.schemaCount++; - return result; } public override async process(): Promise { @@ -65,18 +49,14 @@ export class BenchmarkTransformer extends IModelTransformer { } public override async onExportElement(sourceElement: Element): Promise { - const start = performance.now(); await super.onExportElement(sourceElement); - this._stats.exportElementMs += performance.now() - start; this._stats.exportElementCount++; } public override async onExportRelationship( sourceRelationship: Relationship ): Promise { - const start = performance.now(); await super.onExportRelationship(sourceRelationship); - this._stats.exportRelationshipMs += performance.now() - start; this._stats.exportRelationshipCount++; } } diff --git a/packages/performance-tests/test/benchmarking/index.ts b/packages/performance-tests/test/benchmarking/index.ts index 662dae51f..2f362462d 100644 --- a/packages/performance-tests/test/benchmarking/index.ts +++ b/packages/performance-tests/test/benchmarking/index.ts @@ -3,7 +3,6 @@ * See LICENSE.md in the project root for license terms and full copyright notice. *--------------------------------------------------------------------------------------------*/ -export { BenchmarkImporter } from "./BenchmarkImporter"; export { BenchmarkTransformer } from "./BenchmarkTransformer"; export { BenchmarkStats, From 12ae565a4ce13a7fde2cd7f1dad5cd0b8f24340c Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Date: Tue, 14 Jul 2026 14:45:35 -0400 Subject: [PATCH 4/4] Add VS Code debug config for quick perf benchmarks Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .vscode/launch.json | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 1d5c91ce3..4b5139cdf 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -42,6 +42,21 @@ "run", "test" ], - } + }, + { + "type": "node", + "request": "launch", + "name": "Quick Perf Benchmarks", + "program": "${workspaceFolder}/node_modules/.pnpm/mocha@10.8.2/node_modules/mocha/bin/mocha.js", + "args": [ + "--timeout", "120000", + "--require", "ts-node/register", + "test/QuickBenchmark.test.ts" + ], + "cwd": "${workspaceFolder}/packages/performance-tests/", + "skipFiles": [ + "/**" + ] + } ] } \ No newline at end of file