From 35b149378eb27240f9e76b3f511785b5a2bb28af Mon Sep 17 00:00:00 2001 From: Nam Le <50554904+hl662@users.noreply.github.com> Date: Fri, 24 Jul 2026 12:10:35 -0400 Subject: [PATCH 01/19] perf(tests): add quick incremental benchmark The three-hour weekly suite cannot provide timely feedback while a transformer change is under review. Reconstruct a deterministic local fixture and measure only incremental synchronization so developers can collect comparable results in minutes. Refs #328 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/quick-performance.yml | 70 ++++ packages/performance-tests/.gitignore | 3 + packages/performance-tests/README.md | 45 ++- packages/performance-tests/package.json | 3 + .../test/quick/BenchmarkReporter.ts | 118 ++++++ .../test/quick/BenchmarkRunner.ts | 214 ++++++++++ .../test/quick/DatasetDescriptor.ts | 45 +++ .../test/quick/FixtureCatalog.ts | 98 +++++ .../test/quick/FixtureManifest.quick-unit.ts | 25 ++ .../test/quick/FixtureManifest.ts | 46 +++ .../test/quick/FixtureMaterializer.ts | 89 +++++ .../test/quick/LocalHubFixture.quick-unit.ts | 294 ++++++++++++++ .../test/quick/LocalHubFixture.ts | 190 +++++++++ .../test/quick/QuickPerformance.quick.ts | 41 ++ packages/performance-tests/test/quick/cli.ts | 49 +++ .../test/quick/recipes/balancedIncremental.ts | 332 ++++++++++++++++ .../scenarios/incrementalSynchronization.ts | 55 +++ .../test/quick/schemas/QuickPerf.ecschema.xml | 17 + .../test/quick/statistics.quick-unit.ts | 28 ++ .../test/quick/validation/statistics.ts | 38 ++ .../test/quick/validation/validateFixture.ts | 366 ++++++++++++++++++ 21 files changed, 2165 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/quick-performance.yml create mode 100644 packages/performance-tests/test/quick/BenchmarkReporter.ts create mode 100644 packages/performance-tests/test/quick/BenchmarkRunner.ts create mode 100644 packages/performance-tests/test/quick/DatasetDescriptor.ts create mode 100644 packages/performance-tests/test/quick/FixtureCatalog.ts create mode 100644 packages/performance-tests/test/quick/FixtureManifest.quick-unit.ts create mode 100644 packages/performance-tests/test/quick/FixtureManifest.ts create mode 100644 packages/performance-tests/test/quick/FixtureMaterializer.ts create mode 100644 packages/performance-tests/test/quick/LocalHubFixture.quick-unit.ts create mode 100644 packages/performance-tests/test/quick/LocalHubFixture.ts create mode 100644 packages/performance-tests/test/quick/QuickPerformance.quick.ts create mode 100644 packages/performance-tests/test/quick/cli.ts create mode 100644 packages/performance-tests/test/quick/recipes/balancedIncremental.ts create mode 100644 packages/performance-tests/test/quick/scenarios/incrementalSynchronization.ts create mode 100644 packages/performance-tests/test/quick/schemas/QuickPerf.ecschema.xml create mode 100644 packages/performance-tests/test/quick/statistics.quick-unit.ts create mode 100644 packages/performance-tests/test/quick/validation/statistics.ts create mode 100644 packages/performance-tests/test/quick/validation/validateFixture.ts diff --git a/.github/workflows/quick-performance.yml b/.github/workflows/quick-performance.yml new file mode 100644 index 000000000..b7d376dec --- /dev/null +++ b/.github/workflows/quick-performance.yml @@ -0,0 +1,70 @@ +name: Quick performance + +on: + workflow_dispatch: + +jobs: + balanced-incremental: + name: Balanced incremental + runs-on: windows-latest + timeout-minutes: 30 + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + + - name: Install pnpm + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 + + - name: Use Node.js 24 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version: 24.18.0 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build transformer + run: pnpm --dir packages/transformer build:cjs + + - name: Validate quick harness + run: >- + pnpm --dir packages/performance-tests exec mocha + --timeout 300000 + --require ts-node/register + test/quick/FixtureManifest.quick-unit.ts + test/quick/statistics.quick-unit.ts + test/quick/LocalHubFixture.quick-unit.ts + + - name: Run quick performance suite + run: pnpm --dir packages/performance-tests test:quick + env: + QUICK_PERF_OUTPUT: ${{ runner.temp }}/quick-performance + + - name: Report variance reliability + if: always() + shell: pwsh + run: | + $report = "${{ runner.temp }}/quick-performance/summary.json" + if (Test-Path $report) { + $summary = Get-Content $report | ConvertFrom-Json + "### Quick performance reliability" >> $env:GITHUB_STEP_SUMMARY + "Status: **$($summary.varianceStatus)**" >> $env:GITHUB_STEP_SUMMARY + "CV: $($summary.wallMilliseconds.coefficientOfVariation)" >> $env:GITHUB_STEP_SUMMARY + "Normalized MAD: $($summary.wallMilliseconds.normalizedMad)" >> $env:GITHUB_STEP_SUMMARY + "Unstable samples: $($summary.unstableSamples -join ', ')" >> $env:GITHUB_STEP_SUMMARY + if ($summary.varianceStatus -ne "stable") { + Write-Output "::warning title=Unstable quick performance variance::Do not use this run as regression evidence." + } + } + + - name: Publish report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: QuickPerformanceReport + path: | + ${{ runner.temp }}/quick-performance/samples.jsonl + ${{ runner.temp }}/quick-performance/summary.json + ${{ runner.temp }}/quick-performance/summary.csv diff --git a/packages/performance-tests/.gitignore b/packages/performance-tests/.gitignore index 7600a8a06..1f317dedb 100644 --- a/packages/performance-tests/.gitignore +++ b/packages/performance-tests/.gitignore @@ -1,2 +1,5 @@ .env **/.output +.quick-cache/ +.quick-output/ +**/*.bim diff --git a/packages/performance-tests/README.md b/packages/performance-tests/README.md index b9dc054f4..c16eaf3e7 100644 --- a/packages/performance-tests/README.md +++ b/packages/performance-tests/README.md @@ -4,6 +4,50 @@ A package containing performance tests for the [`@itwin/imodel-transformer` libr ## Tests +### Quick incremental performance + +The quick suite is independent of the cloud-backed weekly regression suite. It +reconstructs a fresh local HubMock from the versioned +`balanced-incremental` recipe for every sample, establishes target provenance, +pushes eight real source changesets, and then times only +`IModelTransformer.process()` with `argsForProcessChanges`. + +Reconstruction, verification, and reporting are outside benchmark timing but +are reported and count against the 15-minute end-to-end budget. The suite does +not use iModelHub credentials or download QA iModels. + +Build the transformer package before running the TypeScript suite: + +```sh +pnpm --dir ../transformer build:cjs +pnpm test:quick +``` + +The default is one warm-up plus eight measured samples. Set +`QUICK_PERF_SAMPLES` only for local diagnostics. Reports are written under +`test/quick/.quick-output/` unless `QUICK_PERF_OUTPUT` is set and include +`samples.jsonl`, `summary.json`, and `summary.csv`. + +The calibrated fixture contains 6,000 base elements, 12,000 aspects, 3,000 +relationships, and 3,000 geometry-bearing elements. Its eight changesets apply +600 element inserts/updates/deletes, 600 aspect inserts/updates, 1,200 aspect +deletes, 300 relationship inserts/updates, 825 relationship deletes, and 150 +geometry updates. This is 25 deterministic repetitions of one balanced content +unit, preserving the original scenario ratios. + +`varianceStatus` requires coefficient of variation and normalized MAD at or +below 5%. An unstable manual run emits a workflow warning and must not be used +as regression evidence. It does not currently fail the manual workflow: six +local calibration suites informed the final scale, and three ratio-correct final +suites on a shared workstation produced 1.38-1.48 second medians but only two +met the CV threshold. A hard gate would therefore have false-failed one of three +final runs. Revisit the failure policy after repeated measurements on the target +Windows runner or a dedicated performance agent. + +`pnpm quick:build-fixture` writes the canonical recipe manifest. +`pnpm quick:verify-fixture` performs two fresh reconstructions (warm-up plus one +measured sample), checks their semantic digests, and writes a diagnostic report. + Here are tests we need but don't have: - *Identity Transform* @@ -42,4 +86,3 @@ Here are tests we need but don't have: ```sh pnpm exec process-results < report.jsonl ``` - diff --git a/packages/performance-tests/package.json b/packages/performance-tests/package.json index 09a07f57e..b95a72ba0 100644 --- a/packages/performance-tests/package.json +++ b/packages/performance-tests/package.json @@ -10,6 +10,9 @@ "test": "mocha --delay --timeout 300000 --require ts-node/register test/**/*.test.ts", "format": "prettier \"./test/**/*.ts\" --write", "test-mocha": "mocha --delay --timeout 300000 \"./lib/**/TransformerRegression.test.js\"", + "test:quick": "mocha --timeout 900000 --require ts-node/register test/quick/QuickPerformance.quick.ts", + "quick:build-fixture": "ts-node test/quick/cli.ts build-fixture", + "quick:verify-fixture": "ts-node test/quick/cli.ts verify-fixture", "process-reports": "node scripts/process-reports" }, "repository": {}, diff --git a/packages/performance-tests/test/quick/BenchmarkReporter.ts b/packages/performance-tests/test/quick/BenchmarkReporter.ts new file mode 100644 index 000000000..2151acdb2 --- /dev/null +++ b/packages/performance-tests/test/quick/BenchmarkReporter.ts @@ -0,0 +1,118 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "fs"; +import * as path from "path"; +import { BenchmarkSample } from "./BenchmarkRunner"; +import { + coefficientOfVariation, + median, + medianAbsoluteDeviation, + percentile, +} from "./validation/statistics"; + +export const maximumCoefficientOfVariation = 0.05; +export const maximumNormalizedMad = 0.05; +export const minimumMeasuredSamplesForReliability = 8; + +export function classifyVariance( + measuredSamples: number, + observedCoefficientOfVariation: number, + normalizedMad: number +): "insufficient-samples" | "stable" | "unstable" { + if (measuredSamples < minimumMeasuredSamplesForReliability) + return "insufficient-samples"; + return observedCoefficientOfVariation <= maximumCoefficientOfVariation && + normalizedMad <= maximumNormalizedMad + ? "stable" + : "unstable"; +} + +export class BenchmarkReporter { + public static write( + outputDir: string, + samples: readonly BenchmarkSample[], + jobMilliseconds?: number + ): void { + const measured = samples.filter((sample) => sample.measured); + const walls = measured.map((sample) => sample.wallMilliseconds); + const wallMedian = median(walls); + const wallCoefficientOfVariation = coefficientOfVariation(walls); + const wallMad = medianAbsoluteDeviation(walls); + const reconstruction = samples.map( + (sample) => sample.reconstructionMilliseconds + ); + const phaseSummary = (values: readonly number[]) => ({ + median: median(values), + maximum: Math.max(...values), + total: values.reduce((sum, value) => sum + value, 0), + }); + const summary = { + fixtureId: measured[0]?.fixtureId, + jobMilliseconds, + measuredSamples: measured.length, + varianceStatus: classifyVariance( + measured.length, + wallCoefficientOfVariation, + wallMad / wallMedian + ), + varianceThresholds: { + coefficientOfVariation: maximumCoefficientOfVariation, + normalizedMad: maximumNormalizedMad, + }, + unstableSamples: measured + .filter( + (sample) => + Math.abs(sample.wallMilliseconds - wallMedian) / wallMedian > 0.15 + ) + .map((sample) => sample.sample), + wallMilliseconds: { + median: wallMedian, + p90: percentile(walls, 0.9), + p95: percentile(walls, 0.95), + mad: wallMad, + normalizedMad: wallMad / wallMedian, + coefficientOfVariation: wallCoefficientOfVariation, + minimum: Math.min(...walls), + maximum: Math.max(...walls), + }, + reconstructionMilliseconds: phaseSummary(reconstruction), + verificationMilliseconds: phaseSummary( + samples.map((sample) => sample.verificationMilliseconds) + ), + teardownMilliseconds: phaseSummary( + samples.map((sample) => sample.teardownMilliseconds) + ), + }; + fs.mkdirSync(outputDir, { recursive: true }); + fs.writeFileSync( + path.join(outputDir, "samples.jsonl"), + `${samples.map((sample) => JSON.stringify(sample)).join("\n")}\n` + ); + fs.writeFileSync( + path.join(outputDir, "summary.json"), + `${JSON.stringify(summary, undefined, 2)}\n` + ); + fs.writeFileSync( + path.join(outputDir, "summary.csv"), + [ + "fixture,measuredSamples,jobMs,medianMs,p90Ms,p95Ms,madMs,cv,reconstructionTotalMs,verificationTotalMs,teardownTotalMs", + [ + summary.fixtureId, + summary.measuredSamples, + summary.jobMilliseconds ?? "", + summary.wallMilliseconds.median, + summary.wallMilliseconds.p90, + summary.wallMilliseconds.p95, + summary.wallMilliseconds.mad, + summary.wallMilliseconds.coefficientOfVariation, + summary.reconstructionMilliseconds.total, + summary.verificationMilliseconds.total, + summary.teardownMilliseconds.total, + ].join(","), + ].join("\n") + ); + } +} diff --git a/packages/performance-tests/test/quick/BenchmarkRunner.ts b/packages/performance-tests/test/quick/BenchmarkRunner.ts new file mode 100644 index 000000000..6ff238685 --- /dev/null +++ b/packages/performance-tests/test/quick/BenchmarkRunner.ts @@ -0,0 +1,214 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { IModelHost } from "@itwin/core-backend"; +import { DatasetDescriptor } from "./DatasetDescriptor"; +import { materializeFixture, PreparedDataset } from "./FixtureMaterializer"; +import { disposeReconstructedHub } from "./LocalHubFixture"; +import { incrementalSynchronization } from "./scenarios/incrementalSynchronization"; + +export const benchmarkOutputMarkerName = + ".imodel-transformer-quick-performance"; + +function normalizeError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} + +function resolveThroughExistingAncestor(fileName: string): string { + let ancestor = path.resolve(fileName); + const suffix: string[] = []; + while (!fs.existsSync(ancestor)) { + const parent = path.dirname(ancestor); + if (parent === ancestor) + throw new Error(`Cannot resolve benchmark output path: ${fileName}`); + suffix.unshift(path.basename(ancestor)); + ancestor = parent; + } + return path.join(fs.realpathSync(ancestor), ...suffix); +} + +function isStrictDescendant(candidate: string, root: string): boolean { + const relative = path.relative(root, candidate); + return ( + relative.length > 0 && + relative !== ".." && + !relative.startsWith(`..${path.sep}`) && + !path.isAbsolute(relative) + ); +} + +export function assertSafeBenchmarkOutputPath(outputDir: string): void { + const candidate = resolveThroughExistingAncestor(outputDir); + const allowedRoots = [ + path.join(__dirname, ".quick-output"), + os.tmpdir(), + process.platform === "win32" ? undefined : "/tmp", + process.env.RUNNER_TEMP, + ] + .filter((root): root is string => root !== undefined) + .map(resolveThroughExistingAncestor); + if (!allowedRoots.some((root) => isStrictDescendant(candidate, root))) + throw new Error( + `Quick performance output must be below the package output or temporary directory: ${outputDir}` + ); +} + +export function prepareBenchmarkOutputDirectory(outputDir: string): void { + assertSafeBenchmarkOutputPath(outputDir); + fs.mkdirSync(outputDir, { recursive: true }); + const marker = path.join(outputDir, benchmarkOutputMarkerName); + const entries = fs.readdirSync(outputDir); + if (!fs.existsSync(marker) && entries.length > 0) + throw new Error( + `Refusing to use non-empty unowned output directory: ${outputDir}` + ); + fs.writeFileSync( + marker, + "Owned by iModel Transformer quick performance tests.\n" + ); + for (const entry of entries) { + if ( + /^sample-\d+$/.test(entry) || + [ + "manifest.json", + "samples.jsonl", + "summary.csv", + "summary.json", + ].includes(entry) + ) + fs.rmSync(path.join(outputDir, entry), { + recursive: true, + force: true, + }); + } +} + +async function cleanupSample( + scenario: ReturnType | undefined, + dataset: PreparedDataset | undefined, + sampleDir: string +): Promise { + const errors: unknown[] = []; + try { + scenario?.abort(); + } catch (error) { + errors.push(error); + } + try { + if (dataset) await disposeReconstructedHub(dataset.hub); + } catch (error) { + errors.push(error); + } + try { + fs.rmSync(sampleDir, { recursive: true, force: true }); + } catch (error) { + errors.push(error); + } + return errors; +} + +export interface BenchmarkSample { + readonly cpuSystemMilliseconds: number; + readonly cpuUserMilliseconds: number; + readonly fixtureId: string; + readonly measured: boolean; + readonly operations: DatasetDescriptor["distribution"]["operations"]; + readonly reconstructionMilliseconds: number; + readonly rssDeltaBytes: number; + readonly sample: number; + readonly semanticDigest: string; + readonly teardownMilliseconds: number; + readonly verificationMilliseconds: number; + readonly wallMilliseconds: number; +} + +export class BenchmarkRunner { + public constructor( + private readonly _descriptor: DatasetDescriptor, + private readonly _outputDir: string + ) {} + + public async run(measuredSamples = 8): Promise { + if (!Number.isInteger(measuredSamples) || measuredSamples < 1) + throw new Error( + "Quick performance requires at least one measured sample" + ); + prepareBenchmarkOutputDirectory(this._outputDir); + const samples: BenchmarkSample[] = []; + await IModelHost.startup(); + try { + for (let sample = 0; sample <= measuredSamples; sample++) { + const sampleDir = path.join(this._outputDir, `sample-${sample}`); + let dataset: PreparedDataset | undefined; + let scenario: ReturnType | undefined; + let operationError: Error | undefined; + let completedSample: + | Omit + | undefined; + try { + dataset = await materializeFixture( + this._descriptor, + sampleDir, + `quick-sample-${sample}` + ); + scenario = incrementalSynchronization(dataset); + const rssBefore = process.memoryUsage().rss; + const cpuBefore = process.cpuUsage(); + const wallStart = process.hrtime.bigint(); + await scenario.measure(); + const wallMilliseconds = + Number(process.hrtime.bigint() - wallStart) / 1_000_000; + const cpu = process.cpuUsage(cpuBefore); + const rssDeltaBytes = process.memoryUsage().rss - rssBefore; + const verificationStart = process.hrtime.bigint(); + const semanticDigest = await scenario.finish(); + const verificationMilliseconds = + Number(process.hrtime.bigint() - verificationStart) / 1_000_000; + completedSample = { + cpuSystemMilliseconds: cpu.system / 1000, + cpuUserMilliseconds: cpu.user / 1000, + fixtureId: this._descriptor.id, + measured: sample !== 0, + operations: this._descriptor.distribution.operations, + reconstructionMilliseconds: dataset.reconstructionMilliseconds, + rssDeltaBytes, + sample, + semanticDigest, + verificationMilliseconds, + wallMilliseconds, + }; + } catch (error) { + operationError = normalizeError(error); + } + const teardownStart = process.hrtime.bigint(); + const cleanupErrors = await cleanupSample(scenario, dataset, sampleDir); + const teardownMilliseconds = + Number(process.hrtime.bigint() - teardownStart) / 1_000_000; + if (operationError && cleanupErrors.length === 0) throw operationError; + if (cleanupErrors.length > 0) + throw new AggregateError( + operationError ? [operationError, ...cleanupErrors] : cleanupErrors, + "Quick performance sample cleanup failed" + ); + if (!completedSample) + throw new Error( + "Quick performance sample completed without a result" + ); + const sampleResult = { ...completedSample, teardownMilliseconds }; + samples.push(sampleResult); + fs.appendFileSync( + path.join(this._outputDir, "samples.jsonl"), + `${JSON.stringify(sampleResult)}\n` + ); + } + } finally { + await IModelHost.shutdown(); + } + return samples; + } +} diff --git a/packages/performance-tests/test/quick/DatasetDescriptor.ts b/packages/performance-tests/test/quick/DatasetDescriptor.ts new file mode 100644 index 000000000..edfd7b259 --- /dev/null +++ b/packages/performance-tests/test/quick/DatasetDescriptor.ts @@ -0,0 +1,45 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +export interface FixtureOperationCounts { + readonly aspects: Readonly>; + readonly elements: Readonly< + Record<"deletes" | "inserts" | "updates", number> + >; + readonly relationships: Readonly< + Record<"deletes" | "inserts" | "updates", number> + >; + readonly geometryUpdates: number; + readonly sourceChangesets: number; +} + +export interface FixtureDistribution { + readonly base: { + readonly aspects: number; + readonly elements: number; + readonly geometricElements: number; + readonly relationships: number; + }; + readonly operations: FixtureOperationCounts; +} + +export interface DatasetDescriptor { + readonly id: string; + readonly version: number; + readonly label: string; + readonly scenarioClaims: readonly string[]; + readonly layout: { + readonly kind: "reconstructed"; + readonly recipe: "balanced-incremental"; + readonly seed: number; + }; + readonly distribution: FixtureDistribution; + readonly generator: { + readonly coreBackend: string; + readonly node: string; + readonly transformer: string; + }; + readonly recipeHash: string; +} diff --git a/packages/performance-tests/test/quick/FixtureCatalog.ts b/packages/performance-tests/test/quick/FixtureCatalog.ts new file mode 100644 index 000000000..1133982d0 --- /dev/null +++ b/packages/performance-tests/test/quick/FixtureCatalog.ts @@ -0,0 +1,98 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "fs"; +import * as path from "path"; +import { DatasetDescriptor } from "./DatasetDescriptor"; +import { canonicalSha256 } from "./FixtureManifest"; + +const scale = 25; +const distribution = { + base: { + aspects: 480 * scale, + elements: 240 * scale, + geometricElements: 120 * scale, + relationships: 120 * scale, + }, + operations: { + elements: { + inserts: 24 * scale, + updates: 24 * scale, + deletes: 24 * scale, + }, + aspects: { + inserts: 24 * scale, + updates: 24 * scale, + deletes: 48 * scale, + }, + relationships: { + inserts: 12 * scale, + updates: 12 * scale, + deletes: 33 * scale, + }, + geometryUpdates: 6 * scale, + sourceChangesets: 8, + }, +} as const; + +function packageVersion(packageName: string): string { + const packageJson = JSON.parse( + fs.readFileSync(require.resolve(`${packageName}/package.json`), "utf8") + ) as { version: string }; + return packageJson.version; +} + +const generator = { + coreBackend: packageVersion("@itwin/core-backend"), + node: process.version, + transformer: packageVersion("@itwin/imodel-transformer"), +}; + +const recipeIdentity = { + schema: "QuickPerf.01.00.00", + seed: 328, + distribution, + inputs: { + recipe: fs.readFileSync( + path.join(__dirname, "recipes/balancedIncremental.ts"), + "utf8" + ), + schema: fs.readFileSync( + path.join(__dirname, "schemas/QuickPerf.ecschema.xml"), + "utf8" + ), + lockfile: fs.readFileSync( + path.join(__dirname, "../../../../pnpm-lock.yaml"), + "utf8" + ), + }, + versions: generator, +}; + +export const balancedIncrementalDescriptor: DatasetDescriptor = { + id: "balanced-incremental", + version: 1, + label: "balanced incremental", + scenarioClaims: [ + "incremental synchronization", + "aspect lifecycle", + "relationship lifecycle", + "mixed scalar and geometry element changes", + ], + layout: { + kind: "reconstructed", + recipe: "balanced-incremental", + seed: 328, + }, + distribution, + generator, + recipeHash: canonicalSha256(recipeIdentity), +}; + +export function getFixtureDescriptor(id: string): DatasetDescriptor { + if (id !== balancedIncrementalDescriptor.id) + throw new Error(`Unknown quick performance fixture: ${id}`); + return balancedIncrementalDescriptor; +} diff --git a/packages/performance-tests/test/quick/FixtureManifest.quick-unit.ts b/packages/performance-tests/test/quick/FixtureManifest.quick-unit.ts new file mode 100644 index 000000000..4d3eb4000 --- /dev/null +++ b/packages/performance-tests/test/quick/FixtureManifest.quick-unit.ts @@ -0,0 +1,25 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { expect } from "chai"; +import { balancedIncrementalDescriptor } from "./FixtureCatalog"; +import { canonicalSha256, validateDescriptor } from "./FixtureManifest"; + +describe("FixtureManifest", () => { + it("hashes objects independently of key insertion order", () => { + expect(canonicalSha256({ first: 1, second: 2 })).to.equal( + canonicalSha256({ second: 2, first: 1 }) + ); + }); + + it("validates the catalog descriptor and rejects invalid input", () => { + expect(validateDescriptor(balancedIncrementalDescriptor)).to.equal( + balancedIncrementalDescriptor + ); + expect(() => validateDescriptor({ id: "invalid" })).to.throw( + "invalid shape" + ); + }); +}); diff --git a/packages/performance-tests/test/quick/FixtureManifest.ts b/packages/performance-tests/test/quick/FixtureManifest.ts new file mode 100644 index 000000000..e266caa32 --- /dev/null +++ b/packages/performance-tests/test/quick/FixtureManifest.ts @@ -0,0 +1,46 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { createHash } from "crypto"; +import { DatasetDescriptor } from "./DatasetDescriptor"; + +function canonicalize(value: unknown): string { + if (Array.isArray(value)) + return `[${value.map((entry) => canonicalize(entry)).join(",")}]`; + if (value !== null && typeof value === "object") { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalize(record[key])}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +export function canonicalSha256(value: unknown): string { + return createHash("sha256").update(canonicalize(value)).digest("hex"); +} + +export function validateDescriptor(value: unknown): DatasetDescriptor { + if (value === null || typeof value !== "object") + throw new Error("Fixture descriptor must be an object"); + const descriptor = value as Partial; + if ( + typeof descriptor.id !== "string" || + typeof descriptor.version !== "number" || + typeof descriptor.label !== "string" || + !Array.isArray(descriptor.scenarioClaims) || + descriptor.layout?.kind !== "reconstructed" || + descriptor.layout.recipe !== "balanced-incremental" || + typeof descriptor.layout.seed !== "number" || + descriptor.distribution === undefined || + typeof descriptor.generator?.coreBackend !== "string" || + typeof descriptor.generator.node !== "string" || + typeof descriptor.generator.transformer !== "string" || + typeof descriptor.recipeHash !== "string" + ) + throw new Error("Fixture descriptor has an invalid shape"); + return descriptor as DatasetDescriptor; +} diff --git a/packages/performance-tests/test/quick/FixtureMaterializer.ts b/packages/performance-tests/test/quick/FixtureMaterializer.ts new file mode 100644 index 000000000..3d1abca9f --- /dev/null +++ b/packages/performance-tests/test/quick/FixtureMaterializer.ts @@ -0,0 +1,89 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as path from "path"; +import { IModelTransformer } from "@itwin/imodel-transformer"; +import { DatasetDescriptor } from "./DatasetDescriptor"; +import { + createStartedEditTxn, + disposeReconstructedHub, + ReconstructedHub, + reconstructHub, +} from "./LocalHubFixture"; +import { + applyBalancedChangesets, + BalancedRecipeState, + createBalancedSeed, +} from "./recipes/balancedIncremental"; +import { assertFixtureDistribution } from "./validation/validateFixture"; + +export interface PreparedDataset { + readonly descriptor: DatasetDescriptor; + readonly hub: ReconstructedHub; + readonly reconstructionMilliseconds: number; +} + +export async function materializeFixture( + descriptor: DatasetDescriptor, + outputDir: string, + sampleName: string +): Promise { + const start = process.hrtime.bigint(); + let recipeState: BalancedRecipeState | undefined; + let hub: ReconstructedHub | undefined; + try { + hub = await reconstructHub(outputDir, sampleName, async (sourceSeed) => { + recipeState = await createBalancedSeed(sourceSeed, descriptor); + }); + if (!recipeState) + throw new Error("Balanced fixture recipe did not create state"); + + const editTxn = createStartedEditTxn(hub.targetDb); + const transformer = new IModelTransformer({ + source: hub.sourceDb, + target: editTxn, + }); + try { + await transformer.processSchemas(); + await transformer.process(); + } finally { + transformer.dispose(); + if (editTxn.isActive) editTxn.end(); + } + await hub.targetDb.pushChanges({ + accessToken: hub.accessToken, + description: "establish quick fixture provenance", + }); + await applyBalancedChangesets( + hub.sourceDb, + hub.accessToken, + descriptor, + recipeState + ); + await assertFixtureDistribution(hub.sourceDb, descriptor); + const reconstructionMilliseconds = + Number(process.hrtime.bigint() - start) / 1_000_000; + return { descriptor, hub, reconstructionMilliseconds }; + } catch (error) { + if (hub) { + try { + await disposeReconstructedHub(hub); + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + "Fixture materialization and cleanup both failed" + ); + } + } + throw error; + } +} + +export function fixtureWorkingDirectory( + root: string, + sampleName: string +): string { + return path.join(root, sampleName); +} diff --git a/packages/performance-tests/test/quick/LocalHubFixture.quick-unit.ts b/packages/performance-tests/test/quick/LocalHubFixture.quick-unit.ts new file mode 100644 index 000000000..b3707d64e --- /dev/null +++ b/packages/performance-tests/test/quick/LocalHubFixture.quick-unit.ts @@ -0,0 +1,294 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { expect } from "chai"; +import { + Code, + IModel, + PhysicalElementProps, + QueryBinder, +} from "@itwin/core-common"; +import { + IModelHost, + PhysicalModel, + PhysicalObject, + SnapshotDb, + SpatialCategory, + withEditTxn, +} from "@itwin/core-backend"; +import { HubMock } from "@itwin/core-backend/lib/cjs/internal/HubMock"; +import { IModelTransformer } from "@itwin/imodel-transformer"; +import { + benchmarkOutputMarkerName, + BenchmarkRunner, + prepareBenchmarkOutputDirectory, +} from "./BenchmarkRunner"; +import { balancedIncrementalDescriptor } from "./FixtureCatalog"; +import { materializeFixture } from "./FixtureMaterializer"; +import { + createStartedEditTxn, + disposeReconstructedHub, + ReconstructedHub, + reconstructHub, +} from "./LocalHubFixture"; +import { assertSynchronizationProvenance } from "./validation/validateFixture"; + +function required(value: T | undefined, name: string): T { + if (value === undefined) throw new Error(`${name} was not initialized`); + return value; +} + +function insertPhysicalObject( + db: SnapshotDb | ReconstructedHub["sourceDb"], + modelId: string, + categoryId: string, + name: string +): string { + return withEditTxn(db, `insert ${name}`, (txn) => { + const props: PhysicalElementProps = { + category: categoryId, + classFullName: PhysicalObject.classFullName, + code: new Code({ + scope: IModel.rootSubjectId, + spec: IModel.rootSubjectId, + value: name, + }), + model: modelId, + userLabel: name, + }; + return txn.insertElement(props); + }); +} + +async function queryElementIdByLabel( + db: ReconstructedHub["sourceDb"] | ReconstructedHub["targetDb"], + label: string +): Promise { + const reader = db.createQueryReader( + "SELECT ECInstanceId id FROM bis.Element WHERE UserLabel=?", + QueryBinder.from([label]), + { usePrimaryConn: true } + ); + return (await reader.step()) ? (reader.current.id as string) : undefined; +} + +describe("LocalHubFixture reconstruction", () => { + let outputDir: string; + + before(async () => { + outputDir = fs.mkdtempSync( + path.join(os.tmpdir(), "quick-perf-reconstruct-") + ); + await IModelHost.startup(); + }); + + after(async () => { + await IModelHost.shutdown(); + fs.rmSync(outputDir, { recursive: true, force: true }); + }); + + it("shuts down HubMock when reconstruction fails", async () => { + let failure: unknown; + try { + await reconstructHub(outputDir, "expected-failure", () => { + throw new Error("expected seed failure"); + }); + } catch (error) { + failure = error; + } + expect(failure).to.be.instanceOf(Error); + expect(HubMock.isValid).to.be.false; + }); + + it("rejects an invalid HubMock output path before startup", async () => { + const invalidOutput = path.join(outputDir, "not-a-directory"); + fs.writeFileSync(invalidOutput, "file"); + let failure: unknown; + try { + await reconstructHub(invalidOutput, "invalid-output", () => undefined); + } catch (error) { + failure = error; + } + expect(failure).to.be.instanceOf(Error); + expect(HubMock.isValid).to.be.false; + }); + + it("disposes a reconstructed hub when materialization fails", async () => { + const invalidDescriptor = { + ...balancedIncrementalDescriptor, + distribution: { + ...balancedIncrementalDescriptor.distribution, + operations: { + ...balancedIncrementalDescriptor.distribution.operations, + sourceChangesets: 9, + }, + }, + }; + let failure: unknown; + try { + await materializeFixture( + invalidDescriptor, + path.join(outputDir, "materialize-failure"), + "materialize-failure" + ); + } catch (error) { + failure = error; + } + expect(failure).to.be.instanceOf(Error); + expect(HubMock.isValid).to.be.false; + }); + + it("does not delete an unowned output directory", async () => { + const unsafeOutput = path.join(outputDir, "unowned-output"); + const sentinel = path.join(unsafeOutput, "sentinel.txt"); + fs.mkdirSync(unsafeOutput, { recursive: true }); + fs.writeFileSync(sentinel, "preserve"); + let failure: unknown; + try { + await new BenchmarkRunner( + balancedIncrementalDescriptor, + unsafeOutput + ).run(1); + } catch (error) { + failure = error; + } + expect(failure).to.be.instanceOf(Error); + expect(fs.readFileSync(sentinel, "utf8")).to.equal("preserve"); + }); + + it("rejects an arbitrary empty output directory", () => { + const unsafeOutput = fs.mkdtempSync( + path.join(process.cwd(), "unsafe-quick-output-") + ); + try { + expect(() => prepareBenchmarkOutputDirectory(unsafeOutput)).to.throw( + "Quick performance output must be below" + ); + expect(fs.existsSync(path.join(unsafeOutput, benchmarkOutputMarkerName))) + .to.be.false; + } finally { + fs.rmSync(unsafeOutput, { recursive: true, force: true }); + } + }); + + it("rejects an empty measured sample set", async () => { + let failure: unknown; + try { + await new BenchmarkRunner( + balancedIncrementalDescriptor, + path.join(outputDir, "zero-samples") + ).run(0); + } catch (error) { + failure = error; + } + expect(failure).to.be.instanceOf(Error); + expect((failure as Error).message).to.equal( + "Quick performance requires at least one measured sample" + ); + }); + + it("reconstructs an offline hub and processes insert, update, and delete changesets", async () => { + let hub: ReconstructedHub | undefined; + let modelId: string | undefined; + let categoryId: string | undefined; + try { + hub = await reconstructHub(outputDir, "phase-zero", (sourceSeed) => { + const db = SnapshotDb.createEmpty(sourceSeed, { + rootSubject: { name: "phase-zero-source" }, + }); + ({ categoryId, modelId } = withEditTxn( + db, + "create base model and category", + (txn) => ({ + modelId: PhysicalModel.insert( + txn, + IModel.rootSubjectId, + "PhysicalModel" + ), + categoryId: SpatialCategory.insert( + txn, + IModel.dictionaryId, + "SpatialCategory", + {} + ), + }) + )); + insertPhysicalObject(db, modelId, categoryId, "update-me"); + insertPhysicalObject(db, modelId, categoryId, "delete-me"); + db.close(); + }); + + const initialTxn = createStartedEditTxn(hub.targetDb); + const initialTransformer = new IModelTransformer({ + source: hub.sourceDb, + target: initialTxn, + }); + await initialTransformer.process(); + initialTransformer.dispose(); + initialTxn.end(); + await hub.targetDb.pushChanges({ + accessToken: hub.accessToken, + description: "establish base provenance", + }); + + insertPhysicalObject( + hub.sourceDb, + required(modelId, "modelId"), + required(categoryId, "categoryId"), + "inserted" + ); + await hub.sourceDb.pushChanges({ + accessToken: hub.accessToken, + description: "insert element", + }); + + const updateId = await queryElementIdByLabel(hub.sourceDb, "update-me"); + expect(updateId).not.to.be.undefined; + const sourceDb = hub.sourceDb; + withEditTxn(sourceDb, "update element", (txn) => { + const props = sourceDb.elements.getElementProps( + required(updateId, "updateId") + ); + txn.updateElement({ ...props, userLabel: "updated" }); + }); + await hub.sourceDb.pushChanges({ + accessToken: hub.accessToken, + description: "update element", + }); + + const deleteId = await queryElementIdByLabel(hub.sourceDb, "delete-me"); + expect(deleteId).not.to.be.undefined; + withEditTxn(hub.sourceDb, "delete element", (txn) => + txn.deleteElement(required(deleteId, "deleteId")) + ); + await hub.sourceDb.pushChanges({ + accessToken: hub.accessToken, + description: "delete element", + }); + + const incrementalTxn = createStartedEditTxn(hub.targetDb); + const incrementalTransformer = new IModelTransformer( + { source: hub.sourceDb, target: incrementalTxn }, + { argsForProcessChanges: {} } + ); + await incrementalTransformer.process(); + incrementalTransformer.dispose(); + incrementalTxn.end(); + await assertSynchronizationProvenance(hub.sourceDb, hub.targetDb); + + expect(await queryElementIdByLabel(hub.targetDb, "inserted")).not.to.be + .undefined; + expect(await queryElementIdByLabel(hub.targetDb, "updated")).not.to.be + .undefined; + expect(await queryElementIdByLabel(hub.targetDb, "delete-me")).to.be + .undefined; + } finally { + if (hub) await disposeReconstructedHub(hub); + } + }); +}); diff --git a/packages/performance-tests/test/quick/LocalHubFixture.ts b/packages/performance-tests/test/quick/LocalHubFixture.ts new file mode 100644 index 000000000..d3f7307df --- /dev/null +++ b/packages/performance-tests/test/quick/LocalHubFixture.ts @@ -0,0 +1,190 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "fs"; +import * as path from "path"; +import { AccessToken } from "@itwin/core-bentley"; +import { + BriefcaseDb, + BriefcaseManager, + EditTxn, + IModelHost, + SnapshotDb, +} from "@itwin/core-backend"; +import { HubMock } from "@itwin/core-backend/lib/cjs/internal/HubMock"; +// eslint-disable-next-line @itwin/no-internal +import { _hubAccess } from "@itwin/core-backend/lib/cjs/internal/Symbols"; + +export interface ReconstructedHub { + readonly accessToken: AccessToken; + readonly iTwinId: string; + readonly sourceDb: BriefcaseDb; + readonly sourceIModelId: string; + readonly targetDb: BriefcaseDb; + readonly targetIModelId: string; +} + +export async function createAndOpenIModel( + accessToken: AccessToken, + iTwinId: string, + iModelName: string, + seedFileName: string +): Promise<{ db: BriefcaseDb; iModelId: string }> { + const iModelId = await IModelHost[_hubAccess].createNewIModel({ + accessToken, + iTwinId, + iModelName, + noLocks: true, + version0: seedFileName, + }); + const briefcase = await BriefcaseManager.downloadBriefcase({ + accessToken, + iTwinId, + iModelId, + }); + try { + return { + db: await BriefcaseDb.open({ fileName: briefcase.fileName }), + iModelId, + }; + } catch (error) { + try { + await BriefcaseManager.deleteBriefcaseFiles( + briefcase.fileName, + accessToken + ); + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + "Briefcase open and cleanup both failed" + ); + } + throw error; + } +} + +export function createEmptySeed( + fileName: string, + rootSubjectName: string +): void { + fs.mkdirSync(path.dirname(fileName), { recursive: true }); + const db = SnapshotDb.createEmpty(fileName, { + rootSubject: { name: rootSubjectName }, + }); + db.close(); +} + +export function createStartedEditTxn(db: BriefcaseDb): EditTxn { + const editTxn = new EditTxn(db, "Quick performance fixture"); + editTxn.start(); + return editTxn; +} + +export async function closeAndDeleteBriefcase( + accessToken: AccessToken, + db: BriefcaseDb +): Promise { + const fileName = db.pathName; + db.close(); + await BriefcaseManager.deleteBriefcaseFiles(fileName, accessToken); +} + +async function cleanupHub( + accessToken: AccessToken, + briefcases: readonly BriefcaseDb[] +): Promise { + const errors: unknown[] = []; + for (const briefcase of briefcases) { + try { + await closeAndDeleteBriefcase(accessToken, briefcase); + } catch (error) { + errors.push(error); + } + } + try { + if (HubMock.isValid) HubMock.shutdown(); + } catch (error) { + errors.push(error); + } + return errors; +} + +export async function reconstructHub( + outputDir: string, + mockName: string, + createSourceSeed: (fileName: string) => Promise | void +): Promise { + if (HubMock.isValid) throw new Error("Only one HubMock may be active"); + + fs.mkdirSync(outputDir, { recursive: true }); + if (!fs.statSync(outputDir).isDirectory()) + throw new Error(`HubMock output path is not a directory: ${outputDir}`); + try { + HubMock.startup(mockName, outputDir); + } catch (error) { + try { + if (HubMock.isValid) HubMock.shutdown(); + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + "HubMock startup and cleanup both failed" + ); + } + throw error; + } + const accessToken = "quick-performance-tests"; + const iTwinId = HubMock.iTwinId; + const openBriefcases: BriefcaseDb[] = []; + try { + const seedDir = path.join(outputDir, "seeds"); + const sourceSeed = path.join(seedDir, `${mockName}-source.bim`); + const targetSeed = path.join(seedDir, `${mockName}-target.bim`); + fs.mkdirSync(seedDir, { recursive: true }); + await createSourceSeed(sourceSeed); + createEmptySeed(targetSeed, `${mockName}-target`); + + const source = await createAndOpenIModel( + accessToken, + iTwinId, + `${mockName}-source`, + sourceSeed + ); + openBriefcases.push(source.db); + const target = await createAndOpenIModel( + accessToken, + iTwinId, + `${mockName}-target`, + targetSeed + ); + openBriefcases.push(target.db); + return { + accessToken, + iTwinId, + sourceDb: source.db, + sourceIModelId: source.iModelId, + targetDb: target.db, + targetIModelId: target.iModelId, + }; + } catch (error) { + const cleanupErrors = await cleanupHub(accessToken, openBriefcases); + if (cleanupErrors.length > 0) + throw new AggregateError( + [error, ...cleanupErrors], + "Hub reconstruction and cleanup both failed" + ); + throw error; + } +} + +export async function disposeReconstructedHub( + hub: ReconstructedHub +): Promise { + const errors = await cleanupHub(hub.accessToken, [ + hub.sourceDb, + hub.targetDb, + ]); + if (errors.length > 0) + throw new AggregateError(errors, "Failed to dispose reconstructed HubMock"); +} diff --git a/packages/performance-tests/test/quick/QuickPerformance.quick.ts b/packages/performance-tests/test/quick/QuickPerformance.quick.ts new file mode 100644 index 000000000..fe3d974de --- /dev/null +++ b/packages/performance-tests/test/quick/QuickPerformance.quick.ts @@ -0,0 +1,41 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as path from "path"; +import { expect } from "chai"; +import { BenchmarkReporter } from "./BenchmarkReporter"; +import { BenchmarkRunner } from "./BenchmarkRunner"; +import { getFixtureDescriptor } from "./FixtureCatalog"; + +describe("quick transformer performance", function () { + this.timeout(15 * 60 * 1000); + + it("runs the balanced incremental synchronization fixture", async () => { + const measuredSamples = Number(process.env.QUICK_PERF_SAMPLES ?? "8"); + const outputDir = + process.env.QUICK_PERF_OUTPUT ?? + path.join(__dirname, ".quick-output", "balanced-incremental"); + const runner = new BenchmarkRunner( + getFixtureDescriptor("balanced-incremental"), + outputDir + ); + const jobStart = process.hrtime.bigint(); + const samples = await runner.run(measuredSamples); + const jobMilliseconds = + Number(process.hrtime.bigint() - jobStart) / 1_000_000; + BenchmarkReporter.write(outputDir, samples, jobMilliseconds); + + expect(samples.filter((sample) => sample.measured)).to.have.length( + measuredSamples + ); + expect( + new Set(samples.map((sample) => sample.semanticDigest)).size + ).to.equal( + 1, + "fresh reconstructions must produce the same semantic digest" + ); + expect(jobMilliseconds).to.be.lessThan(15 * 60 * 1000); + }); +}); diff --git a/packages/performance-tests/test/quick/cli.ts b/packages/performance-tests/test/quick/cli.ts new file mode 100644 index 000000000..b6a1db9fd --- /dev/null +++ b/packages/performance-tests/test/quick/cli.ts @@ -0,0 +1,49 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "fs"; +import * as path from "path"; +import { BenchmarkReporter } from "./BenchmarkReporter"; +import { + BenchmarkRunner, + prepareBenchmarkOutputDirectory, +} from "./BenchmarkRunner"; +import { balancedIncrementalDescriptor } from "./FixtureCatalog"; + +function writeManifest(outputDir: string): void { + fs.writeFileSync( + path.join(outputDir, "manifest.json"), + `${JSON.stringify(balancedIncrementalDescriptor, undefined, 2)}\n` + ); +} + +async function main() { + const command = process.argv[2]; + const outputDir = + process.env.QUICK_PERF_OUTPUT ?? + path.join(__dirname, ".quick-output", balancedIncrementalDescriptor.id); + if (command === "build-fixture") { + prepareBenchmarkOutputDirectory(outputDir); + writeManifest(outputDir); + return; + } + if (command === "verify-fixture") { + const samples = await new BenchmarkRunner( + balancedIncrementalDescriptor, + outputDir + ).run(1); + if (new Set(samples.map((sample) => sample.semanticDigest)).size !== 1) + throw new Error("Fixture reconstruction is not deterministic"); + BenchmarkReporter.write(outputDir, samples); + writeManifest(outputDir); + return; + } + throw new Error(`Unknown quick fixture command: ${command ?? ""}`); +} + +void main().catch((error) => { + process.stderr.write(`${String(error)}\n`); + process.exitCode = 1; +}); diff --git a/packages/performance-tests/test/quick/recipes/balancedIncremental.ts b/packages/performance-tests/test/quick/recipes/balancedIncremental.ts new file mode 100644 index 000000000..87fc6ec61 --- /dev/null +++ b/packages/performance-tests/test/quick/recipes/balancedIncremental.ts @@ -0,0 +1,332 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as path from "path"; +import { Id64String } from "@itwin/core-bentley"; +import { + Code, + ElementAspectProps, + GeometryStreamBuilder, + IModel, + PhysicalElementProps, +} from "@itwin/core-common"; +import { + BriefcaseDb, + ElementGroupsMembers, + ElementOwnsMultiAspects, + ElementOwnsUniqueAspect, + PhysicalModel, + PhysicalObject, + SnapshotDb, + SpatialCategory, + withEditTxn, +} from "@itwin/core-backend"; +import { + Box, + Point3d, + Range3d, + YawPitchRollAngles, +} from "@itwin/core-geometry"; +import { DatasetDescriptor } from "../DatasetDescriptor"; + +const uniqueAspectClass = "QuickPerf:BalancedUniqueAspect"; +const multiAspectClass = "QuickPerf:BalancedMultiAspect"; +const elementsPerUnit = 240; +const relationshipsPerUnit = 120; +const insertedElementsPerUnit = 24; + +function fixtureScale(descriptor: DatasetDescriptor): number { + const scale = descriptor.distribution.base.elements / elementsPerUnit; + if (!Number.isInteger(scale) || scale < 1) + throw new Error( + `Balanced fixture elements must be a positive multiple of ${elementsPerUnit}` + ); + return scale; +} + +export function createBoxGeometry(length = 1) { + const builder = new GeometryStreamBuilder(); + const box = Box.createRange( + Range3d.create( + Point3d.createZero(), + Point3d.create(length, length, length) + ), + true + ); + if (!box) throw new Error("Failed to create deterministic box geometry"); + builder.appendGeometry(box); + return builder.geometryStream; +} + +export interface BalancedRecipeState { + readonly categoryIds: readonly Id64String[]; + readonly elementIds: readonly Id64String[]; + readonly modelIds: readonly Id64String[]; + readonly relationshipIds: readonly Id64String[]; +} + +function elementProps( + modelId: Id64String, + categoryId: Id64String, + index: number, + geometric: boolean +): PhysicalElementProps { + return { + category: categoryId, + classFullName: PhysicalObject.classFullName, + code: new Code({ + scope: IModel.rootSubjectId, + spec: IModel.rootSubjectId, + value: `quick-element-${index}`, + }), + federationGuid: + index % 20 === 0 + ? `00000000-0000-4000-8000-${index.toString().padStart(12, "0")}` + : undefined, + geom: geometric ? createBoxGeometry() : undefined, + model: modelId, + placement: geometric + ? { + angles: YawPitchRollAngles.createDegrees(0, 0, index % 360), + origin: Point3d.create(index, index % 17, 0), + } + : undefined, + userLabel: `base-${index}`, + }; +} + +function insertAspects( + db: SnapshotDb | BriefcaseDb, + ownerId: Id64String, + sequence: number +): Id64String[] { + return withEditTxn(db, "insert quick aspects", (txn) => [ + txn.insertAspect({ + classFullName: uniqueAspectClass, + element: new ElementOwnsUniqueAspect(ownerId), + payload: `unique-${sequence}`, + sequence, + } as ElementAspectProps), + txn.insertAspect({ + classFullName: multiAspectClass, + element: new ElementOwnsMultiAspects(ownerId), + payload: `multi-${sequence}`, + sequence, + } as ElementAspectProps), + ]); +} + +export async function createBalancedSeed( + fileName: string, + descriptor: DatasetDescriptor +): Promise { + const db = SnapshotDb.createEmpty(fileName, { + rootSubject: { name: descriptor.id }, + }); + try { + await db.importSchemas([ + path.join(__dirname, "../schemas/QuickPerf.ecschema.xml"), + ]); + + const { categoryIds, modelIds } = withEditTxn( + db, + "create quick models and categories", + (txn) => ({ + modelIds: Array.from({ length: 4 }, (_, index) => + PhysicalModel.insert(txn, IModel.rootSubjectId, `QuickModel-${index}`) + ), + categoryIds: Array.from({ length: 4 }, (_, index) => + SpatialCategory.insert( + txn, + IModel.dictionaryId, + `QuickCategory-${index}`, + {} + ) + ), + }) + ); + + const elementIds: Id64String[] = []; + for ( + let index = 0; + index < descriptor.distribution.base.elements; + index++ + ) { + const id = withEditTxn(db, `insert base element ${index}`, (txn) => + txn.insertElement( + elementProps( + modelIds[index % modelIds.length], + categoryIds[index % categoryIds.length], + index, + index % elementsPerUnit < + descriptor.distribution.base.geometricElements / + fixtureScale(descriptor) + ) + ) + ); + elementIds.push(id); + insertAspects(db, id, index); + } + + const relationshipIds = withEditTxn( + db, + "insert base relationships", + (txn) => + Array.from( + { length: descriptor.distribution.base.relationships }, + (_, index) => { + const unit = Math.floor(index / relationshipsPerUnit); + const unitOffset = index % relationshipsPerUnit; + const elementIndex = unit * elementsPerUnit + unitOffset; + const relationship = ElementGroupsMembers.create( + db, + elementIds[elementIndex], + elementIds[elementIndex + 1], + unitOffset + ); + return txn.insertRelationship(relationship.toJSON()); + } + ) + ); + return { categoryIds, elementIds, modelIds, relationshipIds }; + } finally { + db.close(); + } +} + +async function push(db: BriefcaseDb, accessToken: string, description: string) { + await db.pushChanges({ accessToken, description }); +} + +export async function applyBalancedChangesets( + db: BriefcaseDb, + accessToken: string, + descriptor: DatasetDescriptor, + state: BalancedRecipeState +): Promise { + const scale = fixtureScale(descriptor); + const insertedIds: Id64String[] = Array.from({ + length: descriptor.distribution.operations.elements.inserts, + }); + for (let batch = 0; batch < 2; batch++) { + withEditTxn(db, `insert delta element batch ${batch + 1}`, (txn) => { + for (let unit = 0; unit < scale; unit++) { + for (let offset = 0; offset < 12; offset++) { + const insertedIndex = + unit * insertedElementsPerUnit + batch * 12 + offset; + const index = descriptor.distribution.base.elements + insertedIndex; + const id = txn.insertElement( + elementProps( + state.modelIds[index % state.modelIds.length], + state.categoryIds[index % state.categoryIds.length], + index, + offset % 2 === 0 + ) + ); + insertedIds[insertedIndex] = id; + txn.insertAspect({ + classFullName: multiAspectClass, + element: new ElementOwnsMultiAspects(id), + payload: `inserted-aspect-${insertedIndex}`, + sequence: insertedIndex, + } as ElementAspectProps); + } + } + }); + await push( + db, + accessToken, + `quick delta ${batch + 1}: element/aspect inserts` + ); + } + + for (let batch = 0; batch < 2; batch++) { + withEditTxn(db, `update delta element batch ${batch + 1}`, (txn) => { + for (let unit = 0; unit < scale; unit++) { + for (let offset = 0; offset < 12; offset++) { + const unitOffset = batch * 12 + offset; + const index = unit * elementsPerUnit + unitOffset; + const ownerId = state.elementIds[index]; + const props = + db.elements.getElementProps(ownerId); + const geometryUpdate = unitOffset < 6; + txn.updateElement({ + ...props, + geom: geometryUpdate + ? createBoxGeometry(unitOffset + 2) + : props.geom, + placement: geometryUpdate + ? { + angles: YawPitchRollAngles.createDegrees(0, 0, unitOffset), + origin: Point3d.create(index + 1000, index, 0), + } + : props.placement, + userLabel: `updated-${index}`, + } as PhysicalElementProps); + const aspect = db.elements.getAspects(ownerId, multiAspectClass)[0]; + txn.updateAspect({ + ...aspect.toJSON(), + payload: `updated-aspect-${index}`, + } as ElementAspectProps); + } + } + }); + await push( + db, + accessToken, + `quick delta ${batch + 3}: element/aspect updates` + ); + } + + withEditTxn(db, "insert delta relationships", (txn) => { + for (let unit = 0; unit < scale; unit++) { + for (let index = 0; index < 12; index++) + txn.insertRelationship( + ElementGroupsMembers.create( + db, + insertedIds[unit * insertedElementsPerUnit + index], + state.elementIds[unit * elementsPerUnit + index + 48], + index + ).toJSON() + ); + } + }); + await push(db, accessToken, "quick delta 5: relationship inserts"); + + withEditTxn(db, "update delta relationships", (txn) => { + for (let unit = 0; unit < scale; unit++) { + for (let index = 0; index < 12; index++) { + const relationship = db.relationships.getInstance( + ElementGroupsMembers.classFullName, + state.relationshipIds[unit * relationshipsPerUnit + index] + ); + relationship.memberPriority += 1000; + txn.updateRelationship(relationship.toJSON()); + } + } + }); + await push(db, accessToken, "quick delta 6: relationship updates"); + + withEditTxn(db, "delete delta relationships", (txn) => { + for (let unit = 0; unit < scale; unit++) { + for (let index = 24; index < 36; index++) { + const relationship = db.relationships.getInstance( + ElementGroupsMembers.classFullName, + state.relationshipIds[unit * relationshipsPerUnit + index] + ); + txn.deleteRelationship(relationship.toJSON()); + } + } + }); + await push(db, accessToken, "quick delta 7: relationship deletes"); + + withEditTxn(db, "delete delta elements and owned aspects", (txn) => { + for (let unit = 0; unit < scale; unit++) { + for (let index = 100; index < 124; index++) + txn.deleteElement(state.elementIds[unit * elementsPerUnit + index]); + } + }); + await push(db, accessToken, "quick delta 8: element/aspect cascade deletes"); +} diff --git a/packages/performance-tests/test/quick/scenarios/incrementalSynchronization.ts b/packages/performance-tests/test/quick/scenarios/incrementalSynchronization.ts new file mode 100644 index 000000000..23afacf36 --- /dev/null +++ b/packages/performance-tests/test/quick/scenarios/incrementalSynchronization.ts @@ -0,0 +1,55 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { IModelTransformer } from "@itwin/imodel-transformer"; +import { HubMock } from "@itwin/core-backend/lib/cjs/internal/HubMock"; +import { PreparedDataset } from "../FixtureMaterializer"; +import { createStartedEditTxn } from "../LocalHubFixture"; +import { + assertSemanticallyEqual, + assertSynchronizationProvenance, +} from "../validation/validateFixture"; + +export interface PreparedIncrementalSynchronization { + abort(): void; + finish(): Promise; + measure(): Promise; +} + +export function incrementalSynchronization( + dataset: PreparedDataset +): PreparedIncrementalSynchronization { + if (!HubMock.isValid) + throw new Error("Quick performance scenarios require an active HubMock"); + const editTxn = createStartedEditTxn(dataset.hub.targetDb); + const transformer = new IModelTransformer( + { source: dataset.hub.sourceDb, target: editTxn }, + { argsForProcessChanges: {} } + ); + let disposed = false; + const dispose = () => { + if (disposed) return; + transformer.dispose(); + if (editTxn.isActive) editTxn.end(); + disposed = true; + }; + return { + abort: dispose, + async measure() { + await transformer.process(); + }, + async finish() { + dispose(); + await assertSynchronizationProvenance( + dataset.hub.sourceDb, + dataset.hub.targetDb + ); + return assertSemanticallyEqual( + dataset.hub.sourceDb, + dataset.hub.targetDb + ); + }, + }; +} diff --git a/packages/performance-tests/test/quick/schemas/QuickPerf.ecschema.xml b/packages/performance-tests/test/quick/schemas/QuickPerf.ecschema.xml new file mode 100644 index 000000000..059e6b3e0 --- /dev/null +++ b/packages/performance-tests/test/quick/schemas/QuickPerf.ecschema.xml @@ -0,0 +1,17 @@ + + + + + + bis:ElementUniqueAspect + + + + + + bis:ElementMultiAspect + + + + diff --git a/packages/performance-tests/test/quick/statistics.quick-unit.ts b/packages/performance-tests/test/quick/statistics.quick-unit.ts new file mode 100644 index 000000000..fd09fe65f --- /dev/null +++ b/packages/performance-tests/test/quick/statistics.quick-unit.ts @@ -0,0 +1,28 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { expect } from "chai"; +import { classifyVariance } from "./BenchmarkReporter"; +import { + coefficientOfVariation, + median, + medianAbsoluteDeviation, + percentile, +} from "./validation/statistics"; + +describe("quick performance statistics", () => { + it("calculates robust summary statistics", () => { + expect(median([4, 1, 3, 2])).to.equal(2.5); + expect(percentile([1, 2, 3, 4, 5], 0.9)).to.equal(4.6); + expect(medianAbsoluteDeviation([1, 2, 3, 4, 100])).to.equal(1); + expect(coefficientOfVariation([10, 10, 10])).to.equal(0); + }); + + it("requires coefficient of variation at or below five percent", () => { + expect(classifyVariance(8, 0.05, 0.05)).to.equal("stable"); + expect(classifyVariance(8, 0.050_001, 0.01)).to.equal("unstable"); + expect(classifyVariance(1, 0, 0)).to.equal("insufficient-samples"); + }); +}); diff --git a/packages/performance-tests/test/quick/validation/statistics.ts b/packages/performance-tests/test/quick/validation/statistics.ts new file mode 100644 index 000000000..830cef711 --- /dev/null +++ b/packages/performance-tests/test/quick/validation/statistics.ts @@ -0,0 +1,38 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +function sorted(values: readonly number[]): number[] { + if (values.length === 0) throw new Error("At least one sample is required"); + return [...values].sort((left, right) => left - right); +} + +export function percentile( + values: readonly number[], + probability: number +): number { + if (probability < 0 || probability > 1) + throw new Error("Probability must be between zero and one"); + const ordered = sorted(values); + const index = (ordered.length - 1) * probability; + const lower = Math.floor(index); + const upper = Math.ceil(index); + return ordered[lower] + (ordered[upper] - ordered[lower]) * (index - lower); +} + +export function median(values: readonly number[]): number { + return percentile(values, 0.5); +} + +export function medianAbsoluteDeviation(values: readonly number[]): number { + const center = median(values); + return median(values.map((value) => Math.abs(value - center))); +} + +export function coefficientOfVariation(values: readonly number[]): number { + const mean = values.reduce((sum, value) => sum + value, 0) / values.length; + const variance = + values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / values.length; + return Math.sqrt(variance) / mean; +} diff --git a/packages/performance-tests/test/quick/validation/validateFixture.ts b/packages/performance-tests/test/quick/validation/validateFixture.ts new file mode 100644 index 000000000..464f2e377 --- /dev/null +++ b/packages/performance-tests/test/quick/validation/validateFixture.ts @@ -0,0 +1,366 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { + BriefcaseDb, + ExternalSourceAspect, + PhysicalObject, +} from "@itwin/core-backend"; +import { IModel, QueryBinder } from "@itwin/core-common"; +import { YawPitchRollAngles } from "@itwin/core-geometry"; +import { DatasetDescriptor } from "../DatasetDescriptor"; +import { canonicalSha256 } from "../FixtureManifest"; +import { createBoxGeometry } from "../recipes/balancedIncremental"; + +function normalizedGeometryStream(geometry: unknown): unknown { + if (!Array.isArray(geometry)) return geometry; + return geometry.filter((entry: unknown) => { + if (typeof entry !== "object" || entry === null) return true; + const properties = Object.keys(entry); + if (properties.length !== 1 || properties[0] !== "header") return true; + const header = (entry as { header?: unknown }).header; + return !( + typeof header === "object" && + header !== null && + Object.keys(header).length === 1 && + (header as { flags?: unknown }).flags === 0 + ); + }); +} + +function normalizedAngles(value: unknown): unknown { + if (typeof value === "number") return Number(value.toFixed(12)); + if (Array.isArray(value)) return value.map(normalizedAngles); + if (typeof value === "object" && value !== null) + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => [ + key, + normalizedAngles(entry), + ]) + ); + return value; +} + +async function queryValues( + db: BriefcaseDb, + ecsql: string, + propertyName: string +): Promise { + const values: unknown[] = []; + const reader = db.createQueryReader(ecsql, undefined, { + usePrimaryConn: true, + }); + while (await reader.step()) values.push(reader.current[propertyName]); + return values; +} + +async function queryRecords( + db: BriefcaseDb, + ecsql: string, + propertyNames: readonly string[] +): Promise { + const values: unknown[] = []; + const reader = db.createQueryReader(ecsql, undefined, { + usePrimaryConn: true, + }); + while (await reader.step()) { + values.push( + Object.fromEntries( + propertyNames.map((propertyName) => [ + propertyName, + reader.current[propertyName], + ]) + ) + ); + } + return values; +} + +async function queryGeometryRecords(db: BriefcaseDb): Promise { + const values: unknown[] = []; + const reader = db.createQueryReader( + "SELECT ECInstanceId id FROM Generic.PhysicalObject WHERE UserLabel IS NOT NULL ORDER BY UserLabel", + undefined, + { usePrimaryConn: true } + ); + while (await reader.step()) { + const element = db.elements.getElement({ + id: reader.current.id as string, + wantGeometry: true, + }); + const range = element.calculateRange3d(); + values.push({ + angles: normalizedAngles(element.placement.angles.toJSON()), + geometry: normalizedGeometryStream(element.geom), + hasGeometry: element.geom !== undefined, + label: element.userLabel, + origin: element.placement.origin.toJSON(), + range: range.isNull + ? undefined + : { + high: range.high.toJSON(), + low: range.low.toJSON(), + }, + }); + } + return values; +} + +async function queryCount(db: BriefcaseDb, ecsql: string): Promise { + const reader = db.createQueryReader(ecsql, undefined, { + usePrimaryConn: true, + }); + if (!(await reader.step())) + throw new Error(`Count query returned no rows: ${ecsql}`); + return reader.current.cnt as number; +} + +async function queryGeometryUpdateCount(db: BriefcaseDb): Promise { + const reader = db.createQueryReader( + "SELECT ECInstanceId id FROM Generic.PhysicalObject WHERE UserLabel LIKE 'updated-%'", + undefined, + { usePrimaryConn: true } + ); + let count = 0; + while (await reader.step()) { + const element = db.elements.getElement({ + id: reader.current.id as string, + wantGeometry: true, + }); + const match = /^updated-(\d+)$/.exec(element.userLabel ?? ""); + if (!match) continue; + const index = Number(match[1]); + const unitOffset = index % 240; + if (unitOffset >= 6) continue; + const range = element.calculateRange3d(); + const expectedLength = unitOffset + 2; + const expectedAngles = YawPitchRollAngles.createDegrees( + 0, + 0, + unitOffset + ).toJSON(); + if ( + element.placement.origin.x !== index + 1000 || + element.placement.origin.y !== index || + element.placement.origin.z !== 0 + ) + continue; + const actualAnglesHash = canonicalSha256( + normalizedAngles(element.placement.angles.toJSON()) + ); + const expectedAnglesHash = canonicalSha256( + normalizedAngles(expectedAngles) + ); + if (actualAnglesHash !== expectedAnglesHash) + throw new Error( + `Geometry update angle mismatch for ${element.userLabel}: expected=${expectedAnglesHash}, actual=${actualAnglesHash}` + ); + const actualGeometryHash = canonicalSha256( + normalizedGeometryStream(element.geom) + ); + const expectedGeometryHash = canonicalSha256( + normalizedGeometryStream(createBoxGeometry(expectedLength)) + ); + if (actualGeometryHash !== expectedGeometryHash) + throw new Error( + `Geometry update stream mismatch for ${element.userLabel}: expected=${expectedGeometryHash}, actual=${actualGeometryHash}` + ); + if ( + range.isNull || + range.high.x <= range.low.x || + range.high.y <= range.low.y || + range.high.z <= range.low.z + ) + throw new Error( + `Geometry update has an invalid range: ${element.userLabel}` + ); + count++; + } + return count; +} + +export async function assertFixtureDistribution( + db: BriefcaseDb, + descriptor: DatasetDescriptor +): Promise { + const expected = { + elements: + descriptor.distribution.base.elements + + descriptor.distribution.operations.elements.inserts - + descriptor.distribution.operations.elements.deletes, + multiAspects: + descriptor.distribution.base.elements + + descriptor.distribution.operations.aspects.inserts - + descriptor.distribution.operations.elements.deletes, + relationships: + descriptor.distribution.base.relationships + + descriptor.distribution.operations.relationships.inserts - + descriptor.distribution.operations.relationships.deletes, + relationshipUpdates: + descriptor.distribution.operations.relationships.updates, + elementUpdates: descriptor.distribution.operations.elements.updates, + aspectUpdates: descriptor.distribution.operations.aspects.updates, + geometryUpdates: descriptor.distribution.operations.geometryUpdates, + uniqueAspects: + descriptor.distribution.base.elements - + descriptor.distribution.operations.elements.deletes, + }; + const actual = { + elements: await queryCount( + db, + "SELECT count(*) cnt FROM Generic.PhysicalObject" + ), + multiAspects: await queryCount( + db, + "SELECT count(*) cnt FROM QuickPerf.BalancedMultiAspect" + ), + relationships: await queryCount( + db, + "SELECT count(*) cnt FROM bis.ElementGroupsMembers" + ), + relationshipUpdates: await queryCount( + db, + "SELECT count(*) cnt FROM bis.ElementGroupsMembers WHERE MemberPriority>=1000" + ), + elementUpdates: await queryCount( + db, + "SELECT count(*) cnt FROM Generic.PhysicalObject WHERE UserLabel LIKE 'updated-%'" + ), + aspectUpdates: await queryCount( + db, + "SELECT count(*) cnt FROM QuickPerf.BalancedMultiAspect WHERE Payload LIKE 'updated-aspect-%'" + ), + geometryUpdates: await queryGeometryUpdateCount(db), + uniqueAspects: await queryCount( + db, + "SELECT count(*) cnt FROM QuickPerf.BalancedUniqueAspect" + ), + }; + if (JSON.stringify(actual) !== JSON.stringify(expected)) + throw new Error( + `Fixture distribution mismatch: expected=${JSON.stringify( + expected + )}, actual=${JSON.stringify(actual)}` + ); + if ( + db.changeset.index !== descriptor.distribution.operations.sourceChangesets + ) + throw new Error( + `Expected ${descriptor.distribution.operations.sourceChangesets} source changesets, got ${db.changeset.index}` + ); +} + +export async function assertSynchronizationProvenance( + sourceDb: BriefcaseDb, + targetDb: BriefcaseDb +): Promise { + const binder = new QueryBinder() + .bindId("elementId", IModel.rootSubjectId) + .bindId("scopeId", IModel.rootSubjectId) + .bindString("kind", ExternalSourceAspect.Kind.Scope) + .bindString("identifier", sourceDb.iModelId); + const reader = targetDb.createQueryReader( + `SELECT Identifier identifier,Version version + FROM bis.ExternalSourceAspect + WHERE Element.Id=:elementId + AND Scope.Id=:scopeId + AND Kind=:kind + AND Identifier=:identifier`, + binder, + { usePrimaryConn: true } + ); + if (!(await reader.step())) + throw new Error("Target synchronization scope provenance was not found"); + const identifier = reader.current.identifier as string; + const version = reader.current.version as string; + if (await reader.step()) + throw new Error( + "Target contains duplicate synchronization scope provenance" + ); + if (identifier !== sourceDb.iModelId) + throw new Error( + `Synchronization source mismatch: expected=${sourceDb.iModelId}, actual=${identifier}` + ); + const expectedVersion = `${sourceDb.changeset.id};${sourceDb.changeset.index}`; + if (version !== expectedVersion) + throw new Error( + `Synchronization version mismatch: expected=${expectedVersion}, actual=${version}` + ); +} + +async function semanticContent(db: BriefcaseDb) { + return { + aspects: await queryRecords( + db, + "SELECT a.Payload payload,a.Sequence sequence,e.UserLabel owner FROM QuickPerf.BalancedMultiAspect a JOIN bis.Element e ON e.ECInstanceId=a.Element.Id ORDER BY e.UserLabel,a.Payload", + ["owner", "payload", "sequence"] + ), + elementLabels: await queryValues( + db, + "SELECT UserLabel label FROM Generic.PhysicalObject WHERE UserLabel IS NOT NULL ORDER BY UserLabel", + "label" + ), + geometry: await queryGeometryRecords(db), + relationships: await queryRecords( + db, + "SELECT s.UserLabel sourceLabel,t.UserLabel targetLabel,r.MemberPriority priority FROM bis.ElementGroupsMembers r JOIN bis.Element s ON s.ECInstanceId=r.SourceECInstanceId JOIN bis.Element t ON t.ECInstanceId=r.TargetECInstanceId ORDER BY s.UserLabel,t.UserLabel,r.MemberPriority", + ["priority", "sourceLabel", "targetLabel"] + ), + uniqueAspects: await queryRecords( + db, + "SELECT a.Payload payload,a.Sequence sequence,e.UserLabel owner FROM QuickPerf.BalancedUniqueAspect a JOIN bis.Element e ON e.ECInstanceId=a.Element.Id ORDER BY e.UserLabel,a.Payload", + ["owner", "payload", "sequence"] + ), + }; +} + +export async function semanticDigest(db: BriefcaseDb): Promise { + return canonicalSha256(await semanticContent(db)); +} + +export async function assertSemanticallyEqual( + sourceDb: BriefcaseDb, + targetDb: BriefcaseDb +): Promise { + const [sourceContent, targetContent] = await Promise.all([ + semanticContent(sourceDb), + semanticContent(targetDb), + ]); + const sourceDigest = canonicalSha256(sourceContent); + const targetDigest = canonicalSha256(targetContent); + if (sourceDigest !== targetDigest) + throw new Error( + `Fixture semantic digest mismatch: source=${sourceDigest}, target=${targetDigest}, sourceCounts=${JSON.stringify( + Object.fromEntries( + Object.entries(sourceContent).map(([key, values]) => [ + key, + values.length, + ]) + ) + )}, targetCounts=${JSON.stringify( + Object.fromEntries( + Object.entries(targetContent).map(([key, values]) => [ + key, + values.length, + ]) + ) + )}, categoryHashes=${JSON.stringify( + Object.fromEntries( + Object.keys(sourceContent).map((key) => [ + key, + { + source: canonicalSha256( + sourceContent[key as keyof typeof sourceContent] + ), + target: canonicalSha256( + targetContent[key as keyof typeof targetContent] + ), + }, + ]) + ) + )}` + ); + return sourceDigest; +} From 1601b6df037b3fc049b44660234f12ea0ca44726 Mon Sep 17 00:00:00 2001 From: Nam Le <50554904+hl662@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:35:52 -0400 Subject: [PATCH 02/19] perf(tests): support selectable quick scenarios Use an injected scenario factory so additional timed operations can share the same deterministic fixture and reporting lifecycle. Run the manual benchmark on Linux and record scenario identity in every report format. Refs #328 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/quick-performance.yml | 79 +++++++--- packages/performance-tests/README.md | 47 ++++-- .../test/quick/BenchmarkReporter.ts | 13 +- .../test/quick/BenchmarkRunner.ts | 16 +- .../test/quick/BenchmarkScenario.ts | 21 +++ .../test/quick/LocalHubFixture.quick-unit.ts | 149 +++++++++++++++++- .../test/quick/QuickPerformance.quick.ts | 10 +- .../test/quick/ScenarioCatalog.quick-unit.ts | 27 ++++ .../test/quick/ScenarioCatalog.ts | 28 ++++ packages/performance-tests/test/quick/cli.ts | 5 +- .../scenarios/incrementalSynchronization.ts | 17 +- 11 files changed, 358 insertions(+), 54 deletions(-) create mode 100644 packages/performance-tests/test/quick/BenchmarkScenario.ts create mode 100644 packages/performance-tests/test/quick/ScenarioCatalog.quick-unit.ts create mode 100644 packages/performance-tests/test/quick/ScenarioCatalog.ts diff --git a/.github/workflows/quick-performance.yml b/.github/workflows/quick-performance.yml index b7d376dec..1f6cc5130 100644 --- a/.github/workflows/quick-performance.yml +++ b/.github/workflows/quick-performance.yml @@ -2,16 +2,31 @@ name: Quick performance on: workflow_dispatch: + inputs: + scenario: + description: Quick performance scenario + required: true + default: incremental-synchronization + type: choice + options: + - incremental-synchronization + +permissions: + contents: read jobs: - balanced-incremental: - name: Balanced incremental - runs-on: windows-latest + quick-performance: + name: ${{ inputs.scenario }} + runs-on: ubuntu-latest timeout-minutes: 30 + env: + NODE_OPTIONS: --disable-warning=MODULE_TYPELESS_PACKAGE_JSON steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false - name: Install pnpm uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 @@ -29,35 +44,51 @@ jobs: run: pnpm --dir packages/transformer build:cjs - name: Validate quick harness - run: >- - pnpm --dir packages/performance-tests exec mocha - --timeout 300000 - --require ts-node/register - test/quick/FixtureManifest.quick-unit.ts - test/quick/statistics.quick-unit.ts - test/quick/LocalHubFixture.quick-unit.ts + run: | + pnpm --dir packages/performance-tests exec mocha \ + --timeout 300000 \ + --require ts-node/register \ + test/quick/FixtureManifest.quick-unit.ts \ + test/quick/ScenarioCatalog.quick-unit.ts \ + test/quick/statistics.quick-unit.ts \ + test/quick/LocalHubFixture.quick-unit.ts \ + 2>&1 | sed \ + -e "s#${GITHUB_WORKSPACE}##g" \ + -e "s#${RUNNER_TEMP}##g" - name: Run quick performance suite - run: pnpm --dir packages/performance-tests test:quick + run: | + pnpm --dir packages/performance-tests test:quick \ + 2>&1 | sed \ + -e "s#${GITHUB_WORKSPACE}##g" \ + -e "s#${RUNNER_TEMP}##g" env: QUICK_PERF_OUTPUT: ${{ runner.temp }}/quick-performance + QUICK_PERF_SCENARIO: ${{ inputs.scenario }} - name: Report variance reliability if: always() - shell: pwsh + shell: bash run: | - $report = "${{ runner.temp }}/quick-performance/summary.json" - if (Test-Path $report) { - $summary = Get-Content $report | ConvertFrom-Json - "### Quick performance reliability" >> $env:GITHUB_STEP_SUMMARY - "Status: **$($summary.varianceStatus)**" >> $env:GITHUB_STEP_SUMMARY - "CV: $($summary.wallMilliseconds.coefficientOfVariation)" >> $env:GITHUB_STEP_SUMMARY - "Normalized MAD: $($summary.wallMilliseconds.normalizedMad)" >> $env:GITHUB_STEP_SUMMARY - "Unstable samples: $($summary.unstableSamples -join ', ')" >> $env:GITHUB_STEP_SUMMARY - if ($summary.varianceStatus -ne "stable") { - Write-Output "::warning title=Unstable quick performance variance::Do not use this run as regression evidence." - } - } + node <<'NODE' + const fs = require("fs"); + const path = require("path"); + const report = path.join(process.env.RUNNER_TEMP, "quick-performance", "summary.json"); + if (!fs.existsSync(report)) + process.exit(0); + const summary = JSON.parse(fs.readFileSync(report, "utf8")); + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, [ + "### Quick performance reliability", + `Scenario: **${summary.scenarioId}**`, + `Status: **${summary.varianceStatus}**`, + `CV: ${summary.wallMilliseconds.coefficientOfVariation}`, + `Normalized MAD: ${summary.wallMilliseconds.normalizedMad}`, + `Unstable samples: ${summary.unstableSamples.join(", ")}`, + "", + ].join("\n")); + if (summary.varianceStatus !== "stable") + console.log("::warning title=Unstable quick performance variance::Do not use this run as regression evidence."); + NODE - name: Publish report if: always() diff --git a/packages/performance-tests/README.md b/packages/performance-tests/README.md index c16eaf3e7..d08efd4a6 100644 --- a/packages/performance-tests/README.md +++ b/packages/performance-tests/README.md @@ -23,10 +23,16 @@ pnpm --dir ../transformer build:cjs pnpm test:quick ``` -The default is one warm-up plus eight measured samples. Set -`QUICK_PERF_SAMPLES` only for local diagnostics. Reports are written under -`test/quick/.quick-output/` unless `QUICK_PERF_OUTPUT` is set and include -`samples.jsonl`, `summary.json`, and `summary.csv`. +The default scenario is `incremental-synchronization`, selected with +`QUICK_PERF_SCENARIO`. Unknown scenario names fail before fixture +reconstruction. The default run is sample 0 as a warm-up plus eight measured +samples. Each sample is a fresh reconstruction, one timed scenario execution, +and untimed verification and cleanup. The warm-up follows the same lifecycle +but is excluded from summary timing statistics. Set `QUICK_PERF_SAMPLES` only +for local diagnostics. Reports are written under `test/quick/.quick-output/` +unless `QUICK_PERF_OUTPUT` is set and include `samples.jsonl`, `summary.json`, +and `summary.csv`. Every sample and report includes the scenario ID, and the +reporter rejects mixed-scenario sample sets. The calibrated fixture contains 6,000 base elements, 12,000 aspects, 3,000 relationships, and 3,000 geometry-bearing elements. Its eight changesets apply @@ -35,14 +41,31 @@ deletes, 300 relationship inserts/updates, 825 relationship deletes, and 150 geometry updates. This is 25 deterministic repetitions of one balanced content unit, preserving the original scenario ratios. -`varianceStatus` requires coefficient of variation and normalized MAD at or -below 5%. An unstable manual run emits a workflow warning and must not be used -as regression evidence. It does not currently fail the manual workflow: six -local calibration suites informed the final scale, and three ratio-correct final -suites on a shared workstation produced 1.38-1.48 second medians but only two -met the CV threshold. A hard gate would therefore have false-failed one of three -final runs. Revisit the failure policy after repeated measurements on the target -Windows runner or a dedicated performance agent. +`varianceStatus` requires coefficient of variation (standard deviation divided +by mean) and normalized MAD at or below 5%. CV can exceed the target when host +scheduling or background load delays one sample. An unstable manual run emits a +workflow warning and must not be used as regression evidence, but correctness +and the 15-minute budget remain hard gates. Variance does not currently fail the +manual workflow: six local calibration suites informed the final scale, and +three ratio-correct final suites on a shared workstation produced 1.38-1.48 +second medians but only two met the CV threshold. A hard gate would therefore +have false-failed one of three final runs. The GitHub workflow now targets +`ubuntu-latest`, but no Linux reliability evidence exists until that workflow +runs. Revisit the failure policy after repeated measurements on the hosted +Linux runner or a dedicated performance agent. + +The GitHub Actions workflow is manual-only. Select the branch with GitHub's +native workflow ref and the scenario with its dispatch input: + +```sh +gh workflow run quick-performance.yml --ref \ + -f scenario=incremental-synchronization +``` + +GitHub can dispatch this workflow only after `quick-performance.yml` exists on +the repository's default branch. The `--ref` selects which committed branch +revision runs after that requirement is met; there is intentionally no custom +branch input and no automatic pull-request or push trigger. `pnpm quick:build-fixture` writes the canonical recipe manifest. `pnpm quick:verify-fixture` performs two fresh reconstructions (warm-up plus one diff --git a/packages/performance-tests/test/quick/BenchmarkReporter.ts b/packages/performance-tests/test/quick/BenchmarkReporter.ts index 2151acdb2..8d54651f3 100644 --- a/packages/performance-tests/test/quick/BenchmarkReporter.ts +++ b/packages/performance-tests/test/quick/BenchmarkReporter.ts @@ -36,6 +36,15 @@ export class BenchmarkReporter { samples: readonly BenchmarkSample[], jobMilliseconds?: number ): void { + if (samples.length === 0) + throw new Error("Cannot report an empty quick performance sample set"); + const scenarioIds = new Set(samples.map((sample) => sample.scenarioId)); + if (scenarioIds.size !== 1) + throw new Error( + `Cannot mix quick performance scenarios in one report: ${[ + ...scenarioIds, + ].join(", ")}` + ); const measured = samples.filter((sample) => sample.measured); const walls = measured.map((sample) => sample.wallMilliseconds); const wallMedian = median(walls); @@ -53,6 +62,7 @@ export class BenchmarkReporter { fixtureId: measured[0]?.fixtureId, jobMilliseconds, measuredSamples: measured.length, + scenarioId: samples[0].scenarioId, varianceStatus: classifyVariance( measured.length, wallCoefficientOfVariation, @@ -98,8 +108,9 @@ export class BenchmarkReporter { fs.writeFileSync( path.join(outputDir, "summary.csv"), [ - "fixture,measuredSamples,jobMs,medianMs,p90Ms,p95Ms,madMs,cv,reconstructionTotalMs,verificationTotalMs,teardownTotalMs", + "scenario,fixture,measuredSamples,jobMs,medianMs,p90Ms,p95Ms,madMs,cv,reconstructionTotalMs,verificationTotalMs,teardownTotalMs", [ + summary.scenarioId, summary.fixtureId, summary.measuredSamples, summary.jobMilliseconds ?? "", diff --git a/packages/performance-tests/test/quick/BenchmarkRunner.ts b/packages/performance-tests/test/quick/BenchmarkRunner.ts index 6ff238685..48146058d 100644 --- a/packages/performance-tests/test/quick/BenchmarkRunner.ts +++ b/packages/performance-tests/test/quick/BenchmarkRunner.ts @@ -7,10 +7,13 @@ import * as fs from "fs"; import * as os from "os"; import * as path from "path"; import { IModelHost } from "@itwin/core-backend"; +import { + BenchmarkScenario, + BenchmarkScenarioDefinition, +} from "./BenchmarkScenario"; import { DatasetDescriptor } from "./DatasetDescriptor"; import { materializeFixture, PreparedDataset } from "./FixtureMaterializer"; import { disposeReconstructedHub } from "./LocalHubFixture"; -import { incrementalSynchronization } from "./scenarios/incrementalSynchronization"; export const benchmarkOutputMarkerName = ".imodel-transformer-quick-performance"; @@ -89,7 +92,7 @@ export function prepareBenchmarkOutputDirectory(outputDir: string): void { } async function cleanupSample( - scenario: ReturnType | undefined, + scenario: BenchmarkScenario | undefined, dataset: PreparedDataset | undefined, sampleDir: string ): Promise { @@ -121,6 +124,7 @@ export interface BenchmarkSample { readonly reconstructionMilliseconds: number; readonly rssDeltaBytes: number; readonly sample: number; + readonly scenarioId: string; readonly semanticDigest: string; readonly teardownMilliseconds: number; readonly verificationMilliseconds: number; @@ -130,7 +134,8 @@ export interface BenchmarkSample { export class BenchmarkRunner { public constructor( private readonly _descriptor: DatasetDescriptor, - private readonly _outputDir: string + private readonly _outputDir: string, + private readonly _scenario: BenchmarkScenarioDefinition ) {} public async run(measuredSamples = 8): Promise { @@ -145,7 +150,7 @@ export class BenchmarkRunner { for (let sample = 0; sample <= measuredSamples; sample++) { const sampleDir = path.join(this._outputDir, `sample-${sample}`); let dataset: PreparedDataset | undefined; - let scenario: ReturnType | undefined; + let scenario: BenchmarkScenario | undefined; let operationError: Error | undefined; let completedSample: | Omit @@ -156,7 +161,7 @@ export class BenchmarkRunner { sampleDir, `quick-sample-${sample}` ); - scenario = incrementalSynchronization(dataset); + scenario = this._scenario.factory(dataset); const rssBefore = process.memoryUsage().rss; const cpuBefore = process.cpuUsage(); const wallStart = process.hrtime.bigint(); @@ -178,6 +183,7 @@ export class BenchmarkRunner { reconstructionMilliseconds: dataset.reconstructionMilliseconds, rssDeltaBytes, sample, + scenarioId: this._scenario.id, semanticDigest, verificationMilliseconds, wallMilliseconds, diff --git a/packages/performance-tests/test/quick/BenchmarkScenario.ts b/packages/performance-tests/test/quick/BenchmarkScenario.ts new file mode 100644 index 000000000..ea153ece9 --- /dev/null +++ b/packages/performance-tests/test/quick/BenchmarkScenario.ts @@ -0,0 +1,21 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { PreparedDataset } from "./FixtureMaterializer"; + +export interface BenchmarkScenario { + abort(): void; + finish(): Promise; + measure(): Promise; +} + +export type BenchmarkScenarioFactory = ( + dataset: PreparedDataset +) => BenchmarkScenario; + +export interface BenchmarkScenarioDefinition { + readonly id: string; + readonly factory: BenchmarkScenarioFactory; +} diff --git a/packages/performance-tests/test/quick/LocalHubFixture.quick-unit.ts b/packages/performance-tests/test/quick/LocalHubFixture.quick-unit.ts index b3707d64e..59b2d28b0 100644 --- a/packages/performance-tests/test/quick/LocalHubFixture.quick-unit.ts +++ b/packages/performance-tests/test/quick/LocalHubFixture.quick-unit.ts @@ -23,6 +23,8 @@ import { } from "@itwin/core-backend"; import { HubMock } from "@itwin/core-backend/lib/cjs/internal/HubMock"; import { IModelTransformer } from "@itwin/imodel-transformer"; +import { BenchmarkReporter } from "./BenchmarkReporter"; +import { BenchmarkScenarioDefinition } from "./BenchmarkScenario"; import { benchmarkOutputMarkerName, BenchmarkRunner, @@ -36,6 +38,10 @@ import { ReconstructedHub, reconstructHub, } from "./LocalHubFixture"; +import { + incrementalSynchronization, + incrementalSynchronizationScenario, +} from "./scenarios/incrementalSynchronization"; import { assertSynchronizationProvenance } from "./validation/validateFixture"; function required(value: T | undefined, name: string): T { @@ -152,7 +158,8 @@ describe("LocalHubFixture reconstruction", () => { try { await new BenchmarkRunner( balancedIncrementalDescriptor, - unsafeOutput + unsafeOutput, + incrementalSynchronizationScenario ).run(1); } catch (error) { failure = error; @@ -181,7 +188,8 @@ describe("LocalHubFixture reconstruction", () => { try { await new BenchmarkRunner( balancedIncrementalDescriptor, - path.join(outputDir, "zero-samples") + path.join(outputDir, "zero-samples"), + incrementalSynchronizationScenario ).run(0); } catch (error) { failure = error; @@ -292,3 +300,140 @@ describe("LocalHubFixture reconstruction", () => { } }); }); + +describe("BenchmarkRunner scenario injection", function () { + this.timeout(5 * 60 * 1000); + + const testDescriptor = { + ...balancedIncrementalDescriptor, + id: "balanced-incremental-runner-test", + distribution: { + base: { + aspects: 480, + elements: 240, + geometricElements: 120, + relationships: 120, + }, + operations: { + aspects: { deletes: 48, inserts: 24, updates: 24 }, + elements: { deletes: 24, inserts: 24, updates: 24 }, + geometryUpdates: 6, + relationships: { deletes: 33, inserts: 12, updates: 12 }, + sourceChangesets: 8, + }, + }, + }; + + it("uses the injected factory, propagates its identity, and cleans every sample", async () => { + const outputDir = fs.mkdtempSync( + path.join(os.tmpdir(), "quick-perf-injected-") + ); + const calls = { abort: 0, factory: 0, finish: 0, measure: 0 }; + const scenario: BenchmarkScenarioDefinition = { + id: "injected-scenario", + factory: (dataset) => { + calls.factory++; + const delegate = incrementalSynchronization(dataset); + return { + abort() { + calls.abort++; + delegate.abort(); + }, + async finish() { + calls.finish++; + return delegate.finish(); + }, + async measure() { + calls.measure++; + await delegate.measure(); + }, + }; + }, + }; + try { + const samples = await new BenchmarkRunner( + testDescriptor, + outputDir, + scenario + ).run(1); + expect(calls).to.deep.equal({ + abort: 2, + factory: 2, + finish: 2, + measure: 2, + }); + expect(samples.map((sample) => sample.scenarioId)).to.deep.equal([ + scenario.id, + scenario.id, + ]); + expect(HubMock.isValid).to.be.false; + expect( + fs.readdirSync(outputDir).filter((entry) => entry.startsWith("sample-")) + ).to.be.empty; + + BenchmarkReporter.write(outputDir, samples); + const jsonLines = fs + .readFileSync(path.join(outputDir, "samples.jsonl"), "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as { scenarioId: string }); + expect(jsonLines.map((sample) => sample.scenarioId)).to.deep.equal([ + scenario.id, + scenario.id, + ]); + const summary = JSON.parse( + fs.readFileSync(path.join(outputDir, "summary.json"), "utf8") + ) as { scenarioId: string }; + expect(summary.scenarioId).to.equal(scenario.id); + expect( + fs.readFileSync(path.join(outputDir, "summary.csv"), "utf8") + ).to.match(/^scenario,fixture,.+\ninjected-scenario,/); + expect(() => + BenchmarkReporter.write(outputDir, [ + samples[0], + { ...samples[1], scenarioId: "different-scenario" }, + ]) + ).to.throw("Cannot mix quick performance scenarios"); + } finally { + fs.rmSync(outputDir, { recursive: true, force: true }); + } + }); + + it("aborts and tears down the reconstructed hub after a scenario failure", async () => { + const outputDir = fs.mkdtempSync( + path.join(os.tmpdir(), "quick-perf-failure-") + ); + let aborts = 0; + const scenario: BenchmarkScenarioDefinition = { + id: "failing-scenario", + factory: () => ({ + abort() { + aborts++; + }, + async finish() { + throw new Error("finish must not run"); + }, + async measure() { + throw new Error("expected scenario failure"); + }, + }), + }; + let failure: unknown; + try { + await new BenchmarkRunner(testDescriptor, outputDir, scenario).run(1); + } catch (error) { + failure = error; + } + try { + expect(failure).to.be.instanceOf(Error); + expect((failure as Error).message).to.equal("expected scenario failure"); + expect(aborts).to.equal(1); + expect(HubMock.isValid).to.be.false; + expect( + fs.readdirSync(outputDir).filter((entry) => entry.startsWith("sample-")) + ).to.be.empty; + } finally { + fs.rmSync(outputDir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/performance-tests/test/quick/QuickPerformance.quick.ts b/packages/performance-tests/test/quick/QuickPerformance.quick.ts index fe3d974de..14e6747cc 100644 --- a/packages/performance-tests/test/quick/QuickPerformance.quick.ts +++ b/packages/performance-tests/test/quick/QuickPerformance.quick.ts @@ -8,18 +8,21 @@ import { expect } from "chai"; import { BenchmarkReporter } from "./BenchmarkReporter"; import { BenchmarkRunner } from "./BenchmarkRunner"; import { getFixtureDescriptor } from "./FixtureCatalog"; +import { getScenarioDefinition } from "./ScenarioCatalog"; describe("quick transformer performance", function () { this.timeout(15 * 60 * 1000); it("runs the balanced incremental synchronization fixture", async () => { + const scenario = getScenarioDefinition(process.env.QUICK_PERF_SCENARIO); const measuredSamples = Number(process.env.QUICK_PERF_SAMPLES ?? "8"); const outputDir = process.env.QUICK_PERF_OUTPUT ?? - path.join(__dirname, ".quick-output", "balanced-incremental"); + path.join(__dirname, ".quick-output", scenario.id); const runner = new BenchmarkRunner( getFixtureDescriptor("balanced-incremental"), - outputDir + outputDir, + scenario ); const jobStart = process.hrtime.bigint(); const samples = await runner.run(measuredSamples); @@ -36,6 +39,9 @@ describe("quick transformer performance", function () { 1, "fresh reconstructions must produce the same semantic digest" ); + expect(new Set(samples.map((sample) => sample.scenarioId))).to.deep.equal( + new Set([scenario.id]) + ); expect(jobMilliseconds).to.be.lessThan(15 * 60 * 1000); }); }); diff --git a/packages/performance-tests/test/quick/ScenarioCatalog.quick-unit.ts b/packages/performance-tests/test/quick/ScenarioCatalog.quick-unit.ts new file mode 100644 index 000000000..65fa9087c --- /dev/null +++ b/packages/performance-tests/test/quick/ScenarioCatalog.quick-unit.ts @@ -0,0 +1,27 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { expect } from "chai"; +import { + defaultQuickPerformanceScenarioId, + getScenarioDefinition, +} from "./ScenarioCatalog"; + +describe("quick performance scenario catalog", () => { + it("selects incremental synchronization by default", () => { + expect(getScenarioDefinition().id).to.equal( + defaultQuickPerformanceScenarioId + ); + expect(defaultQuickPerformanceScenarioId).to.equal( + "incremental-synchronization" + ); + }); + + it("rejects unknown scenarios", () => { + expect(() => getScenarioDefinition("not-a-scenario")).to.throw( + 'Unknown quick performance scenario "not-a-scenario". Available scenarios: incremental-synchronization' + ); + }); +}); diff --git a/packages/performance-tests/test/quick/ScenarioCatalog.ts b/packages/performance-tests/test/quick/ScenarioCatalog.ts new file mode 100644 index 000000000..eb9f5d002 --- /dev/null +++ b/packages/performance-tests/test/quick/ScenarioCatalog.ts @@ -0,0 +1,28 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { BenchmarkScenarioDefinition } from "./BenchmarkScenario"; +import { incrementalSynchronizationScenario } from "./scenarios/incrementalSynchronization"; + +export const defaultQuickPerformanceScenarioId = + incrementalSynchronizationScenario.id; + +const scenarios = new Map([ + [incrementalSynchronizationScenario.id, incrementalSynchronizationScenario], +]); + +export function getScenarioDefinition( + requestedId?: string +): BenchmarkScenarioDefinition { + const scenarioId = requestedId ?? defaultQuickPerformanceScenarioId; + const scenario = scenarios.get(scenarioId); + if (!scenario) + throw new Error( + `Unknown quick performance scenario "${scenarioId}". Available scenarios: ${[ + ...scenarios.keys(), + ].join(", ")}` + ); + return scenario; +} diff --git a/packages/performance-tests/test/quick/cli.ts b/packages/performance-tests/test/quick/cli.ts index b6a1db9fd..85f2b39b0 100644 --- a/packages/performance-tests/test/quick/cli.ts +++ b/packages/performance-tests/test/quick/cli.ts @@ -11,6 +11,7 @@ import { prepareBenchmarkOutputDirectory, } from "./BenchmarkRunner"; import { balancedIncrementalDescriptor } from "./FixtureCatalog"; +import { getScenarioDefinition } from "./ScenarioCatalog"; function writeManifest(outputDir: string): void { fs.writeFileSync( @@ -30,9 +31,11 @@ async function main() { return; } if (command === "verify-fixture") { + const scenario = getScenarioDefinition(process.env.QUICK_PERF_SCENARIO); const samples = await new BenchmarkRunner( balancedIncrementalDescriptor, - outputDir + outputDir, + scenario ).run(1); if (new Set(samples.map((sample) => sample.semanticDigest)).size !== 1) throw new Error("Fixture reconstruction is not deterministic"); diff --git a/packages/performance-tests/test/quick/scenarios/incrementalSynchronization.ts b/packages/performance-tests/test/quick/scenarios/incrementalSynchronization.ts index 23afacf36..0bae9e2ea 100644 --- a/packages/performance-tests/test/quick/scenarios/incrementalSynchronization.ts +++ b/packages/performance-tests/test/quick/scenarios/incrementalSynchronization.ts @@ -5,6 +5,10 @@ import { IModelTransformer } from "@itwin/imodel-transformer"; import { HubMock } from "@itwin/core-backend/lib/cjs/internal/HubMock"; +import { + BenchmarkScenario, + BenchmarkScenarioDefinition, +} from "../BenchmarkScenario"; import { PreparedDataset } from "../FixtureMaterializer"; import { createStartedEditTxn } from "../LocalHubFixture"; import { @@ -12,15 +16,9 @@ import { assertSynchronizationProvenance, } from "../validation/validateFixture"; -export interface PreparedIncrementalSynchronization { - abort(): void; - finish(): Promise; - measure(): Promise; -} - export function incrementalSynchronization( dataset: PreparedDataset -): PreparedIncrementalSynchronization { +): BenchmarkScenario { if (!HubMock.isValid) throw new Error("Quick performance scenarios require an active HubMock"); const editTxn = createStartedEditTxn(dataset.hub.targetDb); @@ -53,3 +51,8 @@ export function incrementalSynchronization( }, }; } + +export const incrementalSynchronizationScenario: BenchmarkScenarioDefinition = { + id: "incremental-synchronization", + factory: incrementalSynchronization, +}; From fdd97d3301d642399d489d9dac9280e55ca2ff9e Mon Sep 17 00:00:00 2001 From: Nam Le <50554904+hl662@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:39:19 -0400 Subject: [PATCH 03/19] ci: trigger quick perf branch validation Temporarily run the synthetic quick-performance workflow from the feature branch before workflow_dispatch is registered on the default branch. Refs #328 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/quick-performance.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/quick-performance.yml b/.github/workflows/quick-performance.yml index 1f6cc5130..349501209 100644 --- a/.github/workflows/quick-performance.yml +++ b/.github/workflows/quick-performance.yml @@ -1,6 +1,9 @@ name: Quick performance on: + push: + branches: + - hl662-plan-quick-perf-pipeline workflow_dispatch: inputs: scenario: @@ -16,7 +19,7 @@ permissions: jobs: quick-performance: - name: ${{ inputs.scenario }} + name: ${{ inputs.scenario || 'incremental-synchronization' }} runs-on: ubuntu-latest timeout-minutes: 30 env: @@ -64,7 +67,7 @@ jobs: -e "s#${RUNNER_TEMP}##g" env: QUICK_PERF_OUTPUT: ${{ runner.temp }}/quick-performance - QUICK_PERF_SCENARIO: ${{ inputs.scenario }} + QUICK_PERF_SCENARIO: ${{ inputs.scenario || 'incremental-synchronization' }} - name: Report variance reliability if: always() From 0ee398fa737b710511c19974d9fa9070ad90b614 Mon Sep 17 00:00:00 2001 From: Nam Le <50554904+hl662@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:45:53 -0400 Subject: [PATCH 04/19] ci: remove temporary quick perf trigger Refs #328 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/quick-performance.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/quick-performance.yml b/.github/workflows/quick-performance.yml index 349501209..597ae6c7f 100644 --- a/.github/workflows/quick-performance.yml +++ b/.github/workflows/quick-performance.yml @@ -1,9 +1,6 @@ name: Quick performance on: - push: - branches: - - hl662-plan-quick-perf-pipeline workflow_dispatch: inputs: scenario: From d14caf847c03f168e71954fd696d126050bbe5e5 Mon Sep 17 00:00:00 2001 From: Nam Le <50554904+hl662@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:55:26 -0400 Subject: [PATCH 05/19] perf(tests): add scan oracle and scenario budgets Groundwork for a second quick-performance scenario that measures ChangedInstanceIds.initialize over changeset files. The scan oracle predicts a scan result from the ops a recipe performed, replaying them through an independently written squash so the check does not validate the transformer against itself. It covers all six ChangedInstanceOps collections; verifying only elements would let aspect and relationship regressions pass silently. Scenario budget moves onto the scenario definition, replacing the three places the 15 minute figure was hardcoded. The workflow job timeout stays fixed: it covers checkout, install and build as well, so it is a different quantity from the measured budget. Reports now stamp the transformer and core-backend versions that ran the measurement, which a fixture descriptor cannot supply because it records the versions that generated the fixture instead. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/quick-performance.yml | 3 + packages/performance-tests/package.json | 2 +- .../test/quick/BenchmarkReporter.ts | 7 +- .../test/quick/BenchmarkScenario.ts | 19 ++ .../test/quick/FixtureCatalog.ts | 14 +- .../test/quick/QuickPerformance.quick.ts | 13 +- .../test/quick/ScenarioCatalog.quick-unit.ts | 28 +++ .../test/quick/scanOracle.quick-unit.ts | 181 ++++++++++++++++ .../test/quick/validation/scanOracle.ts | 199 ++++++++++++++++++ .../performance-tests/test/quick/versions.ts | 31 +++ 10 files changed, 479 insertions(+), 18 deletions(-) create mode 100644 packages/performance-tests/test/quick/scanOracle.quick-unit.ts create mode 100644 packages/performance-tests/test/quick/validation/scanOracle.ts create mode 100644 packages/performance-tests/test/quick/versions.ts diff --git a/.github/workflows/quick-performance.yml b/.github/workflows/quick-performance.yml index 597ae6c7f..165bb9713 100644 --- a/.github/workflows/quick-performance.yml +++ b/.github/workflows/quick-performance.yml @@ -10,6 +10,7 @@ on: type: choice options: - incremental-synchronization + - changeset-scanning permissions: contents: read @@ -50,6 +51,7 @@ jobs: --require ts-node/register \ test/quick/FixtureManifest.quick-unit.ts \ test/quick/ScenarioCatalog.quick-unit.ts \ + test/quick/scanOracle.quick-unit.ts \ test/quick/statistics.quick-unit.ts \ test/quick/LocalHubFixture.quick-unit.ts \ 2>&1 | sed \ @@ -81,6 +83,7 @@ jobs: "### Quick performance reliability", `Scenario: **${summary.scenarioId}**`, `Status: **${summary.varianceStatus}**`, + `Transformer: ${summary.versions?.transformer} / core-backend: ${summary.versions?.coreBackend} / node: ${summary.versions?.node}`, `CV: ${summary.wallMilliseconds.coefficientOfVariation}`, `Normalized MAD: ${summary.wallMilliseconds.normalizedMad}`, `Unstable samples: ${summary.unstableSamples.join(", ")}`, diff --git a/packages/performance-tests/package.json b/packages/performance-tests/package.json index b95a72ba0..8739181af 100644 --- a/packages/performance-tests/package.json +++ b/packages/performance-tests/package.json @@ -10,7 +10,7 @@ "test": "mocha --delay --timeout 300000 --require ts-node/register test/**/*.test.ts", "format": "prettier \"./test/**/*.ts\" --write", "test-mocha": "mocha --delay --timeout 300000 \"./lib/**/TransformerRegression.test.js\"", - "test:quick": "mocha --timeout 900000 --require ts-node/register test/quick/QuickPerformance.quick.ts", + "test:quick": "mocha --require ts-node/register test/quick/QuickPerformance.quick.ts", "quick:build-fixture": "ts-node test/quick/cli.ts build-fixture", "quick:verify-fixture": "ts-node test/quick/cli.ts verify-fixture", "process-reports": "node scripts/process-reports" diff --git a/packages/performance-tests/test/quick/BenchmarkReporter.ts b/packages/performance-tests/test/quick/BenchmarkReporter.ts index 8d54651f3..7b57dfad5 100644 --- a/packages/performance-tests/test/quick/BenchmarkReporter.ts +++ b/packages/performance-tests/test/quick/BenchmarkReporter.ts @@ -12,6 +12,7 @@ import { medianAbsoluteDeviation, percentile, } from "./validation/statistics"; +import { resolvedVersions } from "./versions"; export const maximumCoefficientOfVariation = 0.05; export const maximumNormalizedMad = 0.05; @@ -63,6 +64,7 @@ export class BenchmarkReporter { jobMilliseconds, measuredSamples: measured.length, scenarioId: samples[0].scenarioId, + versions: resolvedVersions(), varianceStatus: classifyVariance( measured.length, wallCoefficientOfVariation, @@ -108,7 +110,7 @@ export class BenchmarkReporter { fs.writeFileSync( path.join(outputDir, "summary.csv"), [ - "scenario,fixture,measuredSamples,jobMs,medianMs,p90Ms,p95Ms,madMs,cv,reconstructionTotalMs,verificationTotalMs,teardownTotalMs", + "scenario,fixture,measuredSamples,jobMs,medianMs,p90Ms,p95Ms,madMs,cv,reconstructionTotalMs,verificationTotalMs,teardownTotalMs,transformer,coreBackend,node", [ summary.scenarioId, summary.fixtureId, @@ -122,6 +124,9 @@ export class BenchmarkReporter { summary.reconstructionMilliseconds.total, summary.verificationMilliseconds.total, summary.teardownMilliseconds.total, + summary.versions.transformer, + summary.versions.coreBackend, + summary.versions.node, ].join(","), ].join("\n") ); diff --git a/packages/performance-tests/test/quick/BenchmarkScenario.ts b/packages/performance-tests/test/quick/BenchmarkScenario.ts index ea153ece9..caf54d41f 100644 --- a/packages/performance-tests/test/quick/BenchmarkScenario.ts +++ b/packages/performance-tests/test/quick/BenchmarkScenario.ts @@ -15,7 +15,26 @@ export type BenchmarkScenarioFactory = ( dataset: PreparedDataset ) => BenchmarkScenario; +export const defaultScenarioBudgetMilliseconds = 15 * 60 * 1000; + export interface BenchmarkScenarioDefinition { readonly id: string; readonly factory: BenchmarkScenarioFactory; + /** + * Wall time allowed for the measured run, excluding checkout, install and build. + * Defaults to {@link defaultScenarioBudgetMilliseconds}. + */ + readonly budgetMilliseconds?: number; +} + +export function scenarioBudgetMilliseconds( + definition: BenchmarkScenarioDefinition +): number { + const budget = + definition.budgetMilliseconds ?? defaultScenarioBudgetMilliseconds; + if (!Number.isFinite(budget) || budget <= 0) + throw new Error( + `Scenario "${definition.id}" declares an invalid budget: ${budget}` + ); + return budget; } diff --git a/packages/performance-tests/test/quick/FixtureCatalog.ts b/packages/performance-tests/test/quick/FixtureCatalog.ts index 1133982d0..50bea9527 100644 --- a/packages/performance-tests/test/quick/FixtureCatalog.ts +++ b/packages/performance-tests/test/quick/FixtureCatalog.ts @@ -7,6 +7,7 @@ import * as fs from "fs"; import * as path from "path"; import { DatasetDescriptor } from "./DatasetDescriptor"; import { canonicalSha256 } from "./FixtureManifest"; +import { resolvedVersions } from "./versions"; const scale = 25; const distribution = { @@ -37,18 +38,7 @@ const distribution = { }, } as const; -function packageVersion(packageName: string): string { - const packageJson = JSON.parse( - fs.readFileSync(require.resolve(`${packageName}/package.json`), "utf8") - ) as { version: string }; - return packageJson.version; -} - -const generator = { - coreBackend: packageVersion("@itwin/core-backend"), - node: process.version, - transformer: packageVersion("@itwin/imodel-transformer"), -}; +const generator = resolvedVersions(); const recipeIdentity = { schema: "QuickPerf.01.00.00", diff --git a/packages/performance-tests/test/quick/QuickPerformance.quick.ts b/packages/performance-tests/test/quick/QuickPerformance.quick.ts index 14e6747cc..4cb194309 100644 --- a/packages/performance-tests/test/quick/QuickPerformance.quick.ts +++ b/packages/performance-tests/test/quick/QuickPerformance.quick.ts @@ -7,14 +7,19 @@ import * as path from "path"; import { expect } from "chai"; import { BenchmarkReporter } from "./BenchmarkReporter"; import { BenchmarkRunner } from "./BenchmarkRunner"; +import { scenarioBudgetMilliseconds } from "./BenchmarkScenario"; import { getFixtureDescriptor } from "./FixtureCatalog"; import { getScenarioDefinition } from "./ScenarioCatalog"; +/** Headroom above the scenario budget so the budget assertion reports before Mocha kills the test. */ +const budgetHeadroomMilliseconds = 60 * 1000; + describe("quick transformer performance", function () { - this.timeout(15 * 60 * 1000); + const scenario = getScenarioDefinition(process.env.QUICK_PERF_SCENARIO); + const budget = scenarioBudgetMilliseconds(scenario); + this.timeout(budget + budgetHeadroomMilliseconds); - it("runs the balanced incremental synchronization fixture", async () => { - const scenario = getScenarioDefinition(process.env.QUICK_PERF_SCENARIO); + it(`runs the ${scenario.id} scenario within its budget`, async () => { const measuredSamples = Number(process.env.QUICK_PERF_SAMPLES ?? "8"); const outputDir = process.env.QUICK_PERF_OUTPUT ?? @@ -42,6 +47,6 @@ describe("quick transformer performance", function () { expect(new Set(samples.map((sample) => sample.scenarioId))).to.deep.equal( new Set([scenario.id]) ); - expect(jobMilliseconds).to.be.lessThan(15 * 60 * 1000); + expect(jobMilliseconds).to.be.lessThan(budget); }); }); diff --git a/packages/performance-tests/test/quick/ScenarioCatalog.quick-unit.ts b/packages/performance-tests/test/quick/ScenarioCatalog.quick-unit.ts index 65fa9087c..fd25d5d96 100644 --- a/packages/performance-tests/test/quick/ScenarioCatalog.quick-unit.ts +++ b/packages/performance-tests/test/quick/ScenarioCatalog.quick-unit.ts @@ -4,6 +4,10 @@ *--------------------------------------------------------------------------------------------*/ import { expect } from "chai"; +import { + defaultScenarioBudgetMilliseconds, + scenarioBudgetMilliseconds, +} from "./BenchmarkScenario"; import { defaultQuickPerformanceScenarioId, getScenarioDefinition, @@ -19,6 +23,30 @@ describe("quick performance scenario catalog", () => { ); }); + it("defaults the budget when a scenario does not declare one", () => { + expect(scenarioBudgetMilliseconds(getScenarioDefinition())).to.equal( + defaultScenarioBudgetMilliseconds + ); + }); + + it("honours a declared budget", () => { + expect( + scenarioBudgetMilliseconds({ + ...getScenarioDefinition(), + budgetMilliseconds: 1000, + }) + ).to.equal(1000); + }); + + it("rejects a non-positive budget", () => { + expect(() => + scenarioBudgetMilliseconds({ + ...getScenarioDefinition(), + budgetMilliseconds: 0, + }) + ).to.throw(/invalid budget/); + }); + it("rejects unknown scenarios", () => { expect(() => getScenarioDefinition("not-a-scenario")).to.throw( 'Unknown quick performance scenario "not-a-scenario". Available scenarios: incremental-synchronization' diff --git a/packages/performance-tests/test/quick/scanOracle.quick-unit.ts b/packages/performance-tests/test/quick/scanOracle.quick-unit.ts new file mode 100644 index 000000000..6a241d74b --- /dev/null +++ b/packages/performance-tests/test/quick/scanOracle.quick-unit.ts @@ -0,0 +1,181 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { expect } from "chai"; +import { + assertScanMatchesOracle, + normalizeScanResult, + ScanCollection, + scanCollections, + scanDigest, + ScanLedger, + squashLedger, +} from "./validation/scanOracle"; + +function ledgerOf( + ...entries: readonly [ + ScanCollection, + "Deleted" | "Inserted" | "Updated", + string, + ][] +): ScanLedger { + const ledger = new ScanLedger(); + for (const [collection, op, id] of entries) ledger.record(collection, op, id); + return ledger; +} + +describe("changeset scan oracle", () => { + it("keeps a repeatedly updated instance in updateIds", () => { + const squashed = squashLedger( + ledgerOf( + ["element", "Updated", "0x20"], + ["element", "Updated", "0x20"], + ["element", "Updated", "0x20"] + ) + ); + expect([...squashed.element.updateIds]).to.deep.equal(["0x20"]); + expect(squashed.element.insertIds.size).to.equal(0); + expect(squashed.element.deleteIds.size).to.equal(0); + }); + + it("turns update-then-delete into a delete", () => { + const squashed = squashLedger( + ledgerOf(["element", "Updated", "0x21"], ["element", "Deleted", "0x21"]) + ); + expect([...squashed.element.deleteIds]).to.deep.equal(["0x21"]); + expect(squashed.element.updateIds.size).to.equal(0); + }); + + it("keeps insert-then-update as an insert", () => { + const squashed = squashLedger( + ledgerOf(["element", "Inserted", "0x22"], ["element", "Updated", "0x22"]) + ); + expect([...squashed.element.insertIds]).to.deep.equal(["0x22"]); + expect(squashed.element.updateIds.size).to.equal(0); + }); + + it("cancels insert-then-delete entirely", () => { + const squashed = squashLedger( + ledgerOf(["element", "Inserted", "0x23"], ["element", "Deleted", "0x23"]) + ); + for (const op of ["insertIds", "updateIds", "deleteIds"] as const) + expect(squashed.element[op].size, op).to.equal(0); + }); + + it("reinstates an insert after a delete", () => { + const squashed = squashLedger( + ledgerOf( + ["relationship", "Deleted", "0x24"], + ["relationship", "Inserted", "0x24"] + ) + ); + expect([...squashed.relationship.insertIds]).to.deep.equal(["0x24"]); + expect(squashed.relationship.deleteIds.size).to.equal(0); + }); + + it("keeps collections independent", () => { + const squashed = squashLedger( + ledgerOf( + ["element", "Inserted", "0x30"], + ["model", "Inserted", "0x30"], + ["aspect", "Deleted", "0x30"], + ["codeSpec", "Updated", "0x30"] + ) + ); + expect([...squashed.element.insertIds]).to.deep.equal(["0x30"]); + expect([...squashed.model.insertIds]).to.deep.equal(["0x30"]); + expect([...squashed.aspect.deleteIds]).to.deep.equal(["0x30"]); + expect([...squashed.codeSpec.updateIds]).to.deep.equal(["0x30"]); + expect(squashed.font.insertIds.size).to.equal(0); + }); + + it("drops repeated ops without changing the squashed outcome", () => { + const ledger = new ScanLedger(); + for (let index = 0; index < 50; index++) + ledger.record("element", "Updated", "0x40"); + ledger.record("element", "Deleted", "0x40"); + expect(ledger.entries).to.have.length(2); + expect([...squashLedger(ledger).element.deleteIds]).to.deep.equal(["0x40"]); + }); + + it("records batches of ids", () => { + const ledger = new ScanLedger(); + ledger.record("aspect", "Inserted", ["0x50", "0x51"]); + expect([...squashLedger(ledger).aspect.insertIds]).to.deep.equal([ + "0x50", + "0x51", + ]); + }); + + it("orders digest ids numerically rather than lexically", () => { + const ledger = new ScanLedger(); + ledger.record("element", "Inserted", ["0x100", "0x9", "0x20"]); + expect( + normalizeScanResult(squashLedger(ledger)).element.insert + ).to.deep.equal(["0x9", "0x20", "0x100"]); + }); + + it("produces an insertion-order-independent digest", () => { + const forward = new ScanLedger(); + forward.record("element", "Inserted", ["0x60", "0x61"]); + const reverse = new ScanLedger(); + reverse.record("element", "Inserted", ["0x61", "0x60"]); + expect(scanDigest(squashLedger(forward))).to.equal( + scanDigest(squashLedger(reverse)) + ); + }); + + it("covers all six ChangedInstanceIds collections", () => { + expect([...scanCollections].sort()).to.deep.equal([ + "aspect", + "codeSpec", + "element", + "font", + "model", + "relationship", + ]); + }); + + it("accepts a scan that matches the recipe", () => { + const expected = squashLedger(ledgerOf(["element", "Updated", "0x70"])); + expect(() => + assertScanMatchesOracle( + squashLedger(ledgerOf(["element", "Updated", "0x70"])), + expected + ) + ).to.not.throw(); + }); + + it("reports missing and unexpected ids per collection", () => { + const expected = squashLedger( + ledgerOf(["aspect", "Updated", "0x80"], ["aspect", "Updated", "0x81"]) + ); + const actual = squashLedger( + ledgerOf(["aspect", "Updated", "0x80"], ["aspect", "Updated", "0x82"]) + ); + let message = ""; + try { + assertScanMatchesOracle(actual, expected); + } catch (error) { + message = (error as Error).message; + } + expect(message).to.match(/aspect\.updateIds/); + expect(message).to.match(/missing=\[0x81\]/); + expect(message).to.match(/unexpected=\[0x82\]/); + }); + + it("fails when a collection outside element regresses", () => { + const expected = squashLedger(ledgerOf(["element", "Updated", "0x90"])); + const actual = squashLedger( + ledgerOf( + ["element", "Updated", "0x90"], + ["relationship", "Deleted", "0x91"] + ) + ); + expect(() => assertScanMatchesOracle(actual, expected)).to.throw( + /relationship\.deleteIds/ + ); + }); +}); diff --git a/packages/performance-tests/test/quick/validation/scanOracle.ts b/packages/performance-tests/test/quick/validation/scanOracle.ts new file mode 100644 index 000000000..e57241637 --- /dev/null +++ b/packages/performance-tests/test/quick/validation/scanOracle.ts @@ -0,0 +1,199 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { Id64String } from "@itwin/core-bentley"; +import { canonicalSha256 } from "../FixtureManifest"; + +/** + * The six collections carried by `ChangedInstanceIds`. Verifying only `element` would let aspect and + * relationship regressions pass silently, so the scan oracle covers all of them. + */ +export const scanCollections = [ + "aspect", + "codeSpec", + "element", + "font", + "model", + "relationship", +] as const; + +export type ScanCollection = (typeof scanCollections)[number]; + +/** Mirrors `SqliteChangeOp` without importing it, so the oracle stays independent of transformer internals. */ +export type ScanOp = "Deleted" | "Inserted" | "Updated"; + +export interface ScanLedgerEntry { + readonly collection: ScanCollection; + readonly id: Id64String; + readonly op: ScanOp; +} + +export interface ScanOps { + readonly insertIds: ReadonlySet; + readonly updateIds: ReadonlySet; + readonly deleteIds: ReadonlySet; +} + +export type ScanExpectation = Readonly>; + +/** + * Ordered record of every change a recipe performs, used to predict the scan result. + * + * Repeating an op for the same instance is idempotent under all three squash rules, so the recorder + * drops repeats. That keeps an update-heavy recipe's ledger proportional to touched instances rather + * than to touched instances times changesets, without changing the squashed outcome. + */ +export class ScanLedger { + private readonly _entries: ScanLedgerEntry[] = []; + private readonly _lastOp = new Map(); + + public record( + collection: ScanCollection, + op: ScanOp, + ids: Id64String | Iterable + ): void { + const iterable = typeof ids === "string" ? [ids] : ids; + for (const id of iterable) { + const key = `${collection}\u0000${id}`; + if (this._lastOp.get(key) === op) continue; + this._lastOp.set(key, op); + this._entries.push({ collection, id, op }); + } + } + + public get entries(): readonly ScanLedgerEntry[] { + return this._entries; + } +} + +interface MutableScanOps { + insertIds: Set; + updateIds: Set; + deleteIds: Set; +} + +/** + * Squash semantics, written independently of `ChangedInstanceIds.handleChange`. + * + * Reusing the transformer's implementation would check the code against itself. The rules are: + * an insert clears a pending delete; an update is dropped when the instance is already an insert; + * a delete cancels a pending insert outright and otherwise supersedes a pending update. + */ +function applyOp(ops: MutableScanOps, op: ScanOp, id: Id64String): void { + switch (op) { + case "Inserted": + ops.insertIds.add(id); + ops.deleteIds.delete(id); + return; + case "Updated": + if (!ops.insertIds.has(id)) ops.updateIds.add(id); + return; + case "Deleted": + if (ops.insertIds.has(id)) { + ops.insertIds.delete(id); + return; + } + ops.updateIds.delete(id); + ops.deleteIds.add(id); + return; + } +} + +export function squashLedger(ledger: ScanLedger): ScanExpectation { + const result = Object.fromEntries( + scanCollections.map((collection) => [ + collection, + { + insertIds: new Set(), + updateIds: new Set(), + deleteIds: new Set(), + }, + ]) + ) as Record; + for (const entry of ledger.entries) + applyOp(result[entry.collection], entry.op, entry.id); + return result; +} + +/** Numeric ordering for hex `Id64String`s, so the digest does not depend on string collation. */ +export function compareIds(left: Id64String, right: Id64String): number { + const leftValue = BigInt(left); + const rightValue = BigInt(right); + return leftValue < rightValue ? -1 : leftValue > rightValue ? 1 : 0; +} + +function sortedIds(ids: ReadonlySet): Id64String[] { + return [...ids].sort(compareIds); +} + +export function normalizeScanResult( + result: ScanExpectation +): Record> { + return Object.fromEntries( + scanCollections.map((collection) => [ + collection, + { + delete: sortedIds(result[collection].deleteIds), + insert: sortedIds(result[collection].insertIds), + update: sortedIds(result[collection].updateIds), + }, + ]) + ) as Record>; +} + +function difference( + actual: ReadonlySet, + expected: ReadonlySet +): { missing: Id64String[]; unexpected: Id64String[] } { + return { + missing: sortedIds(new Set([...expected].filter((id) => !actual.has(id)))), + unexpected: sortedIds( + new Set([...actual].filter((id) => !expected.has(id))) + ), + }; +} + +/** + * Asserts the scan result against what the recipe says it produced. This is strictly stronger than + * comparing a digest to its own previous value, which can only detect a change of behaviour and not + * a behaviour that was wrong from the start. + */ +export function assertScanMatchesOracle( + actual: ScanExpectation, + expected: ScanExpectation +): void { + const mismatches: string[] = []; + for (const collection of scanCollections) { + for (const op of ["insertIds", "updateIds", "deleteIds"] as const) { + const { missing, unexpected } = difference( + actual[collection][op], + expected[collection][op] + ); + if (missing.length === 0 && unexpected.length === 0) continue; + mismatches.push( + `${collection}.${op}: expected ${ + expected[collection][op].size + }, got ${actual[collection][op].size}; missing=[${missing.join( + "," + )}], unexpected=[${unexpected.join(",")}]` + ); + } + } + if (mismatches.length > 0) + throw new Error( + `Changeset scan did not match the recipe oracle:\n ${mismatches.join( + "\n " + )}` + ); +} + +/** + * Stable fingerprint over all six collections. Every sample scans a copy of one immutable artifact, + * so ids are identical across samples and equality is meaningful; it also serves as a cross-arm + * behaviour gate for A/B comparison. + */ +export function scanDigest(result: ScanExpectation): string { + return canonicalSha256(normalizeScanResult(result)); +} diff --git a/packages/performance-tests/test/quick/versions.ts b/packages/performance-tests/test/quick/versions.ts new file mode 100644 index 000000000..20937a97a --- /dev/null +++ b/packages/performance-tests/test/quick/versions.ts @@ -0,0 +1,31 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "fs"; + +export interface ResolvedVersions { + readonly coreBackend: string; + readonly node: string; + readonly transformer: string; +} + +export function packageVersion(packageName: string): string { + const packageJson = JSON.parse( + fs.readFileSync(require.resolve(`${packageName}/package.json`), "utf8") + ) as { version: string }; + return packageJson.version; +} + +/** + * Versions actually loaded by the measuring process. A fixture descriptor records the versions that + * *generated* it, which are not necessarily the versions that measure it. + */ +export function resolvedVersions(): ResolvedVersions { + return { + coreBackend: packageVersion("@itwin/core-backend"), + node: process.version, + transformer: packageVersion("@itwin/imodel-transformer"), + }; +} From 09f4a4f246275d82d54da283e46e00c4e303eb04 Mon Sep 17 00:00:00 2001 From: Nam Le <50554904+hl662@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:06:38 -0400 Subject: [PATCH 06/19] Migrate weekly performance tests to Vitest Replace Mocha delayed registration with serialized Vitest collection and explicit worker lifecycle cleanup while preserving the weekly report and authentication contracts. Validated transformer build, performance test type-check, lint, and focused cleanup/registration tests. The credentialed suite has not been executed locally; live Bentley Hub runtime behavior remains to be validated by the weekly ADO pipeline. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .vscode/launch.json | 16 +- packages/performance-tests/README.md | 14 +- packages/performance-tests/package.json | 13 +- packages/performance-tests/test/Cleanup.ts | 73 +++++ .../test/RegressionTestRegistration.ts | 38 +++ packages/performance-tests/test/TestUtils.ts | 4 +- .../test/TransformerRegression.test.ts | 297 +++++++++++------- packages/performance-tests/test/rawInserts.ts | 216 +++++++------ .../test/unit/Cleanup.test.ts | 55 ++++ .../unit/RegressionTestRegistration.test.ts | 56 ++++ packages/performance-tests/test/vitest.d.ts | 8 + packages/performance-tests/tsconfig.json | 4 +- packages/performance-tests/vitest.config.ts | 21 ++ pnpm-lock.yaml | 190 +---------- 14 files changed, 567 insertions(+), 438 deletions(-) create mode 100644 packages/performance-tests/test/Cleanup.ts create mode 100644 packages/performance-tests/test/RegressionTestRegistration.ts create mode 100644 packages/performance-tests/test/unit/Cleanup.test.ts create mode 100644 packages/performance-tests/test/unit/RegressionTestRegistration.test.ts create mode 100644 packages/performance-tests/test/vitest.d.ts create mode 100644 packages/performance-tests/vitest.config.ts diff --git a/.vscode/launch.json b/.vscode/launch.json index 1d5c91ce3..f91b11e63 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,8 +11,7 @@ "runtimeExecutable": "npm", "cwd": "${workspaceFolder}/packages/performance-tests/", "runtimeArgs": [ - "run", - "test-mocha" + "test" ], "skipFiles": [ "/**" @@ -31,17 +30,6 @@ "skipFiles": [ "/**" ] - }, - { - "type": "node", - "request": "launch", - "name": "Performance tests", - "runtimeExecutable": "npm", - "cwd": "${workspaceFolder}/packages/performance-tests/test", - "runtimeArgs": [ - "run", - "test" - ], } ] -} \ No newline at end of file +} diff --git a/packages/performance-tests/README.md b/packages/performance-tests/README.md index b9dc054f4..b520154e2 100644 --- a/packages/performance-tests/README.md +++ b/packages/performance-tests/README.md @@ -1,4 +1,4 @@ -# Presentation Performance Tests +# Transformer Performance Tests A package containing performance tests for the [`@itwin/imodel-transformer` library](../../README.md). @@ -17,7 +17,6 @@ Here are tests we need but don't have: - *Processing Changes* - *More Branching Stuff* - ## Usage 1. Clone the repository. @@ -30,16 +29,11 @@ Here are tests we need but don't have: 3. Create `.env` file using `template.env` template. -5. Run: +4. Run the serialized Vitest suite: ```sh pnpm test ``` - -6. Review results like: - -```sh -pnpm exec process-results < report.jsonl -``` - +5. Review `test/.output/report.csv`. This path is also the artifact contract used by + the weekly Azure pipeline. diff --git a/packages/performance-tests/package.json b/packages/performance-tests/package.json index 09a07f57e..2b9885e29 100644 --- a/packages/performance-tests/package.json +++ b/packages/performance-tests/package.json @@ -4,12 +4,11 @@ "license": "MIT", "version": "0.1.0", "scripts": { - "build": "tsc 1>&2", + "build": "tsc --noEmit --incremental false 1>&2", "clean": "rimraf lib", "lint": "eslint \"./test/**/*.ts\" 1>&2", - "test": "mocha --delay --timeout 300000 --require ts-node/register test/**/*.test.ts", + "test": "vitest run", "format": "prettier \"./test/**/*.ts\" --write", - "test-mocha": "mocha --delay --timeout 300000 \"./lib/**/TransformerRegression.test.js\"", "process-reports": "node scripts/process-reports" }, "repository": {}, @@ -37,19 +36,15 @@ "@itwin/eslint-plugin": "^5.2.1", "@itwin/itwins-client": "^1.6.1", "@itwin/oidc-signin-tool": "^6.0.0", - "@types/chai": "^4.1.4", "@types/fs-extra": "^4.0.7", - "@types/mocha": "^8.2.2", "@types/node": "^22", "@types/yargs": "^12.0.5", - "chai": "^4.3.6", "eslint": "^9.11.1", "eslint-config-prettier": "^9.1.0", - "mocha": "^10.0.0", "prettier": "^3.1.1", "rimraf": "^3.0.2", - "ts-node": "^10.7.0", - "typescript": "~5.6.2" + "typescript": "~5.6.2", + "vitest": "4.1.10" }, "eslintConfig": { "plugins": [ diff --git a/packages/performance-tests/test/Cleanup.ts b/packages/performance-tests/test/Cleanup.ts new file mode 100644 index 000000000..d014ee946 --- /dev/null +++ b/packages/performance-tests/test/Cleanup.ts @@ -0,0 +1,73 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +export interface CleanupTask { + name: string; + run(): void | Promise; +} + +interface NamedCleanupError { + name: string; + error: unknown; +} + +function toError({ name, error }: NamedCleanupError): Error { + return new Error(`Cleanup failed: ${name}`, { cause: error }); +} + +async function collectCleanupErrors( + cleanupTasks: CleanupTask[] +): Promise { + const errors: NamedCleanupError[] = []; + for (const cleanupTask of cleanupTasks) { + try { + await cleanupTask.run(); + } catch (error) { + errors.push({ name: cleanupTask.name, error }); + } + } + return errors.map(toError); +} + +export async function runCleanupTasks( + cleanupTasks: CleanupTask[] +): Promise { + const errors = await collectCleanupErrors(cleanupTasks); + if (errors.length === 1) { + throw errors[0]; + } + if (errors.length > 1) { + throw new AggregateError(errors, "Multiple cleanup tasks failed"); + } +} + +export async function throwAfterCleanup( + primaryError: unknown, + cleanupTasks: CleanupTask[] +): Promise { + const cleanupErrors = await collectCleanupErrors(cleanupTasks); + if (cleanupErrors.length === 0) { + throw primaryError; + } + throw new AggregateError( + [primaryError, ...cleanupErrors], + "Operation and cleanup both failed", + { cause: primaryError } + ); +} + +export async function runWithCleanup( + operation: () => Promise, + cleanupTasks: CleanupTask[] +): Promise { + let result!: T; + try { + result = await operation(); + } catch (error) { + await throwAfterCleanup(error, cleanupTasks); + } + await runCleanupTasks(cleanupTasks); + return result; +} diff --git a/packages/performance-tests/test/RegressionTestRegistration.ts b/packages/performance-tests/test/RegressionTestRegistration.ts new file mode 100644 index 000000000..80cac5e74 --- /dev/null +++ b/packages/performance-tests/test/RegressionTestRegistration.ts @@ -0,0 +1,38 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { TestTransformerModule } from "./TestTransformerModule"; + +export interface RegressionTestCase { + testCase: T; + functionNameToValidate: keyof TestTransformerModule; +} + +export interface RegressionTestDefinition { + testCaseName: string; + testCase: T; + moduleName: string; + transformerModule: TestTransformerModule; +} + +export function getRegressionTestDefinitions( + testCases: ReadonlyMap>, + transformerModules: ReadonlyMap +): RegressionTestDefinition[] { + const definitions: RegressionTestDefinition[] = []; + for (const [testCaseName, testCaseDefinition] of testCases) { + for (const [moduleName, transformerModule] of transformerModules) { + if (transformerModule[testCaseDefinition.functionNameToValidate]) { + definitions.push({ + testCaseName, + testCase: testCaseDefinition.testCase, + moduleName, + transformerModule, + }); + } + } + } + return definitions; +} diff --git a/packages/performance-tests/test/TestUtils.ts b/packages/performance-tests/test/TestUtils.ts index 13d82f621..fb38d835d 100644 --- a/packages/performance-tests/test/TestUtils.ts +++ b/packages/performance-tests/test/TestUtils.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import * as fs from "node:fs"; import * as path from "node:path"; -import { assert } from "chai"; +import { assert } from "vitest"; import { IModelDb } from "@itwin/core-backend"; import { DbResult, StopWatch } from "@itwin/core-bentley"; import { GeometryStreamBuilder, GeometryStreamProps } from "@itwin/core-common"; @@ -83,7 +83,7 @@ export function timed>( } } -// Mocha tests must know the test cases ahead time, so we collect the the Imodels first before beginning the tests +// Vitest must know the test cases during collection, so collect the iModels first. export async function preFetchAsyncIterator( iter: AsyncGenerator ): Promise { diff --git a/packages/performance-tests/test/TransformerRegression.test.ts b/packages/performance-tests/test/TransformerRegression.test.ts index 25e330905..799550459 100644 --- a/packages/performance-tests/test/TransformerRegression.test.ts +++ b/packages/performance-tests/test/TransformerRegression.test.ts @@ -3,49 +3,57 @@ * See LICENSE.md in the project root for license terms and full copyright notice. *--------------------------------------------------------------------------------------------*/ -/* - * Tests where we perform "identity" transforms, that is just rebuilding an entire identical iModel (minus IDs) - * through the transformation process. - */ - import "./setup"; import * as fs from "node:fs"; import * as path from "node:path"; -import { BackendIModelsAccess } from "@itwin/imodels-access-backend"; +import assert from "node:assert"; import { BriefcaseDb, IModelHost, IModelHostConfiguration, } from "@itwin/core-backend"; import { Logger, LogLevel } from "@itwin/core-bentley"; +import { TransformerLoggerCategory } from "@itwin/imodel-transformer"; +import { BackendIModelsAccess } from "@itwin/imodels-access-backend"; import { IModelsClient } from "@itwin/imodels-client-authoring"; import { NodeCliAuthorizationClient } from "@itwin/node-cli-authorization"; -import { Reporter } from "@itwin/perf-tools"; -import { ReporterInfo } from "./ReporterUtils"; -import { TestBrowserAuthorizationClient } from "@itwin/oidc-signin-tool"; -import { TestTransformerModule } from "./TestTransformerModule"; -import { TransformerLoggerCategory } from "@itwin/imodel-transformer"; import { AzureClientStorage, BlockBlobClientWrapperFactory, } from "@itwin/object-storage-azure"; +import { Reporter } from "@itwin/perf-tools"; +import { TestBrowserAuthorizationClient } from "@itwin/oidc-signin-tool"; +import { + CleanupTask, + runCleanupTasks, + runWithCleanup, + throwAfterCleanup, +} from "./Cleanup"; +import { getBranchName } from "./GitUtils"; +import { + getRegressionTestDefinitions, + RegressionTestCase, +} from "./RegressionTestRegistration"; +import { ReporterInfo } from "./ReporterUtils"; +import { getTestIModels, TestIModel } from "./TestContext"; +import { TestTransformerModule } from "./TestTransformerModule"; import { filterIModels, initOutputFile, preFetchAsyncIterator, } from "./TestUtils"; -import { getBranchName } from "./GitUtils"; -import { getTestIModels } from "./TestContext"; -import assert from "node:assert"; +import identityTransformer from "./cases/identity-transformer"; +import prepareFork from "./cases/prepare-fork"; +import rawInserts from "./rawInserts"; import nativeTransformerTestModule from "./transformers/NativeTransformer"; import rawForkOperationsTestModule from "./transformers/RawForkOperations"; -import rawInserts from "./rawInserts"; -// cases -import identityTransformer from "./cases/identity-transformer"; -import prepareFork from "./cases/prepare-fork"; +type AuthorizationClient = + | NodeCliAuthorizationClient + | TestBrowserAuthorizationClient; +type PerformanceTestCase = typeof identityTransformer; -const testCasesMap = new Map([ +const testCasesMap = new Map>([ [ "identity transform (provenance)", { @@ -65,27 +73,65 @@ const testCasesMap = new Map([ const loggerCategory = "Transformer Performance Regression Tests"; const outputDir = path.join(__dirname, ".output"); -const loadTransformers = async () => { +class WorkerLifecycle { + private _authClient?: AuthorizationClient; + private _hostStarted = false; + + public setAuthClient(authClient: AuthorizationClient): void { + this._authClient = authClient; + } + + public markHostStarted(): void { + this._hostStarted = true; + } + + public async shutdown(): Promise { + const hostStarted = this._hostStarted; + const authClient = this._authClient; + this._hostStarted = false; + this._authClient = undefined; + + const cleanupTasks: CleanupTask[] = []; + if (hostStarted) { + cleanupTasks.push({ + name: "IModelHost shutdown", + run: async () => IModelHost.shutdown(), + }); + } + if (authClient) { + cleanupTasks.push({ + name: "authorization sign out", + run: async () => authClient.signOut(), + }); + } + await runCleanupTasks(cleanupTasks); + } +} + +async function loadTransformers(): Promise> { const modulePaths = process.env.EXTRA_TRANSFORMERS?.split(",") .map((name) => name.trim()) .filter(Boolean) ?? []; - const envSpecifiedExtraTransformerCases = (await Promise.all( - modulePaths.map(async (m) => [m, (await import(m)).default]) + const extraTransformerCases = (await Promise.all( + modulePaths.map(async (modulePath) => [ + modulePath, + (await import(modulePath)).default, + ]) )) as [string, TestTransformerModule][]; - const transformerModules = new Map([ + return new Map([ ["NativeTransformer", nativeTransformerTestModule], ["RawForkOperations", rawForkOperationsTestModule], - ...envSpecifiedExtraTransformerCases, + ...extraTransformerCases, ]); - return transformerModules; -}; +} -const setupTestData = async () => { +async function setupTestData( + workerLifecycle: WorkerLifecycle +): Promise { const logLevel = process.env.LOG_LEVEL ? Number(process.env.LOG_LEVEL) : LogLevel.Error; - assert(LogLevel[logLevel] !== undefined, "unknown log level"); Logger.initializeToConsole(); @@ -109,10 +155,7 @@ const setupTestData = async () => { assert(usrEmail, "user name was not configured"); assert(usrPass, "user password was not configured"); - const user = { - email: usrEmail, - password: usrPass, - }; + const user = { email: usrEmail, password: usrPass }; assert(process.env.OIDC_CLIENT_ID, "OIDC_CLIENT_ID not set"); assert(process.env.OIDC_REDIRECT, "OIDC_REDIRECT not set"); @@ -135,7 +178,7 @@ const setupTestData = async () => { redirectUri: process.env.OIDC_REDIRECT, scope: process.env.OIDC_SCOPES, }); - + workerLifecycle.setAuthClient(authClient); await authClient.signIn(); const hostConfig = new IModelHostConfiguration(); @@ -148,37 +191,62 @@ const setupTestData = async () => { }); hostConfig.hubAccess = new BackendIModelsAccess(hubClient); await IModelHost.startup(hostConfig); + workerLifecycle.markHostStarted(); return preFetchAsyncIterator(getTestIModels(filterIModels)); -}; +} -async function runRegressionTests() { - const testIModels = await setupTestData(); +async function collectRegressionInputs(workerLifecycle: WorkerLifecycle) { + const testIModels = await setupTestData(workerLifecycle); const transformerModules = await loadTransformers(); const reporter = new Reporter(); const reportPath = initOutputFile("report.csv", outputDir); const branchName = await getBranchName(); + return { + branchName, + reportPath, + reporter, + testIModels, + transformerModules, + }; +} + +const lifecycle = new WorkerLifecycle(); +const { + branchName: currentBranchName, + reportPath: csvReportPath, + reporter: performanceReporter, + testIModels: collectedIModels, + transformerModules: loadedTransformerModules, +} = await collectRegressionInputs(lifecycle).catch(async (error: unknown) => + throwAfterCleanup(error, [ + { name: "worker lifecycle", run: async () => lifecycle.shutdown() }, + ]) +); +const regressionTestDefinitions = getRegressionTestDefinitions( + testCasesMap, + loadedTransformerModules +); - describe("Transformer Regression Tests", function () { - testIModels.forEach(async (iModel) => { - let sourceDb: BriefcaseDb; +describe("Transformer Regression Tests", () => { + for (const iModel of collectedIModels) { + describe(`Transforms of ${iModel.name}`, () => { + let sourceDb: BriefcaseDb | undefined; let reportInfo: ReporterInfo; let sourceFileName: string; - describe(`Transforms of ${iModel.name}`, async () => { - before(async () => { - Logger.logInfo( - loggerCategory, - `processing iModel '${ - iModel.name - }' of size '${iModel.tShirtSize.toUpperCase()}'` - ); - sourceFileName = await iModel.getFileName(); - sourceDb = await BriefcaseDb.open({ - fileName: sourceFileName, - readonly: true, - }); - const fedGuidReader = sourceDb.createQueryReader( + beforeAll(async () => { + Logger.logInfo( + loggerCategory, + `processing iModel '${iModel.name}' of size '${iModel.tShirtSize.toUpperCase()}'` + ); + sourceFileName = await iModel.getFileName(); + const metadataDb = await BriefcaseDb.open({ + fileName: sourceFileName, + readonly: true, + }); + await runWithCleanup(async () => { + const fedGuidReader = metadataDb.createQueryReader( "SELECT CAST(SUM(IIF(FederationGuid IS NOT NULL, 1, 0)) AS DOUBLE)/COUNT(*) AS ratio FROM bis.Element", undefined, { usePrimaryConn: true } @@ -190,80 +258,81 @@ async function runRegressionTests() { loggerCategory, `Federation Guid Saturation '${fedGuidSaturation}'` ); - const toGb = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(2)}Gb`; - const sizeInGb = toGb(fs.statSync(sourceDb.pathName).size); + const sizeInGb = `${( + fs.statSync(metadataDb.pathName).size / + 1024 ** 3 + ).toFixed(2)}Gb`; Logger.logInfo(loggerCategory, `loaded (${sizeInGb})'`); reportInfo = { Id: iModel.iModelId, "T-shirt size": iModel.tShirtSize, "Gb size": sizeInGb, - "Branch Name": branchName, + "Branch Name": currentBranchName, "Federation Guid Saturation 0-1": fedGuidSaturation, }; - sourceDb.close(); - }); + }, [ + { + name: `${iModel.name} metadata briefcase`, + run: () => metadataDb.close(), + }, + ]); + }); - beforeEach(async () => { - sourceDb = await BriefcaseDb.open({ - fileName: sourceFileName, - readonly: true, - }); + beforeEach(async () => { + sourceDb = await BriefcaseDb.open({ + fileName: sourceFileName, + readonly: true, }); + }); - afterEach(async () => { - sourceDb.close(); // closing to ensure connection cache reusage doesn't affect results - }); + afterEach(() => { + const dbToClose = sourceDb; + sourceDb = undefined; + dbToClose?.close(); + }); - testCasesMap.forEach( - async ({ testCase, functionNameToValidate }, testCaseName) => { - transformerModules.forEach( - ( - transformerModule: TestTransformerModule, - moduleName: string - ) => { - const moduleFunc = - transformerModule[ - functionNameToValidate as keyof TestTransformerModule - ]; - if (moduleFunc) { - it(`${testCaseName} on ${moduleName}`, async () => { - const addReport = ( - iModelName: string, - valDescription: string, - value: number - ) => { - reporter.addEntry( - `${testCaseName} ${moduleName}`, - iModelName, - valDescription, - value, - reportInfo - ); - }; - await testCase({ sourceDb, transformerModule, addReport }); - // eslint-disable-next-line no-console - console.log("Finished the test"); - }).timeout(0); - } - } + for (const definition of regressionTestDefinitions) { + test(`${definition.testCaseName} on ${definition.moduleName}`, async () => { + assert(sourceDb, "source briefcase was not opened"); + const addReport = ( + iModelName: string, + valDescription: string, + value: number + ) => { + performanceReporter.addEntry( + `${definition.testCaseName} ${definition.moduleName}`, + iModelName, + valDescription, + value, + reportInfo ); - } - ); - }); + }; + await definition.testCase({ + sourceDb, + transformerModule: definition.transformerModule, + addReport, + }); + // eslint-disable-next-line no-console + console.log("Finished the test"); + }); + } }); + } - const _15minutes = 15 * 60 * 1000; - - it("Transform vs raw inserts", async () => { - return rawInserts(reporter, branchName); - }).timeout(0); - }); - - after(async () => { - reporter.exportCSV(reportPath); + test("Transform vs raw inserts", async () => { + await rawInserts(performanceReporter, currentBranchName); }); +}); - run(); -} - -void runRegressionTests(); +afterAll(async () => { + await runCleanupTasks([ + { + name: "report export", + run: () => performanceReporter.exportCSV(csvReportPath), + }, + { + name: "worker lifecycle", + run: async () => lifecycle.shutdown(), + }, + ]); +}); diff --git a/packages/performance-tests/test/rawInserts.ts b/packages/performance-tests/test/rawInserts.ts index 02373e219..03ee6d1a5 100644 --- a/packages/performance-tests/test/rawInserts.ts +++ b/packages/performance-tests/test/rawInserts.ts @@ -18,6 +18,7 @@ import { generateTestIModel } from "./iModelUtils"; import { count, initOutputFile, timed } from "./TestUtils"; import assert from "node:assert"; import path from "node:path"; +import { runWithCleanup } from "./Cleanup"; const loggerCategory = "Raw Inserts"; const outputDir = path.join(__dirname, ".output"); @@ -31,108 +32,121 @@ export default async function rawInserts( reporter: Reporter, branchName: string ) { - Logger.logInfo(loggerCategory, "starting 150k entity inserts"); - - let testIModel: TestIModel | undefined; - const [insertsTimer] = timed(() => { - testIModel = generateTestIModel({ - numElements: 100_000, - fedGuids: true, - fileName: "RawInserts-source.bim", + let sourceDb: StandaloneDb | undefined; + let targetDb: SnapshotDb | undefined; + let targetNoProvDb: SnapshotDb | undefined; + await runWithCleanup(async () => { + Logger.logInfo(loggerCategory, "starting 150k entity inserts"); + + let testIModel: TestIModel | undefined; + const [insertsTimer] = timed(() => { + testIModel = generateTestIModel({ + numElements: 100_000, + fedGuids: true, + fileName: "RawInserts-source.bim", + }); }); - }); - - if (testIModel === undefined) - throw Error("Generated iModel not correctly defined"); // needed because TS does not know that timer will run before insertsTimer - const fileName = await testIModel.getFileName(); - const sourceDb = StandaloneDb.openFile(fileName, OpenMode.ReadWrite); - - reporter.addEntry( - "populate by insert", - iModelName, - "time elapsed (seconds)", - insertsTimer?.elapsedSeconds ?? -1, - { - "Element Count": count(sourceDb, Element.classFullName), - "Relationship Count": count(sourceDb, ElementGroupsMembers.classFullName), - "Branch Name": branchName, - } - ); - - Logger.logInfo( - loggerCategory, - "Done. Starting with-provenance transformation of same content" - ); - - const targetPath = initOutputFile("RawInserts-Target.bim", outputDir); - const targetDb = SnapshotDb.createEmpty(targetPath, { - rootSubject: { name: "RawInsertsTarget" }, - }); - const withProvEditTxn = new EditTxn(targetDb, "IModelTransformer"); - withProvEditTxn.start(); - const transformerWithProv = new IModelTransformer( - { source: sourceDb, target: withProvEditTxn }, - { - noProvenance: false, - } - ); - - const [transformWithProvTimer] = await timed(async () => { - await transformerWithProv.process(); - }); - withProvEditTxn.end(); - - reporter.addEntry( - "populate by transform (adding provenance)", - iModelName, - "time elapsed (seconds)", - transformWithProvTimer?.elapsedSeconds ?? -1, - { - "Element Count": count(sourceDb, Element.classFullName), - "Relationship Count": count(sourceDb, ElementGroupsMembers.classFullName), - "Branch Name": branchName, - } - ); - Logger.logInfo( - loggerCategory, - "Done. Starting without-provenance transformation of same content" - ); - - const targetNoProvPath = initOutputFile( - "RawInserts-TargetNoProv.bim", - outputDir - ); - const targetNoProvDb = SnapshotDb.createEmpty(targetNoProvPath, { - rootSubject: { name: "RawInsertsTarget" }, - }); - const noProvEditTxn = new EditTxn(targetNoProvDb, "IModelTransformer"); - noProvEditTxn.start(); - const transformerNoProv = new IModelTransformer( - { source: sourceDb, target: noProvEditTxn }, - { - noProvenance: true, - } - ); - - const [transformNoProvTimer] = await timed(async () => { - await transformerNoProv.process(); - }); - noProvEditTxn.end(); - - reporter.addEntry( - "populate by transform", - iModelName, - "time elapsed (seconds)", - transformNoProvTimer?.elapsedSeconds ?? -1, + if (testIModel === undefined) + throw Error("Generated iModel not correctly defined"); + const fileName = await testIModel.getFileName(); + sourceDb = StandaloneDb.openFile(fileName, OpenMode.ReadWrite); + + reporter.addEntry( + "populate by insert", + iModelName, + "time elapsed (seconds)", + insertsTimer?.elapsedSeconds ?? -1, + { + "Element Count": count(sourceDb, Element.classFullName), + "Relationship Count": count( + sourceDb, + ElementGroupsMembers.classFullName + ), + "Branch Name": branchName, + } + ); + + Logger.logInfo( + loggerCategory, + "Done. Starting with-provenance transformation of same content" + ); + + const targetPath = initOutputFile("RawInserts-Target.bim", outputDir); + targetDb = SnapshotDb.createEmpty(targetPath, { + rootSubject: { name: "RawInsertsTarget" }, + }); + const withProvEditTxn = new EditTxn(targetDb, "IModelTransformer"); + withProvEditTxn.start(); + const transformerWithProv = new IModelTransformer( + { source: sourceDb, target: withProvEditTxn }, + { noProvenance: false } + ); + + const [transformWithProvTimer] = await timed(async () => { + await transformerWithProv.process(); + }); + withProvEditTxn.end(); + + reporter.addEntry( + "populate by transform (adding provenance)", + iModelName, + "time elapsed (seconds)", + transformWithProvTimer?.elapsedSeconds ?? -1, + { + "Element Count": count(sourceDb, Element.classFullName), + "Relationship Count": count( + sourceDb, + ElementGroupsMembers.classFullName + ), + "Branch Name": branchName, + } + ); + + Logger.logInfo( + loggerCategory, + "Done. Starting without-provenance transformation of same content" + ); + + const targetNoProvPath = initOutputFile( + "RawInserts-TargetNoProv.bim", + outputDir + ); + targetNoProvDb = SnapshotDb.createEmpty(targetNoProvPath, { + rootSubject: { name: "RawInsertsTarget" }, + }); + const noProvEditTxn = new EditTxn(targetNoProvDb, "IModelTransformer"); + noProvEditTxn.start(); + const transformerNoProv = new IModelTransformer( + { source: sourceDb, target: noProvEditTxn }, + { noProvenance: true } + ); + + const [transformNoProvTimer] = await timed(async () => { + await transformerNoProv.process(); + }); + noProvEditTxn.end(); + + reporter.addEntry( + "populate by transform", + iModelName, + "time elapsed (seconds)", + transformNoProvTimer?.elapsedSeconds ?? -1, + { + "Element Count": count(sourceDb, Element.classFullName), + "Relationship Count": count( + sourceDb, + ElementGroupsMembers.classFullName + ), + "Branch Name": branchName, + } + ); + }, [ { - "Element Count": count(sourceDb, Element.classFullName), - "Relationship Count": count(sourceDb, ElementGroupsMembers.classFullName), - "Branch Name": branchName, - } - ); - - sourceDb.close(); - targetDb.close(); - targetNoProvDb.close(); + name: "raw inserts no-provenance target", + run: () => targetNoProvDb?.close(), + }, + { name: "raw inserts provenance target", run: () => targetDb?.close() }, + { name: "raw inserts source", run: () => sourceDb?.close() }, + ]); } diff --git a/packages/performance-tests/test/unit/Cleanup.test.ts b/packages/performance-tests/test/unit/Cleanup.test.ts new file mode 100644 index 000000000..192ff0149 --- /dev/null +++ b/packages/performance-tests/test/unit/Cleanup.test.ts @@ -0,0 +1,55 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { describe, expect, it, vi } from "vitest"; +import { runCleanupTasks, runWithCleanup, throwAfterCleanup } from "../Cleanup"; + +describe("cleanup", () => { + it("runs every cleanup task when one fails", async () => { + const first = vi.fn(() => { + throw new Error("first cleanup"); + }); + const second = vi.fn(); + + await expect( + runCleanupTasks([ + { name: "first", run: first }, + { name: "second", run: second }, + ]) + ).rejects.toThrow("Cleanup failed: first"); + expect(first).toHaveBeenCalledOnce(); + expect(second).toHaveBeenCalledOnce(); + }); + + it("retains the primary failure when cleanup also fails", async () => { + const primaryError = new Error("primary"); + const cleanupError = new Error("cleanup"); + + const error = await throwAfterCleanup(primaryError, [ + { + name: "resource", + run: () => { + throw cleanupError; + }, + }, + ]).catch((caughtError: unknown) => caughtError); + + expect(error).toBeInstanceOf(AggregateError); + expect((error as AggregateError).errors[0]).toBe(primaryError); + expect((error as Error).cause).toBe(primaryError); + expect((error as AggregateError).errors[1]).toMatchObject({ + cause: cleanupError, + }); + }); + + it("cleans up after a successful operation", async () => { + const cleanup = vi.fn(); + + await expect( + runWithCleanup(async () => 42, [{ name: "resource", run: cleanup }]) + ).resolves.toBe(42); + expect(cleanup).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/performance-tests/test/unit/RegressionTestRegistration.test.ts b/packages/performance-tests/test/unit/RegressionTestRegistration.test.ts new file mode 100644 index 000000000..e25bdbffa --- /dev/null +++ b/packages/performance-tests/test/unit/RegressionTestRegistration.test.ts @@ -0,0 +1,56 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { describe, expect, it } from "vitest"; +import { getRegressionTestDefinitions } from "../RegressionTestRegistration"; +import { TestTransformerModule } from "../TestTransformerModule"; + +describe("getRegressionTestDefinitions", () => { + it("registers each supported case and skips unsupported combinations", () => { + const identityCase = () => "identity"; + const forkCase = () => "fork"; + const allOperations: TestTransformerModule = { + createIdentityTransform: async () => ({ run: async () => {} }), + createForkInitTransform: async () => ({ run: async () => {} }), + }; + const identityOnly: TestTransformerModule = { + createIdentityTransform: async () => ({ run: async () => {} }), + }; + + const definitions = getRegressionTestDefinitions( + new Map([ + [ + "identity", + { + testCase: identityCase, + functionNameToValidate: "createIdentityTransform", + }, + ], + [ + "fork", + { + testCase: forkCase, + functionNameToValidate: "createForkInitTransform", + }, + ], + ]), + new Map([ + ["all", allOperations], + ["identity-only", identityOnly], + ]) + ); + + expect( + definitions.map(({ testCaseName, moduleName }) => [ + testCaseName, + moduleName, + ]) + ).toEqual([ + ["identity", "all"], + ["identity", "identity-only"], + ["fork", "all"], + ]); + }); +}); diff --git a/packages/performance-tests/test/vitest.d.ts b/packages/performance-tests/test/vitest.d.ts new file mode 100644 index 000000000..03a558a91 --- /dev/null +++ b/packages/performance-tests/test/vitest.d.ts @@ -0,0 +1,8 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +/// + +export {}; diff --git a/packages/performance-tests/tsconfig.json b/packages/performance-tests/tsconfig.json index 2942eed50..30b1b5890 100644 --- a/packages/performance-tests/tsconfig.json +++ b/packages/performance-tests/tsconfig.json @@ -2,8 +2,8 @@ "extends": "./node_modules/@itwin/build-tools/tsconfig-base.json", "compilerOptions": { "skipLibCheck": true, - "outDir": "./lib", - "esModuleInterop": true + "esModuleInterop": true, + "module": "ESNext" }, "include": ["./test/**/*.ts"] } diff --git a/packages/performance-tests/vitest.config.ts b/packages/performance-tests/vitest.config.ts new file mode 100644 index 000000000..38c500a75 --- /dev/null +++ b/packages/performance-tests/vitest.config.ts @@ -0,0 +1,21 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + globals: true, + environment: "node", + include: ["test/TransformerRegression.test.ts", "test/unit/**/*.test.ts"], + setupFiles: ["./test/setup.ts"], + // Transformations and worker-owned startup/teardown may legitimately run for hours. + testTimeout: 0, + hookTimeout: 0, + pool: "forks", + maxWorkers: 1, + fileParallelism: false, + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7764f6a6d..f03ea5473 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,45 +111,33 @@ importers: '@itwin/oidc-signin-tool': specifier: ^6.0.0 version: 6.0.0(@itwin/core-bentley@5.10.3)(@itwin/service-authorization@2.1.0(@itwin/core-bentley@5.10.3)) - '@types/chai': - specifier: ^4.1.4 - version: 4.3.20 '@types/fs-extra': specifier: ^4.0.7 version: 4.0.15 - '@types/mocha': - specifier: ^8.2.2 - version: 8.2.3 '@types/node': specifier: ^22 version: 22.18.12 '@types/yargs': specifier: ^12.0.5 version: 12.0.20 - chai: - specifier: ^4.3.6 - version: 4.5.0 eslint: specifier: ^9.11.1 version: 9.38.0(supports-color@8.1.1) eslint-config-prettier: specifier: ^9.1.0 version: 9.1.2(eslint@9.38.0(supports-color@8.1.1)) - mocha: - specifier: ^10.0.0 - version: 10.8.2 prettier: specifier: ^3.1.1 version: 3.6.2 rimraf: specifier: ^3.0.2 version: 3.0.2 - ts-node: - specifier: ^10.7.0 - version: 10.9.2(@types/node@22.18.12)(typescript@5.6.3) typescript: specifier: ~5.6.2 version: 5.6.3 + vitest: + specifier: 4.1.10 + version: 4.1.10(@types/node@22.18.12)(@vitest/coverage-v8@4.1.10)(vite@8.1.5(@types/node@22.18.12)(yaml@2.9.0)) packages/test-app: dependencies: @@ -437,10 +425,6 @@ packages: '@bentley/imodeljs-native@5.10.35': resolution: {integrity: sha512-jK26ydqHHCCrTFEuGy4CzJVHAEFlSvp4zBipWr1q/7o6kb2xDhkbmD/LNpwjS98ur3cZh38tCJkzVdArwzy7Kg==} - '@cspotcode/source-map-support@0.8.1': - resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} - engines: {node: '>=12'} - '@emnapi/core@1.11.1': resolution: {integrity: sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==} @@ -738,9 +722,6 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} - '@jridgewell/trace-mapping@0.3.9': - resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} - '@js-sdsl/ordered-map@4.4.2': resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} @@ -981,18 +962,6 @@ packages: resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==} engines: {node: '>= 10'} - '@tsconfig/node10@1.0.11': - resolution: {integrity: sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==} - - '@tsconfig/node12@1.0.11': - resolution: {integrity: sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==} - - '@tsconfig/node14@1.0.3': - resolution: {integrity: sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==} - - '@tsconfig/node16@1.0.4': - resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} - '@tybys/wasm-util@0.10.3': resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==} @@ -1009,9 +978,6 @@ packages: '@types/chai@4.3.1': resolution: {integrity: sha512-/zPMqDkzSZ8t3VtxOa4KPq7uzzW978M9Tvh+j7GHKuo6k6GTLxPJ4J5gE5cjfJ26pnXst0N5Hax8Sr0T2Mi9zQ==} - '@types/chai@4.3.20': - resolution: {integrity: sha512-/pC9HAB5I/xMlc5FP77qjCnI16ChlJfW0tGa0IUcFn38VJrTV6DeZ60NU5KZBtaOZqjdpwTWohz5HU1RrhiYxQ==} - '@types/chai@5.2.3': resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} @@ -1214,10 +1180,6 @@ packages: peerDependencies: acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 - acorn-walk@8.3.4: - resolution: {integrity: sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==} - engines: {node: '>=0.4.0'} - acorn@8.15.0: resolution: {integrity: sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==} engines: {node: '>=0.4.0'} @@ -1296,9 +1258,6 @@ packages: resolution: {integrity: sha512-ixiS0nLNNG5jNQzgZJNoUpBKdo9yTYZMGJ+QgT2jmjR7G7+QHRCc4v6LQ3NgE7EBJq+o0ams3waJwkrlBom8Ig==} engines: {node: '>=14'} - arg@4.1.3: - resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} - argparse@1.0.10: resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} @@ -1352,9 +1311,6 @@ packages: resolution: {integrity: sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==} engines: {node: '>=8'} - assertion-error@1.1.0: - resolution: {integrity: sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==} - assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} @@ -1488,10 +1444,6 @@ packages: canonical-path@1.0.0: resolution: {integrity: sha512-feylzsbDxi1gPZ1IjystzIQZagYYLvfKrSuygUCgf7z6x790VEzze5QEkdSV1U58RA7Hi0+v6fv4K54atOzATg==} - chai@4.5.0: - resolution: {integrity: sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==} - engines: {node: '>=4'} - chai@6.2.2: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} @@ -1511,9 +1463,6 @@ packages: charenc@0.0.2: resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==} - check-error@1.0.3: - resolution: {integrity: sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==} - chokidar@3.6.0: resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==} engines: {node: '>= 8.10.0'} @@ -1600,9 +1549,6 @@ packages: engines: {node: ^20.0.0 || >=22.0.0, npm: '>=10'} hasBin: true - create-require@1.1.1: - resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} - cross-env@5.2.1: resolution: {integrity: sha512-1yHhtcfAd1r4nwQgknowuUNfIT9E8dOMMspC36g45dN+iD1blloi7xp8X/xAIDnjHWyt1uQ8PHk2fkNaym7soQ==} engines: {node: '>=4.0'} @@ -1675,10 +1621,6 @@ packages: resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} engines: {node: '>=10'} - deep-eql@4.1.4: - resolution: {integrity: sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg==} - engines: {node: '>=6'} - deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} @@ -1727,10 +1669,6 @@ packages: engines: {node: '>= 4.2.1'} hasBin: true - diff@4.0.2: - resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==} - engines: {node: '>=0.3.1'} - diff@5.2.0: resolution: {integrity: sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==} engines: {node: '>=0.3.1'} @@ -2216,9 +2154,6 @@ packages: resolution: {integrity: sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==} engines: {node: '>=18'} - get-func-name@2.0.2: - resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==} - get-intrinsic@1.3.0: resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} engines: {node: '>= 0.4'} @@ -2917,9 +2852,6 @@ packages: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true - loupe@2.3.7: - resolution: {integrity: sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==} - lowercase-keys@3.0.0: resolution: {integrity: sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -2951,9 +2883,6 @@ packages: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} engines: {node: '>=10'} - make-error@1.3.6: - resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==} - markdown-it@14.2.0: resolution: {integrity: sha512-1TGiQiJVRQ3NPmZH6sx5Cfnmg6GQm9jvC1ch4TK511NjSJvjzKLzn5pPfZRNZkRPZP0HqCioSndqH8v2nRaWVQ==} hasBin: true @@ -3338,9 +3267,6 @@ packages: pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} - pathval@1.1.1: - resolution: {integrity: sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==} - picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -3890,20 +3816,6 @@ packages: peerDependencies: typescript: '>=4.8.4' - ts-node@10.9.2: - resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==} - hasBin: true - peerDependencies: - '@swc/core': '>=1.2.50' - '@swc/wasm': '>=1.2.50' - '@types/node': '*' - typescript: '>=2.7' - peerDependenciesMeta: - '@swc/core': - optional: true - '@swc/wasm': - optional: true - tsconfig-paths@3.15.0: resolution: {integrity: sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg==} @@ -3914,10 +3826,6 @@ packages: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} - type-detect@4.1.0: - resolution: {integrity: sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==} - engines: {node: '>=4'} - type-is@1.6.18: resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} engines: {node: '>= 0.6'} @@ -4013,9 +3921,6 @@ packages: deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). hasBin: true - v8-compile-cache-lib@3.0.1: - resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} - validate-npm-package-license@3.0.4: resolution: {integrity: sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==} @@ -4247,10 +4152,6 @@ packages: resolution: {integrity: sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==} engines: {node: '>=12'} - yn@3.1.1: - resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} - engines: {node: '>=6'} - yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -4465,10 +4366,6 @@ snapshots: '@bentley/imodeljs-native@5.10.35': {} - '@cspotcode/source-map-support@0.8.1': - dependencies: - '@jridgewell/trace-mapping': 0.3.9 - '@emnapi/core@1.11.1': dependencies: '@emnapi/wasi-threads': 1.2.2 @@ -4904,11 +4801,6 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 - '@jridgewell/trace-mapping@0.3.9': - dependencies: - '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.5 - '@js-sdsl/ordered-map@4.4.2': {} '@microsoft/api-extractor-model@7.33.8(@types/node@22.18.12)': @@ -5129,14 +5021,6 @@ snapshots: '@tootallnate/once@2.0.0': {} - '@tsconfig/node10@1.0.11': {} - - '@tsconfig/node12@1.0.11': {} - - '@tsconfig/node14@1.0.3': {} - - '@tsconfig/node16@1.0.4': {} - '@tybys/wasm-util@0.10.3': dependencies: tslib: 2.8.1 @@ -5152,8 +5036,6 @@ snapshots: '@types/chai@4.3.1': {} - '@types/chai@4.3.20': {} - '@types/chai@5.2.3': dependencies: '@types/deep-eql': 4.0.2 @@ -5494,10 +5376,6 @@ snapshots: dependencies: acorn: 8.15.0 - acorn-walk@8.3.4: - dependencies: - acorn: 8.15.0 - acorn@8.15.0: {} address@1.2.2: {} @@ -5561,8 +5439,6 @@ snapshots: are-docs-informative@0.0.2: {} - arg@4.1.3: {} - argparse@1.0.10: dependencies: sprintf-js: 1.0.3 @@ -5644,8 +5520,6 @@ snapshots: arrify@2.0.1: {} - assertion-error@1.1.0: {} - assertion-error@2.0.1: {} ast-types-flow@0.0.8: {} @@ -5802,16 +5676,6 @@ snapshots: canonical-path@1.0.0: {} - chai@4.5.0: - dependencies: - assertion-error: 1.1.0 - check-error: 1.0.3 - deep-eql: 4.1.4 - get-func-name: 2.0.2 - loupe: 2.3.7 - pathval: 1.1.1 - type-detect: 4.1.0 - chai@6.2.2: {} chalk@2.4.2: @@ -5832,10 +5696,6 @@ snapshots: charenc@0.0.2: {} - check-error@1.0.3: - dependencies: - get-func-name: 2.0.2 - chokidar@3.6.0: dependencies: anymatch: 3.1.3 @@ -5930,8 +5790,6 @@ snapshots: shell-quote: 1.9.0 subarg: 1.0.0 - create-require@1.1.1: {} - cross-env@5.2.1: dependencies: cross-spawn: 6.0.6 @@ -5996,10 +5854,6 @@ snapshots: dependencies: mimic-response: 3.1.0 - deep-eql@4.1.4: - dependencies: - type-detect: 4.1.0 - deep-is@0.1.4: {} default-browser-id@5.0.0: {} @@ -6040,8 +5894,6 @@ snapshots: transitivePeerDependencies: - supports-color - diff@4.0.2: {} - diff@5.2.0: {} diff@7.0.0: {} @@ -6841,8 +6693,6 @@ snapshots: get-east-asian-width@1.6.0: {} - get-func-name@2.0.2: {} - get-intrinsic@1.3.0: dependencies: call-bind-apply-helpers: 1.0.2 @@ -7587,10 +7437,6 @@ snapshots: dependencies: js-tokens: 4.0.0 - loupe@2.3.7: - dependencies: - get-func-name: 2.0.2 - lowercase-keys@3.0.0: {} lru-cache@10.4.3: {} @@ -7622,8 +7468,6 @@ snapshots: dependencies: semver: 7.8.5 - make-error@1.3.6: {} - markdown-it@14.2.0: dependencies: argparse: 2.0.1 @@ -8014,8 +7858,6 @@ snapshots: pathe@2.0.3: {} - pathval@1.1.1: {} - picocolors@1.1.1: {} picomatch@2.3.2: {} @@ -8628,24 +8470,6 @@ snapshots: dependencies: typescript: 5.9.3 - ts-node@10.9.2(@types/node@22.18.12)(typescript@5.6.3): - dependencies: - '@cspotcode/source-map-support': 0.8.1 - '@tsconfig/node10': 1.0.11 - '@tsconfig/node12': 1.0.11 - '@tsconfig/node14': 1.0.3 - '@tsconfig/node16': 1.0.4 - '@types/node': 22.18.12 - acorn: 8.15.0 - acorn-walk: 8.3.4 - arg: 4.1.3 - create-require: 1.1.1 - diff: 4.0.2 - make-error: 1.3.6 - typescript: 5.6.3 - v8-compile-cache-lib: 3.0.1 - yn: 3.1.1 - tsconfig-paths@3.15.0: dependencies: '@types/json5': 0.0.29 @@ -8659,8 +8483,6 @@ snapshots: dependencies: prelude-ls: 1.2.1 - type-detect@4.1.0: {} - type-is@1.6.18: dependencies: media-typer: 0.3.0 @@ -8759,8 +8581,6 @@ snapshots: uuid@9.0.1: {} - v8-compile-cache-lib@3.0.1: {} - validate-npm-package-license@3.0.4: dependencies: spdx-correct: 3.2.0 @@ -8794,7 +8614,7 @@ snapshots: magic-string: 0.30.21 obug: 2.1.4 pathe: 2.0.3 - picomatch: 4.0.4 + picomatch: 4.0.5 std-env: 4.2.0 tinybench: 2.9.0 tinyexec: 1.1.1 @@ -8977,6 +8797,4 @@ snapshots: y18n: 5.0.8 yargs-parser: 21.1.1 - yn@3.1.1: {} - yocto-queue@0.1.0: {} From a52aeffca190fc48609bd7fc8a910b974cf3227a Mon Sep 17 00:00:00 2001 From: Nam Le <50554904+hl662@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:06:45 -0400 Subject: [PATCH 07/19] Build quick-perf fixtures once per run instead of once per sample BenchmarkRunner rebuilt the entire fixture -- seeds, hub, iModels, changesets -- inside the sample loop, so a run paid that cost 9 times. On balanced-incremental that is 94s of reconstruction to measure 1.4s of work, and a heavier changeset-scanning fixture would not fit the 15-minute budget at all. It also made A/B comparison impossible: two arms could not measure the same fixture. Split fixture handling into two stages behind a FixtureProvider seam. Stage 1 builds an immutable artifact once; stage 2 hands each sample a fresh filesystem copy. Samples still start pristine -- only the expensive generation moved out of the loop, not the isolation. Which stage does the real work depends on topology, not recipe: - source-and-empty-target keeps the live hub. Its measured region re-enters BriefcaseManager and pushes provenance to a target briefcase, so it cannot be captured as relocatable bytes. Stage 1 is a no-op there and behaviour is unchanged. - source-only captures the briefcase and its changeset files. This works because BriefcaseDb.open makes no hub calls and ChangedInstanceIds accepts csFileProps without one -- capturing csFileProps is the only hub-free input mode, not an optimization. Measured: 5.3s build once, 29ms per sample thereafter. Fixture selection was hardcoded at both entry points and getFixtureDescriptor threw on anything but balanced-incremental. Scenarios now declare a default fixture id and capabilities; resolution validates the resolved pair. Capabilities validate, they do not select. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/quick-performance.yml | 7 + .../test/quick/BenchmarkReporter.ts | 9 +- .../quick/BenchmarkResolution.quick-unit.ts | 77 ++++++ .../test/quick/BenchmarkResolution.ts | 73 +++++ .../test/quick/BenchmarkRunner.ts | 167 +++++++----- .../test/quick/BenchmarkScenario.ts | 24 ++ .../test/quick/DatasetDescriptor.ts | 18 +- .../test/quick/FixtureArtifact.quick-unit.ts | 167 ++++++++++++ .../test/quick/FixtureArtifact.ts | 170 ++++++++++++ .../test/quick/FixtureCatalog.ts | 73 ++++- .../test/quick/FixtureManifest.ts | 6 +- .../test/quick/FixtureMaterializer.ts | 108 +++----- .../test/quick/FixtureProvider.ts | 74 ++++++ .../test/quick/FixtureRecipe.ts | 60 +++++ .../test/quick/LocalHubFixture.quick-unit.ts | 8 +- .../test/quick/LocalHubFixture.ts | 110 ++++++-- .../test/quick/QuickPerformance.quick.ts | 47 ++-- .../test/quick/ScenarioCatalog.ts | 18 +- packages/performance-tests/test/quick/cli.ts | 18 +- .../providers/detachedBriefcaseProvider.ts | 251 ++++++++++++++++++ .../test/quick/providers/liveHubProvider.ts | 110 ++++++++ .../scenarios/incrementalSynchronization.ts | 25 +- 22 files changed, 1387 insertions(+), 233 deletions(-) create mode 100644 packages/performance-tests/test/quick/BenchmarkResolution.quick-unit.ts create mode 100644 packages/performance-tests/test/quick/BenchmarkResolution.ts create mode 100644 packages/performance-tests/test/quick/FixtureArtifact.quick-unit.ts create mode 100644 packages/performance-tests/test/quick/FixtureArtifact.ts create mode 100644 packages/performance-tests/test/quick/FixtureProvider.ts create mode 100644 packages/performance-tests/test/quick/FixtureRecipe.ts create mode 100644 packages/performance-tests/test/quick/providers/detachedBriefcaseProvider.ts create mode 100644 packages/performance-tests/test/quick/providers/liveHubProvider.ts diff --git a/.github/workflows/quick-performance.yml b/.github/workflows/quick-performance.yml index 597ae6c7f..131e678c8 100644 --- a/.github/workflows/quick-performance.yml +++ b/.github/workflows/quick-performance.yml @@ -10,6 +10,10 @@ on: type: choice options: - incremental-synchronization + fixture: + description: Fixture id (blank uses the scenario's default) + required: false + type: string permissions: contents: read @@ -50,8 +54,10 @@ jobs: --require ts-node/register \ test/quick/FixtureManifest.quick-unit.ts \ test/quick/ScenarioCatalog.quick-unit.ts \ + test/quick/BenchmarkResolution.quick-unit.ts \ test/quick/statistics.quick-unit.ts \ test/quick/LocalHubFixture.quick-unit.ts \ + test/quick/FixtureArtifact.quick-unit.ts \ 2>&1 | sed \ -e "s#${GITHUB_WORKSPACE}##g" \ -e "s#${RUNNER_TEMP}##g" @@ -65,6 +71,7 @@ jobs: env: QUICK_PERF_OUTPUT: ${{ runner.temp }}/quick-performance QUICK_PERF_SCENARIO: ${{ inputs.scenario || 'incremental-synchronization' }} + QUICK_PERF_FIXTURE: ${{ inputs.fixture }} - name: Report variance reliability if: always() diff --git a/packages/performance-tests/test/quick/BenchmarkReporter.ts b/packages/performance-tests/test/quick/BenchmarkReporter.ts index 8d54651f3..946c28999 100644 --- a/packages/performance-tests/test/quick/BenchmarkReporter.ts +++ b/packages/performance-tests/test/quick/BenchmarkReporter.ts @@ -35,7 +35,7 @@ export class BenchmarkReporter { outputDir: string, samples: readonly BenchmarkSample[], jobMilliseconds?: number - ): void { + ) { if (samples.length === 0) throw new Error("Cannot report an empty quick performance sample set"); const scenarioIds = new Set(samples.map((sample) => sample.scenarioId)); @@ -60,6 +60,9 @@ export class BenchmarkReporter { }); const summary = { fixtureId: measured[0]?.fixtureId, + topology: measured[0]?.topology, + /** Stage 1 runs once per job, so this is a scalar, not a per-sample distribution. */ + fixtureBuildMilliseconds: samples[0].fixtureBuildMilliseconds, jobMilliseconds, measuredSamples: measured.length, scenarioId: samples[0].scenarioId, @@ -108,12 +111,13 @@ export class BenchmarkReporter { fs.writeFileSync( path.join(outputDir, "summary.csv"), [ - "scenario,fixture,measuredSamples,jobMs,medianMs,p90Ms,p95Ms,madMs,cv,reconstructionTotalMs,verificationTotalMs,teardownTotalMs", + "scenario,fixture,measuredSamples,jobMs,fixtureBuildMs,medianMs,p90Ms,p95Ms,madMs,cv,reconstructionTotalMs,verificationTotalMs,teardownTotalMs", [ summary.scenarioId, summary.fixtureId, summary.measuredSamples, summary.jobMilliseconds ?? "", + summary.fixtureBuildMilliseconds, summary.wallMilliseconds.median, summary.wallMilliseconds.p90, summary.wallMilliseconds.p95, @@ -125,5 +129,6 @@ export class BenchmarkReporter { ].join(","), ].join("\n") ); + return summary; } } diff --git a/packages/performance-tests/test/quick/BenchmarkResolution.quick-unit.ts b/packages/performance-tests/test/quick/BenchmarkResolution.quick-unit.ts new file mode 100644 index 000000000..413579bd0 --- /dev/null +++ b/packages/performance-tests/test/quick/BenchmarkResolution.quick-unit.ts @@ -0,0 +1,77 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { expect } from "chai"; +import { + assertScenarioSupportsFixture, + resolveBenchmarkRun, + resolveBenchmarkRunFromEnvironment, +} from "./BenchmarkResolution"; +import { BenchmarkScenarioDefinition } from "./BenchmarkScenario"; +import { + balancedIncrementalDescriptor, + balancedIncrementalSourceOnlyDescriptor, +} from "./FixtureCatalog"; +import { incrementalSynchronizationScenario } from "./scenarios/incrementalSynchronization"; + +describe("benchmark resolution", () => { + it("resolves the scenario's declared default fixture", () => { + const resolved = resolveBenchmarkRun(); + expect(resolved.scenario.id).to.equal("incremental-synchronization"); + expect(resolved.descriptor.id).to.equal( + incrementalSynchronizationScenario.defaultFixtureId + ); + }); + + it("lets an explicit fixture id override the default", () => { + const scenario: BenchmarkScenarioDefinition = { + ...incrementalSynchronizationScenario, + capabilities: { topology: "source-only" }, + }; + expect(() => + assertScenarioSupportsFixture( + scenario, + balancedIncrementalSourceOnlyDescriptor + ) + ).to.not.throw(); + }); + + it("rejects a fixture whose topology the scenario cannot consume", () => { + expect(() => + assertScenarioSupportsFixture( + incrementalSynchronizationScenario, + balancedIncrementalSourceOnlyDescriptor + ) + ).to.throw(/requires a "source-and-empty-target" fixture/); + }); + + it("rejects a fixture that does not make a required claim", () => { + const scenario: BenchmarkScenarioDefinition = { + ...incrementalSynchronizationScenario, + capabilities: { + topology: "source-and-empty-target", + requiredClaims: ["time travel"], + }, + }; + expect(() => + assertScenarioSupportsFixture(scenario, balancedIncrementalDescriptor) + ).to.throw(/does not claim \[time travel\]/); + }); + + it("reports unknown fixture ids with the available set", () => { + expect(() => + resolveBenchmarkRun("incremental-synchronization", "no-such-fixture") + ).to.throw(/Available fixtures: balanced-incremental/); + }); + + it("treats blank environment inputs as unspecified", () => { + const resolved = resolveBenchmarkRunFromEnvironment({ + QUICK_PERF_SCENARIO: "", + QUICK_PERF_FIXTURE: " ", + }); + expect(resolved.scenario.id).to.equal("incremental-synchronization"); + expect(resolved.descriptor.id).to.equal("balanced-incremental"); + }); +}); diff --git a/packages/performance-tests/test/quick/BenchmarkResolution.ts b/packages/performance-tests/test/quick/BenchmarkResolution.ts new file mode 100644 index 000000000..6eaee644c --- /dev/null +++ b/packages/performance-tests/test/quick/BenchmarkResolution.ts @@ -0,0 +1,73 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { BenchmarkScenarioDefinition } from "./BenchmarkScenario"; +import { DatasetDescriptor } from "./DatasetDescriptor"; +import { getFixtureDescriptor } from "./FixtureCatalog"; +import { getFixtureRecipe } from "./FixtureRecipe"; +import { getScenarioDefinition } from "./ScenarioCatalog"; + +export interface ResolvedBenchmarkRun { + readonly scenario: BenchmarkScenarioDefinition; + readonly descriptor: DatasetDescriptor; +} + +/** + * Validate a resolved scenario/fixture pair. + * + * Capabilities describe what a scenario *needs*; they do not choose a fixture. Selection has + * already happened by the time this runs, so a mismatch is a configuration error, not a signal to + * pick something else. + */ +export function assertScenarioSupportsFixture( + scenario: BenchmarkScenarioDefinition, + descriptor: DatasetDescriptor +): void { + const { capabilities } = scenario; + if (descriptor.layout.topology !== capabilities.topology) + throw new Error( + `Scenario "${scenario.id}" requires a "${capabilities.topology}" fixture but "${descriptor.id}" is "${descriptor.layout.topology}"` + ); + const missing = (capabilities.requiredClaims ?? []).filter( + (claim) => !descriptor.scenarioClaims.includes(claim) + ); + if (missing.length > 0) + throw new Error( + `Fixture "${descriptor.id}" does not claim [${missing.join( + ", " + )}] required by scenario "${scenario.id}"` + ); + // A fixture is useless if nothing knows how to generate it. + getFixtureRecipe(descriptor.layout.recipe); +} + +/** + * Resolve the scenario and the fixture it will run against. `fixtureId` overrides the scenario's + * declared default; omit it for the normal path. + */ +export function resolveBenchmarkRun( + scenarioId?: string, + fixtureId?: string +): ResolvedBenchmarkRun { + const scenario = getScenarioDefinition(scenarioId); + const descriptor = getFixtureDescriptor( + fixtureId ?? scenario.defaultFixtureId + ); + assertScenarioSupportsFixture(scenario, descriptor); + return { scenario, descriptor }; +} + +/** Resolve from the environment, as both entry points do. */ +export function resolveBenchmarkRunFromEnvironment( + env: NodeJS.ProcessEnv = process.env +): ResolvedBenchmarkRun { + // CI passes unset inputs through as empty strings; treat those as "not specified". + const orUndefined = (value: string | undefined) => + value === undefined || value.trim() === "" ? undefined : value.trim(); + return resolveBenchmarkRun( + orUndefined(env.QUICK_PERF_SCENARIO), + orUndefined(env.QUICK_PERF_FIXTURE) + ); +} diff --git a/packages/performance-tests/test/quick/BenchmarkRunner.ts b/packages/performance-tests/test/quick/BenchmarkRunner.ts index 48146058d..a3ebd1757 100644 --- a/packages/performance-tests/test/quick/BenchmarkRunner.ts +++ b/packages/performance-tests/test/quick/BenchmarkRunner.ts @@ -7,17 +7,21 @@ import * as fs from "fs"; import * as os from "os"; import * as path from "path"; import { IModelHost } from "@itwin/core-backend"; +import { assertScenarioSupportsFixture } from "./BenchmarkResolution"; import { BenchmarkScenario, BenchmarkScenarioDefinition, } from "./BenchmarkScenario"; -import { DatasetDescriptor } from "./DatasetDescriptor"; -import { materializeFixture, PreparedDataset } from "./FixtureMaterializer"; -import { disposeReconstructedHub } from "./LocalHubFixture"; +import { DatasetDescriptor, FixtureTopology } from "./DatasetDescriptor"; +import { PreparedDataset } from "./FixtureMaterializer"; +import { FixtureProvider, getFixtureProvider } from "./FixtureProvider"; export const benchmarkOutputMarkerName = ".imodel-transformer-quick-performance"; +/** Where stage 1 writes its artifact, relative to the run output directory. */ +export const fixtureArtifactDirectoryName = "fixture-artifact"; + function normalizeError(error: unknown): Error { return error instanceof Error ? error : new Error(String(error)); } @@ -78,6 +82,7 @@ export function prepareBenchmarkOutputDirectory(outputDir: string): void { if ( /^sample-\d+$/.test(entry) || [ + fixtureArtifactDirectoryName, "manifest.json", "samples.jsonl", "summary.csv", @@ -92,6 +97,7 @@ export function prepareBenchmarkOutputDirectory(outputDir: string): void { } async function cleanupSample( + provider: FixtureProvider, scenario: BenchmarkScenario | undefined, dataset: PreparedDataset | undefined, sampleDir: string @@ -103,7 +109,7 @@ async function cleanupSample( errors.push(error); } try { - if (dataset) await disposeReconstructedHub(dataset.hub); + if (dataset) await provider.disposeSample(dataset); } catch (error) { errors.push(error); } @@ -121,12 +127,16 @@ export interface BenchmarkSample { readonly fixtureId: string; readonly measured: boolean; readonly operations: DatasetDescriptor["distribution"]["operations"]; + /** Stage-1 cost, identical on every sample: the fixture is built once per run. */ + readonly fixtureBuildMilliseconds: number; + /** Stage-2 cost for this sample: producing its pristine working copy. */ readonly reconstructionMilliseconds: number; readonly rssDeltaBytes: number; readonly sample: number; readonly scenarioId: string; readonly semanticDigest: string; readonly teardownMilliseconds: number; + readonly topology: FixtureTopology; readonly verificationMilliseconds: number; readonly wallMilliseconds: number; } @@ -136,7 +146,9 @@ export class BenchmarkRunner { private readonly _descriptor: DatasetDescriptor, private readonly _outputDir: string, private readonly _scenario: BenchmarkScenarioDefinition - ) {} + ) { + assertScenarioSupportsFixture(_scenario, _descriptor); + } public async run(measuredSamples = 8): Promise { if (!Number.isInteger(measuredSamples) || measuredSamples < 1) @@ -144,73 +156,94 @@ export class BenchmarkRunner { "Quick performance requires at least one measured sample" ); prepareBenchmarkOutputDirectory(this._outputDir); + const provider = getFixtureProvider(this._descriptor); const samples: BenchmarkSample[] = []; await IModelHost.startup(); try { - for (let sample = 0; sample <= measuredSamples; sample++) { - const sampleDir = path.join(this._outputDir, `sample-${sample}`); - let dataset: PreparedDataset | undefined; - let scenario: BenchmarkScenario | undefined; - let operationError: Error | undefined; - let completedSample: - | Omit - | undefined; - try { - dataset = await materializeFixture( - this._descriptor, - sampleDir, - `quick-sample-${sample}` - ); - scenario = this._scenario.factory(dataset); - const rssBefore = process.memoryUsage().rss; - const cpuBefore = process.cpuUsage(); - const wallStart = process.hrtime.bigint(); - await scenario.measure(); - const wallMilliseconds = - Number(process.hrtime.bigint() - wallStart) / 1_000_000; - const cpu = process.cpuUsage(cpuBefore); - const rssDeltaBytes = process.memoryUsage().rss - rssBefore; - const verificationStart = process.hrtime.bigint(); - const semanticDigest = await scenario.finish(); - const verificationMilliseconds = - Number(process.hrtime.bigint() - verificationStart) / 1_000_000; - completedSample = { - cpuSystemMilliseconds: cpu.system / 1000, - cpuUserMilliseconds: cpu.user / 1000, - fixtureId: this._descriptor.id, - measured: sample !== 0, - operations: this._descriptor.distribution.operations, - reconstructionMilliseconds: dataset.reconstructionMilliseconds, - rssDeltaBytes, - sample, - scenarioId: this._scenario.id, - semanticDigest, - verificationMilliseconds, - wallMilliseconds, - }; - } catch (error) { - operationError = normalizeError(error); - } - const teardownStart = process.hrtime.bigint(); - const cleanupErrors = await cleanupSample(scenario, dataset, sampleDir); - const teardownMilliseconds = - Number(process.hrtime.bigint() - teardownStart) / 1_000_000; - if (operationError && cleanupErrors.length === 0) throw operationError; - if (cleanupErrors.length > 0) - throw new AggregateError( - operationError ? [operationError, ...cleanupErrors] : cleanupErrors, - "Quick performance sample cleanup failed" + // Stage 1: build the fixture exactly once, outside the sample loop. + const built = await provider.build( + this._descriptor, + path.join(this._outputDir, fixtureArtifactDirectoryName) + ); + try { + for (let sample = 0; sample <= measuredSamples; sample++) { + const sampleDir = path.join(this._outputDir, `sample-${sample}`); + let dataset: PreparedDataset | undefined; + let scenario: BenchmarkScenario | undefined; + let operationError: Error | undefined; + let completedSample: + | Omit + | undefined; + try { + // Stage 2: a pristine working copy per sample. Mutation is the scenario's business. + dataset = await provider.materialize( + built, + sampleDir, + `quick-sample-${sample}` + ); + scenario = this._scenario.factory(dataset); + const rssBefore = process.memoryUsage().rss; + const cpuBefore = process.cpuUsage(); + const wallStart = process.hrtime.bigint(); + await scenario.measure(); + const wallMilliseconds = + Number(process.hrtime.bigint() - wallStart) / 1_000_000; + const cpu = process.cpuUsage(cpuBefore); + const rssDeltaBytes = process.memoryUsage().rss - rssBefore; + const verificationStart = process.hrtime.bigint(); + const semanticDigest = await scenario.finish(); + const verificationMilliseconds = + Number(process.hrtime.bigint() - verificationStart) / 1_000_000; + completedSample = { + cpuSystemMilliseconds: cpu.system / 1000, + cpuUserMilliseconds: cpu.user / 1000, + fixtureBuildMilliseconds: built.buildMilliseconds, + fixtureId: this._descriptor.id, + measured: sample !== 0, + operations: this._descriptor.distribution.operations, + reconstructionMilliseconds: dataset.reconstructionMilliseconds, + rssDeltaBytes, + sample, + scenarioId: this._scenario.id, + semanticDigest, + topology: this._descriptor.layout.topology, + verificationMilliseconds, + wallMilliseconds, + }; + } catch (error) { + operationError = normalizeError(error); + } + const teardownStart = process.hrtime.bigint(); + const cleanupErrors = await cleanupSample( + provider, + scenario, + dataset, + sampleDir ); - if (!completedSample) - throw new Error( - "Quick performance sample completed without a result" + const teardownMilliseconds = + Number(process.hrtime.bigint() - teardownStart) / 1_000_000; + if (operationError && cleanupErrors.length === 0) + throw operationError; + if (cleanupErrors.length > 0) + throw new AggregateError( + operationError + ? [operationError, ...cleanupErrors] + : cleanupErrors, + "Quick performance sample cleanup failed" + ); + if (!completedSample) + throw new Error( + "Quick performance sample completed without a result" + ); + const sampleResult = { ...completedSample, teardownMilliseconds }; + samples.push(sampleResult); + fs.appendFileSync( + path.join(this._outputDir, "samples.jsonl"), + `${JSON.stringify(sampleResult)}\n` ); - const sampleResult = { ...completedSample, teardownMilliseconds }; - samples.push(sampleResult); - fs.appendFileSync( - path.join(this._outputDir, "samples.jsonl"), - `${JSON.stringify(sampleResult)}\n` - ); + } + } finally { + await provider.disposeBuild(built); } } finally { await IModelHost.shutdown(); diff --git a/packages/performance-tests/test/quick/BenchmarkScenario.ts b/packages/performance-tests/test/quick/BenchmarkScenario.ts index ea153ece9..9fa281bee 100644 --- a/packages/performance-tests/test/quick/BenchmarkScenario.ts +++ b/packages/performance-tests/test/quick/BenchmarkScenario.ts @@ -3,6 +3,7 @@ * See LICENSE.md in the project root for license terms and full copyright notice. *--------------------------------------------------------------------------------------------*/ +import { FixtureTopology } from "./DatasetDescriptor"; import { PreparedDataset } from "./FixtureMaterializer"; export interface BenchmarkScenario { @@ -15,7 +16,30 @@ export type BenchmarkScenarioFactory = ( dataset: PreparedDataset ) => BenchmarkScenario; +/** + * What a scenario needs from a fixture. Capabilities *validate* a resolved scenario/fixture pair; + * they never select one. Selection is {@link BenchmarkScenarioDefinition.defaultFixtureId} or an + * explicit override. + */ +export interface BenchmarkScenarioCapabilities { + readonly topology: FixtureTopology; + /** Claims the fixture must advertise in `scenarioClaims`. */ + readonly requiredClaims?: readonly string[]; +} + export interface BenchmarkScenarioDefinition { readonly id: string; + readonly defaultFixtureId: string; + readonly capabilities: BenchmarkScenarioCapabilities; readonly factory: BenchmarkScenarioFactory; + /** Wall-clock budget for the whole run, in milliseconds. */ + readonly budgetMilliseconds?: number; +} + +export const defaultScenarioBudgetMilliseconds = 15 * 60 * 1000; + +export function scenarioBudgetMilliseconds( + scenario: BenchmarkScenarioDefinition +): number { + return scenario.budgetMilliseconds ?? defaultScenarioBudgetMilliseconds; } diff --git a/packages/performance-tests/test/quick/DatasetDescriptor.ts b/packages/performance-tests/test/quick/DatasetDescriptor.ts index edfd7b259..67cc8e1e0 100644 --- a/packages/performance-tests/test/quick/DatasetDescriptor.ts +++ b/packages/performance-tests/test/quick/DatasetDescriptor.ts @@ -25,6 +25,21 @@ export interface FixtureDistribution { readonly operations: FixtureOperationCounts; } +/** + * The *shape* of a fixture, independent of the change mix a recipe applies to it. + * + * - `source-and-empty-target`: a source briefcase plus an empty target that has been transformed + * into. Requires a live hub at measure time, so its working copy is a full per-sample rebuild. + * - `source-only`: a source briefcase and its pushed changeset files, with no target and no hub at + * measure time. Built once into an immutable artifact and copied per sample. + */ +export type FixtureTopology = "source-and-empty-target" | "source-only"; + +export const fixtureTopologies: readonly FixtureTopology[] = [ + "source-and-empty-target", + "source-only", +]; + export interface DatasetDescriptor { readonly id: string; readonly version: number; @@ -32,7 +47,8 @@ export interface DatasetDescriptor { readonly scenarioClaims: readonly string[]; readonly layout: { readonly kind: "reconstructed"; - readonly recipe: "balanced-incremental"; + readonly topology: FixtureTopology; + readonly recipe: string; readonly seed: number; }; readonly distribution: FixtureDistribution; diff --git a/packages/performance-tests/test/quick/FixtureArtifact.quick-unit.ts b/packages/performance-tests/test/quick/FixtureArtifact.quick-unit.ts new file mode 100644 index 000000000..f715d360e --- /dev/null +++ b/packages/performance-tests/test/quick/FixtureArtifact.quick-unit.ts @@ -0,0 +1,167 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { expect } from "chai"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { IModelHost } from "@itwin/core-backend"; +import { HubMock } from "@itwin/core-backend/lib/cjs/internal/HubMock"; +import { ChangedInstanceIds } from "@itwin/imodel-transformer"; +import { + artifactBriefcasePath, + artifactManifestFileName, + readChangesetFileProps, + readFixtureArtifact, +} from "./FixtureArtifact"; +import { balancedIncrementalSourceOnlyDescriptor } from "./FixtureCatalog"; +import { requireDetachedDataset } from "./FixtureMaterializer"; +import { BuiltFixture, getFixtureProvider } from "./FixtureProvider"; +import { detachedBriefcaseFixtureProvider } from "./providers/detachedBriefcaseProvider"; + +describe("detached fixture artifact", () => { + const descriptor = balancedIncrementalSourceOnlyDescriptor; + let root: string; + let built: BuiltFixture; + + before(async function () { + this.timeout(600_000); + root = fs.mkdtempSync(path.join(os.tmpdir(), "quick-artifact-")); + await IModelHost.startup(); + built = await detachedBriefcaseFixtureProvider.build( + descriptor, + path.join(root, "fixture-artifact") + ); + }); + + after(async () => { + await detachedBriefcaseFixtureProvider.disposeBuild(built); + if (IModelHost.isValid) await IModelHost.shutdown(); + fs.rmSync(root, { recursive: true, force: true }); + }); + + it("selects the detached provider from the descriptor topology", () => { + expect(getFixtureProvider(descriptor)).to.equal( + detachedBriefcaseFixtureProvider + ); + }); + + it("emits a self-describing artifact and leaves no build scaffolding", () => { + const entries = fs.readdirSync(built.directory).sort(); + expect(entries).to.deep.equal([ + "briefcase.bim", + "changesets", + "csFileProps.json", + artifactManifestFileName, + ]); + const artifact = readFixtureArtifact(built.directory); + expect(artifact.manifest.descriptor.recipeHash).to.equal( + descriptor.recipeHash + ); + expect(artifact.manifest.changesets.count).to.equal( + descriptor.distribution.operations.sourceChangesets + ); + expect( + fs.readdirSync(path.join(built.directory, "changesets")) + ).to.have.length(artifact.manifest.changesets.count); + }); + + it("stores relocatable changeset pathnames", () => { + const raw = JSON.parse( + fs.readFileSync(path.join(built.directory, "csFileProps.json"), "utf8") + ) as { pathname: string }[]; + for (const props of raw) { + expect(path.isAbsolute(props.pathname)).to.equal( + false, + "artifact pathnames must be relative so the artifact can be copied" + ); + expect(props.pathname.startsWith("changesets/")).to.be.true; + } + // The reader is the only supported consumer: it rebases to the copy's location. + for (const props of readChangesetFileProps(built.directory)) + expect(fs.existsSync(props.pathname)).to.be.true; + }); + + it("materializes independent working copies without a hub", async function () { + this.timeout(300_000); + expect(HubMock.isValid).to.equal( + false, + "stage 2 must not require a live hub" + ); + const copyCosts: number[] = []; + const digests: string[] = []; + for (let sample = 0; sample < 3; sample++) { + const dataset = requireDetachedDataset( + await detachedBriefcaseFixtureProvider.materialize( + built, + path.join(root, `sample-${sample}`), + `sample-${sample}` + ) + ); + try { + copyCosts.push(dataset.reconstructionMilliseconds); + const changes = await ChangedInstanceIds.initialize({ + iModel: dataset.sourceDb, + csFileProps: dataset.csFileProps, + }); + if (!changes) + throw new Error("ChangedInstanceIds.initialize returned nothing"); + digests.push( + JSON.stringify({ + inserts: changes.element.insertIds.size, + updates: changes.element.updateIds.size, + deletes: changes.element.deleteIds.size, + }) + ); + } finally { + await detachedBriefcaseFixtureProvider.disposeSample(dataset); + } + } + expect(new Set(digests).size).to.equal( + 1, + "every working copy must present the same fixture" + ); + // Not an assertion of speed — a record of the budget stage 2 consumes per sample. + const median = [...copyCosts].sort((a, b) => a - b)[1]; + const artifactBytes = fs + .readdirSync(built.directory, { recursive: true, withFileTypes: true }) + .filter((entry) => entry.isFile()) + .reduce( + (sum, entry) => + sum + fs.statSync(path.join(entry.parentPath, entry.name)).size, + 0 + ); + process.stdout.write( + `\n artifact: ${(artifactBytes / 1024 / 1024).toFixed( + 2 + )}MiB; stage-1 build: ${built.buildMilliseconds.toFixed( + 0 + )}ms; stage-2 copy per sample: ${copyCosts + .map((cost) => cost.toFixed(1)) + .join(", ")}ms (median ${median.toFixed(1)}ms)\n` + ); + }); + + it("rejects an artifact whose manifest is inconsistent with its contents", () => { + const corrupt = path.join(root, "corrupt-artifact"); + fs.cpSync(built.directory, corrupt, { recursive: true }); + fs.rmSync(artifactBriefcasePath(corrupt)); + expect(() => readFixtureArtifact(corrupt)).to.throw(); + fs.rmSync(corrupt, { recursive: true, force: true }); + }); + + it("rejects changeset props that escape the artifact directory", () => { + const corrupt = path.join(root, "escaping-artifact"); + fs.cpSync(built.directory, corrupt, { recursive: true }); + const propsFile = path.join(corrupt, "csFileProps.json"); + const props = JSON.parse(fs.readFileSync(propsFile, "utf8")) as { + pathname: string; + }[]; + props[0].pathname = "/tmp/elsewhere.cs"; + fs.writeFileSync(propsFile, JSON.stringify(props)); + expect(() => readChangesetFileProps(corrupt)).to.throw(); + fs.rmSync(corrupt, { recursive: true, force: true }); + }); +}); diff --git a/packages/performance-tests/test/quick/FixtureArtifact.ts b/packages/performance-tests/test/quick/FixtureArtifact.ts new file mode 100644 index 000000000..a9d50670c --- /dev/null +++ b/packages/performance-tests/test/quick/FixtureArtifact.ts @@ -0,0 +1,170 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "fs"; +import * as path from "path"; +import { ChangesetFileProps } from "@itwin/core-common"; +import { DatasetDescriptor } from "./DatasetDescriptor"; +import { validateDescriptor } from "./FixtureManifest"; + +/** + * Version of the on-disk artifact layout. Bump when the directory contract changes in a way that + * an older reader cannot interpret. + */ +export const fixtureArtifactVersion = 1; + +export const artifactBriefcaseFileName = "briefcase.bim"; +export const artifactChangesetDirectoryName = "changesets"; +export const artifactChangesetPropsFileName = "csFileProps.json"; +export const artifactManifestFileName = "manifest.json"; + +export interface FixtureArtifactManifest { + readonly artifactVersion: number; + readonly descriptor: DatasetDescriptor; + readonly briefcase: { + readonly fileName: string; + readonly briefcaseId: number; + readonly changeset: { readonly id: string; readonly index?: number }; + readonly byteLength: number; + }; + readonly changesets: { + readonly directory: string; + readonly propsFile: string; + readonly count: number; + /** Index of the changeset immediately before the first captured changeset. */ + readonly baseChangesetIndex: number; + readonly firstIndex?: number; + readonly lastIndex?: number; + }; + readonly buildMilliseconds: number; + readonly builtAt: string; +} + +/** A stage-1 fixture artifact: immutable bytes that stage 2 copies from. */ +export interface FixtureArtifact { + readonly directory: string; + readonly manifest: FixtureArtifactManifest; +} + +export function changesetArtifactFileName( + changeset: Pick +): string { + return `${String(changeset.index).padStart(6, "0")}-${changeset.id}.cs`; +} + +/** Rewrite `pathname` to a POSIX path relative to the artifact root. */ +export function toRelativeChangesetProps( + changesets: readonly ChangesetFileProps[] +): ChangesetFileProps[] { + return changesets.map((changeset) => ({ + ...changeset, + pathname: `${artifactChangesetDirectoryName}/${changesetArtifactFileName( + changeset + )}`, + })); +} + +function assertRelativeChangesetPath(pathname: unknown, index: number): string { + if (typeof pathname !== "string" || pathname.length === 0) + throw new Error(`Changeset ${index} is missing a pathname`); + if (path.isAbsolute(pathname) || pathname.includes("..")) + throw new Error( + `Changeset ${index} pathname must be relative to the artifact root: ${pathname}` + ); + return pathname; +} + +/** + * Read `csFileProps.json` from an artifact or working copy, rebasing every `pathname` to an + * absolute path below `directory`. This is the only supported reader: `ChangedInstanceIds` opens + * changesets by `pathname` verbatim, so relative paths must never reach it. + */ +export function readChangesetFileProps( + directory: string +): ChangesetFileProps[] { + const propsFile = path.join(directory, artifactChangesetPropsFileName); + const parsed: unknown = JSON.parse(fs.readFileSync(propsFile, "utf8")); + if (!Array.isArray(parsed)) + throw new Error(`Fixture artifact ${propsFile} must contain an array`); + return parsed.map((entry, index) => { + const changeset = entry as ChangesetFileProps; + const relative = assertRelativeChangesetPath(changeset.pathname, index); + const pathname = path.join(directory, ...relative.split("/")); + if (!fs.existsSync(pathname)) + throw new Error( + `Fixture artifact is missing changeset file: ${pathname}` + ); + return { ...changeset, pathname }; + }); +} + +export function artifactBriefcasePath(directory: string): string { + return path.join(directory, artifactBriefcaseFileName); +} + +export function writeFixtureArtifactManifest( + directory: string, + manifest: FixtureArtifactManifest +): void { + fs.writeFileSync( + path.join(directory, artifactManifestFileName), + `${JSON.stringify(manifest, undefined, 2)}\n` + ); +} + +export function validateFixtureArtifactManifest( + value: unknown +): FixtureArtifactManifest { + if (value === null || typeof value !== "object") + throw new Error("Fixture artifact manifest must be an object"); + const manifest = value as Partial; + if (manifest.artifactVersion !== fixtureArtifactVersion) + throw new Error( + `Unsupported fixture artifact version ${String( + manifest.artifactVersion + )}; expected ${fixtureArtifactVersion}` + ); + if ( + typeof manifest.briefcase?.fileName !== "string" || + typeof manifest.briefcase.briefcaseId !== "number" || + typeof manifest.briefcase.changeset?.id !== "string" || + typeof manifest.briefcase.byteLength !== "number" || + typeof manifest.changesets?.directory !== "string" || + typeof manifest.changesets.propsFile !== "string" || + typeof manifest.changesets.count !== "number" || + typeof manifest.changesets.baseChangesetIndex !== "number" || + typeof manifest.buildMilliseconds !== "number" || + typeof manifest.builtAt !== "string" + ) + throw new Error("Fixture artifact manifest has an invalid shape"); + validateDescriptor(manifest.descriptor); + return manifest as FixtureArtifactManifest; +} + +/** + * Read and validate an artifact directory. Verifies the manifest against what is actually on disk + * so a truncated or partially-copied artifact fails here rather than deep inside a benchmark. + */ +export function readFixtureArtifact(directory: string): FixtureArtifact { + const manifest = validateFixtureArtifactManifest( + JSON.parse( + fs.readFileSync(path.join(directory, artifactManifestFileName), "utf8") + ) + ); + const briefcase = path.join(directory, manifest.briefcase.fileName); + if (!fs.existsSync(briefcase)) + throw new Error(`Fixture artifact is missing its briefcase: ${briefcase}`); + const byteLength = fs.statSync(briefcase).size; + if (byteLength !== manifest.briefcase.byteLength) + throw new Error( + `Fixture artifact briefcase is ${byteLength} bytes but its manifest declares ${manifest.briefcase.byteLength}` + ); + const changesets = readChangesetFileProps(directory); + if (changesets.length !== manifest.changesets.count) + throw new Error( + `Fixture artifact has ${changesets.length} changesets but its manifest declares ${manifest.changesets.count}` + ); + return { directory, manifest }; +} diff --git a/packages/performance-tests/test/quick/FixtureCatalog.ts b/packages/performance-tests/test/quick/FixtureCatalog.ts index 1133982d0..124a50cb0 100644 --- a/packages/performance-tests/test/quick/FixtureCatalog.ts +++ b/packages/performance-tests/test/quick/FixtureCatalog.ts @@ -50,9 +50,10 @@ const generator = { transformer: packageVersion("@itwin/imodel-transformer"), }; -const recipeIdentity = { +const recipeIdentity = (topology: string) => ({ schema: "QuickPerf.01.00.00", seed: 328, + topology, distribution, inputs: { recipe: fs.readFileSync( @@ -69,30 +70,78 @@ const recipeIdentity = { ), }, versions: generator, -}; +}); + +const scenarioClaims = [ + "incremental synchronization", + "aspect lifecycle", + "relationship lifecycle", + "mixed scalar and geometry element changes", +]; export const balancedIncrementalDescriptor: DatasetDescriptor = { id: "balanced-incremental", version: 1, label: "balanced incremental", - scenarioClaims: [ - "incremental synchronization", - "aspect lifecycle", - "relationship lifecycle", - "mixed scalar and geometry element changes", - ], + scenarioClaims, layout: { kind: "reconstructed", + topology: "source-and-empty-target", recipe: "balanced-incremental", seed: 328, }, distribution, generator, - recipeHash: canonicalSha256(recipeIdentity), + recipeHash: canonicalSha256(recipeIdentity("source-and-empty-target")), }; +/** + * The same change mix captured as a relocatable artifact instead of rebuilt per sample. + * + * Topology, not recipe, is what differs: there is no target iModel, so nothing re-enters the hub + * and the source briefcase plus its changesets can be copied per sample. + */ +export const balancedIncrementalSourceOnlyDescriptor: DatasetDescriptor = { + id: "balanced-incremental-source-only", + version: 1, + label: "balanced incremental (source only)", + scenarioClaims: [...scenarioClaims, "changeset scanning"], + layout: { + kind: "reconstructed", + topology: "source-only", + recipe: "balanced-incremental", + seed: 328, + }, + distribution, + generator, + recipeHash: canonicalSha256(recipeIdentity("source-only")), +}; + +const fixtures = new Map([ + [balancedIncrementalDescriptor.id, balancedIncrementalDescriptor], + [ + balancedIncrementalSourceOnlyDescriptor.id, + balancedIncrementalSourceOnlyDescriptor, + ], +]); + +export function registerFixtureDescriptor(descriptor: DatasetDescriptor): void { + if (fixtures.has(descriptor.id)) + throw new Error(`Duplicate quick performance fixture: ${descriptor.id}`); + fixtures.set(descriptor.id, descriptor); +} + +export function listFixtureIds(): string[] { + return [...fixtures.keys()]; +} + export function getFixtureDescriptor(id: string): DatasetDescriptor { - if (id !== balancedIncrementalDescriptor.id) - throw new Error(`Unknown quick performance fixture: ${id}`); - return balancedIncrementalDescriptor; + const descriptor = fixtures.get(id); + if (!descriptor) + throw new Error( + `Unknown quick performance fixture "${id}". Available fixtures: ${listFixtureIds().join( + ", " + )}` + ); + return descriptor; } diff --git a/packages/performance-tests/test/quick/FixtureManifest.ts b/packages/performance-tests/test/quick/FixtureManifest.ts index e266caa32..59221bd46 100644 --- a/packages/performance-tests/test/quick/FixtureManifest.ts +++ b/packages/performance-tests/test/quick/FixtureManifest.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { createHash } from "crypto"; -import { DatasetDescriptor } from "./DatasetDescriptor"; +import { DatasetDescriptor, fixtureTopologies } from "./DatasetDescriptor"; function canonicalize(value: unknown): string { if (Array.isArray(value)) @@ -33,7 +33,9 @@ export function validateDescriptor(value: unknown): DatasetDescriptor { typeof descriptor.label !== "string" || !Array.isArray(descriptor.scenarioClaims) || descriptor.layout?.kind !== "reconstructed" || - descriptor.layout.recipe !== "balanced-incremental" || + !fixtureTopologies.includes(descriptor.layout.topology) || + typeof descriptor.layout.recipe !== "string" || + descriptor.layout.recipe.length === 0 || typeof descriptor.layout.seed !== "number" || descriptor.distribution === undefined || typeof descriptor.generator?.coreBackend !== "string" || diff --git a/packages/performance-tests/test/quick/FixtureMaterializer.ts b/packages/performance-tests/test/quick/FixtureMaterializer.ts index 3d1abca9f..9a898a7d8 100644 --- a/packages/performance-tests/test/quick/FixtureMaterializer.ts +++ b/packages/performance-tests/test/quick/FixtureMaterializer.ts @@ -4,81 +4,55 @@ *--------------------------------------------------------------------------------------------*/ import * as path from "path"; -import { IModelTransformer } from "@itwin/imodel-transformer"; +import { BriefcaseDb } from "@itwin/core-backend"; +import { ChangesetFileProps } from "@itwin/core-common"; import { DatasetDescriptor } from "./DatasetDescriptor"; -import { - createStartedEditTxn, - disposeReconstructedHub, - ReconstructedHub, - reconstructHub, -} from "./LocalHubFixture"; -import { - applyBalancedChangesets, - BalancedRecipeState, - createBalancedSeed, -} from "./recipes/balancedIncremental"; -import { assertFixtureDistribution } from "./validation/validateFixture"; +import { FixtureArtifactManifest } from "./FixtureArtifact"; +import { ReconstructedHub } from "./LocalHubFixture"; -export interface PreparedDataset { +interface PreparedDatasetBase { readonly descriptor: DatasetDescriptor; - readonly hub: ReconstructedHub; + /** Stage-2 cost for this sample: what it took to hand the scenario a pristine working copy. */ readonly reconstructionMilliseconds: number; } -export async function materializeFixture( - descriptor: DatasetDescriptor, - outputDir: string, - sampleName: string -): Promise { - const start = process.hrtime.bigint(); - let recipeState: BalancedRecipeState | undefined; - let hub: ReconstructedHub | undefined; - try { - hub = await reconstructHub(outputDir, sampleName, async (sourceSeed) => { - recipeState = await createBalancedSeed(sourceSeed, descriptor); - }); - if (!recipeState) - throw new Error("Balanced fixture recipe did not create state"); +/** A live HubMock with an open source briefcase and an already-transformed-into target. */ +export interface PreparedLiveHubDataset extends PreparedDatasetBase { + readonly topology: "source-and-empty-target"; + readonly hub: ReconstructedHub; +} + +/** A working copy of a stage-1 artifact: an open readonly source and its changeset files. */ +export interface PreparedDetachedDataset extends PreparedDatasetBase { + readonly topology: "source-only"; + /** Root of this sample's working copy. */ + readonly directory: string; + readonly sourceDb: BriefcaseDb; + /** Changeset props whose `pathname` values are absolute and point into `directory`. */ + readonly csFileProps: ChangesetFileProps[]; + readonly manifest: FixtureArtifactManifest; +} + +export type PreparedDataset = PreparedLiveHubDataset | PreparedDetachedDataset; + +export function requireLiveHubDataset( + dataset: PreparedDataset +): PreparedLiveHubDataset { + if (dataset.topology !== "source-and-empty-target") + throw new Error( + `Scenario requires a "source-and-empty-target" fixture but received "${dataset.topology}"` + ); + return dataset; +} - const editTxn = createStartedEditTxn(hub.targetDb); - const transformer = new IModelTransformer({ - source: hub.sourceDb, - target: editTxn, - }); - try { - await transformer.processSchemas(); - await transformer.process(); - } finally { - transformer.dispose(); - if (editTxn.isActive) editTxn.end(); - } - await hub.targetDb.pushChanges({ - accessToken: hub.accessToken, - description: "establish quick fixture provenance", - }); - await applyBalancedChangesets( - hub.sourceDb, - hub.accessToken, - descriptor, - recipeState +export function requireDetachedDataset( + dataset: PreparedDataset +): PreparedDetachedDataset { + if (dataset.topology !== "source-only") + throw new Error( + `Scenario requires a "source-only" fixture but received "${dataset.topology}"` ); - await assertFixtureDistribution(hub.sourceDb, descriptor); - const reconstructionMilliseconds = - Number(process.hrtime.bigint() - start) / 1_000_000; - return { descriptor, hub, reconstructionMilliseconds }; - } catch (error) { - if (hub) { - try { - await disposeReconstructedHub(hub); - } catch (cleanupError) { - throw new AggregateError( - [error, cleanupError], - "Fixture materialization and cleanup both failed" - ); - } - } - throw error; - } + return dataset; } export function fixtureWorkingDirectory( diff --git a/packages/performance-tests/test/quick/FixtureProvider.ts b/packages/performance-tests/test/quick/FixtureProvider.ts new file mode 100644 index 000000000..c3a8681a7 --- /dev/null +++ b/packages/performance-tests/test/quick/FixtureProvider.ts @@ -0,0 +1,74 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { DatasetDescriptor } from "./DatasetDescriptor"; +import { FixtureArtifact } from "./FixtureArtifact"; +import { PreparedDataset } from "./FixtureMaterializer"; +import { detachedBriefcaseFixtureProvider } from "./providers/detachedBriefcaseProvider"; +import { liveHubFixtureProvider } from "./providers/liveHubProvider"; + +/** + * The result of stage 1. + * + * `artifact` is present only for topologies that can be captured as relocatable bytes. A + * live-hub topology cannot: its measured region re-enters the hub, so there is nothing to + * capture and stage 1 is structural only. + */ +export interface BuiltFixture { + readonly descriptor: DatasetDescriptor; + readonly directory: string; + readonly buildMilliseconds: number; + readonly artifact?: FixtureArtifact; +} + +export function requireFixtureArtifact(built: BuiltFixture): FixtureArtifact { + if (!built.artifact) + throw new Error( + `Fixture "${built.descriptor.id}" did not produce an on-disk artifact` + ); + return built.artifact; +} + +/** + * Two-stage fixture lifecycle. + * + * Stage 1 ({@link FixtureProvider.build}) runs once per benchmark run. Nothing measures against + * its output directly. Stage 2 ({@link FixtureProvider.materialize}) hands each sample its own + * pristine working copy, so a scenario may mutate what it is given without the framework caring. + * The default is always a fresh copy per sample; the split only moves the expensive part out of + * the loop. + */ +export interface FixtureProvider { + /** Build once, outside the sample loop. */ + build( + descriptor: DatasetDescriptor, + artifactDir: string + ): Promise; + /** Produce a fresh working copy for one sample. */ + materialize( + built: BuiltFixture, + sampleDir: string, + sampleName: string + ): Promise; + /** Release one sample's working copy. */ + disposeSample(dataset: PreparedDataset): Promise; + /** Release stage-1 state, after every sample has been disposed. */ + disposeBuild(built: BuiltFixture): Promise; +} + +export function getFixtureProvider( + descriptor: DatasetDescriptor +): FixtureProvider { + switch (descriptor.layout.topology) { + case "source-and-empty-target": + return liveHubFixtureProvider; + case "source-only": + return detachedBriefcaseFixtureProvider; + default: { + const unreachable: never = descriptor.layout.topology; + throw new Error(`Unknown fixture topology: ${String(unreachable)}`); + } + } +} diff --git a/packages/performance-tests/test/quick/FixtureRecipe.ts b/packages/performance-tests/test/quick/FixtureRecipe.ts new file mode 100644 index 000000000..173996307 --- /dev/null +++ b/packages/performance-tests/test/quick/FixtureRecipe.ts @@ -0,0 +1,60 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { AccessToken } from "@itwin/core-bentley"; +import { BriefcaseDb } from "@itwin/core-backend"; +import { DatasetDescriptor } from "./DatasetDescriptor"; +import { + applyBalancedChangesets, + BalancedRecipeState, + createBalancedSeed, +} from "./recipes/balancedIncremental"; + +/** + * A recipe produces the *change mix* for a fixture: it seeds the source iModel and then applies a + * deterministic series of pushed changesets. It never touches HubMock; the fixture provider owns + * the hub lifecycle. + */ +export interface FixtureRecipe { + readonly id: string; + /** Create the source seed file. Returns state carried into {@link applySourceChangesets}. */ + createSeed(fileName: string, descriptor: DatasetDescriptor): Promise; + /** Apply and push the recipe's changesets to an open source briefcase. */ + applySourceChangesets( + db: BriefcaseDb, + accessToken: AccessToken, + descriptor: DatasetDescriptor, + state: TState + ): Promise; +} + +export const balancedIncrementalRecipe: FixtureRecipe = { + id: "balanced-incremental", + createSeed: async (fileName, descriptor) => + createBalancedSeed(fileName, descriptor), + applySourceChangesets: async (db, accessToken, descriptor, state) => + applyBalancedChangesets(db, accessToken, descriptor, state), +}; + +const recipes = new Map>([ + [balancedIncrementalRecipe.id, balancedIncrementalRecipe], +]); + +export function registerFixtureRecipe(recipe: FixtureRecipe): void { + if (recipes.has(recipe.id)) + throw new Error(`Duplicate quick performance recipe: ${recipe.id}`); + recipes.set(recipe.id, recipe); +} + +export function getFixtureRecipe(id: string): FixtureRecipe { + const recipe = recipes.get(id); + if (!recipe) + throw new Error( + `Unknown quick performance recipe "${id}". Available recipes: ${[ + ...recipes.keys(), + ].join(", ")}` + ); + return recipe; +} diff --git a/packages/performance-tests/test/quick/LocalHubFixture.quick-unit.ts b/packages/performance-tests/test/quick/LocalHubFixture.quick-unit.ts index 59b2d28b0..eab5c382e 100644 --- a/packages/performance-tests/test/quick/LocalHubFixture.quick-unit.ts +++ b/packages/performance-tests/test/quick/LocalHubFixture.quick-unit.ts @@ -31,7 +31,7 @@ import { prepareBenchmarkOutputDirectory, } from "./BenchmarkRunner"; import { balancedIncrementalDescriptor } from "./FixtureCatalog"; -import { materializeFixture } from "./FixtureMaterializer"; +import { materializeLiveHubFixture } from "./providers/liveHubProvider"; import { createStartedEditTxn, disposeReconstructedHub, @@ -137,7 +137,7 @@ describe("LocalHubFixture reconstruction", () => { }; let failure: unknown; try { - await materializeFixture( + await materializeLiveHubFixture( invalidDescriptor, path.join(outputDir, "materialize-failure"), "materialize-failure" @@ -331,6 +331,8 @@ describe("BenchmarkRunner scenario injection", function () { const calls = { abort: 0, factory: 0, finish: 0, measure: 0 }; const scenario: BenchmarkScenarioDefinition = { id: "injected-scenario", + defaultFixtureId: balancedIncrementalDescriptor.id, + capabilities: { topology: "source-and-empty-target" }, factory: (dataset) => { calls.factory++; const delegate = incrementalSynchronization(dataset); @@ -406,6 +408,8 @@ describe("BenchmarkRunner scenario injection", function () { let aborts = 0; const scenario: BenchmarkScenarioDefinition = { id: "failing-scenario", + defaultFixtureId: balancedIncrementalDescriptor.id, + capabilities: { topology: "source-and-empty-target" }, factory: () => ({ abort() { aborts++; diff --git a/packages/performance-tests/test/quick/LocalHubFixture.ts b/packages/performance-tests/test/quick/LocalHubFixture.ts index d3f7307df..bf43d1080 100644 --- a/packages/performance-tests/test/quick/LocalHubFixture.ts +++ b/packages/performance-tests/test/quick/LocalHubFixture.ts @@ -17,11 +17,14 @@ import { HubMock } from "@itwin/core-backend/lib/cjs/internal/HubMock"; // eslint-disable-next-line @itwin/no-internal import { _hubAccess } from "@itwin/core-backend/lib/cjs/internal/Symbols"; -export interface ReconstructedHub { +export interface ReconstructedSourceHub { readonly accessToken: AccessToken; readonly iTwinId: string; readonly sourceDb: BriefcaseDb; readonly sourceIModelId: string; +} + +export interface ReconstructedHub extends ReconstructedSourceHub { readonly targetDb: BriefcaseDb; readonly targetIModelId: string; } @@ -91,6 +94,10 @@ export async function closeAndDeleteBriefcase( await BriefcaseManager.deleteBriefcaseFiles(fileName, accessToken); } +export function shutdownHubMock(): void { + if (HubMock.isValid) HubMock.shutdown(); +} + async function cleanupHub( accessToken: AccessToken, briefcases: readonly BriefcaseDb[] @@ -104,18 +111,27 @@ async function cleanupHub( } } try { - if (HubMock.isValid) HubMock.shutdown(); + shutdownHubMock(); } catch (error) { errors.push(error); } return errors; } -export async function reconstructHub( +async function reconstruct( outputDir: string, mockName: string, - createSourceSeed: (fileName: string) => Promise | void -): Promise { + createSourceSeed: (fileName: string) => Promise | void, + finish: ( + context: { + accessToken: AccessToken; + iTwinId: string; + seedDir: string; + track: (db: BriefcaseDb) => void; + }, + source: { db: BriefcaseDb; iModelId: string } + ) => Promise +): Promise { if (HubMock.isValid) throw new Error("Only one HubMock may be active"); fs.mkdirSync(outputDir, { recursive: true }); @@ -140,10 +156,8 @@ export async function reconstructHub( try { const seedDir = path.join(outputDir, "seeds"); const sourceSeed = path.join(seedDir, `${mockName}-source.bim`); - const targetSeed = path.join(seedDir, `${mockName}-target.bim`); fs.mkdirSync(seedDir, { recursive: true }); await createSourceSeed(sourceSeed); - createEmptySeed(targetSeed, `${mockName}-target`); const source = await createAndOpenIModel( accessToken, @@ -152,21 +166,15 @@ export async function reconstructHub( sourceSeed ); openBriefcases.push(source.db); - const target = await createAndOpenIModel( - accessToken, - iTwinId, - `${mockName}-target`, - targetSeed + return await finish( + { + accessToken, + iTwinId, + seedDir, + track: (db) => openBriefcases.push(db), + }, + source ); - openBriefcases.push(target.db); - return { - accessToken, - iTwinId, - sourceDb: source.db, - sourceIModelId: source.iModelId, - targetDb: target.db, - targetIModelId: target.iModelId, - }; } catch (error) { const cleanupErrors = await cleanupHub(accessToken, openBriefcases); if (cleanupErrors.length > 0) @@ -178,13 +186,63 @@ export async function reconstructHub( } } +/** Start a HubMock holding only a source iModel. Used by artifact-backed topologies. */ +export async function reconstructSourceHub( + outputDir: string, + mockName: string, + createSourceSeed: (fileName: string) => Promise | void +): Promise { + return reconstruct( + outputDir, + mockName, + createSourceSeed, + async ({ accessToken, iTwinId }, source) => ({ + accessToken, + iTwinId, + sourceDb: source.db, + sourceIModelId: source.iModelId, + }) + ); +} + +/** Start a HubMock holding a source iModel and an empty target iModel. */ +export async function reconstructHub( + outputDir: string, + mockName: string, + createSourceSeed: (fileName: string) => Promise | void +): Promise { + return reconstruct( + outputDir, + mockName, + createSourceSeed, + async ({ accessToken, iTwinId, seedDir, track }, source) => { + const targetSeed = path.join(seedDir, `${mockName}-target.bim`); + createEmptySeed(targetSeed, `${mockName}-target`); + const target = await createAndOpenIModel( + accessToken, + iTwinId, + `${mockName}-target`, + targetSeed + ); + track(target.db); + return { + accessToken, + iTwinId, + sourceDb: source.db, + sourceIModelId: source.iModelId, + targetDb: target.db, + targetIModelId: target.iModelId, + }; + } + ); +} + export async function disposeReconstructedHub( - hub: ReconstructedHub + hub: ReconstructedHub | ReconstructedSourceHub ): Promise { - const errors = await cleanupHub(hub.accessToken, [ - hub.sourceDb, - hub.targetDb, - ]); + const briefcases: BriefcaseDb[] = [hub.sourceDb]; + if ("targetDb" in hub) briefcases.push(hub.targetDb); + const errors = await cleanupHub(hub.accessToken, briefcases); if (errors.length > 0) throw new AggregateError(errors, "Failed to dispose reconstructed HubMock"); } diff --git a/packages/performance-tests/test/quick/QuickPerformance.quick.ts b/packages/performance-tests/test/quick/QuickPerformance.quick.ts index 14e6747cc..4dc98ada7 100644 --- a/packages/performance-tests/test/quick/QuickPerformance.quick.ts +++ b/packages/performance-tests/test/quick/QuickPerformance.quick.ts @@ -3,45 +3,36 @@ * See LICENSE.md in the project root for license terms and full copyright notice. *--------------------------------------------------------------------------------------------*/ -import * as path from "path"; import { expect } from "chai"; +import * as path from "path"; import { BenchmarkReporter } from "./BenchmarkReporter"; +import { resolveBenchmarkRunFromEnvironment } from "./BenchmarkResolution"; import { BenchmarkRunner } from "./BenchmarkRunner"; -import { getFixtureDescriptor } from "./FixtureCatalog"; -import { getScenarioDefinition } from "./ScenarioCatalog"; +import { scenarioBudgetMilliseconds } from "./BenchmarkScenario"; -describe("quick transformer performance", function () { - this.timeout(15 * 60 * 1000); +describe("quick performance", () => { + const { descriptor, scenario } = resolveBenchmarkRunFromEnvironment(); + const budgetMilliseconds = scenarioBudgetMilliseconds(scenario); - it("runs the balanced incremental synchronization fixture", async () => { - const scenario = getScenarioDefinition(process.env.QUICK_PERF_SCENARIO); - const measuredSamples = Number(process.env.QUICK_PERF_SAMPLES ?? "8"); + it(`${scenario.id} completes within budget on ${descriptor.id}`, async function () { + this.timeout(budgetMilliseconds); const outputDir = process.env.QUICK_PERF_OUTPUT ?? - path.join(__dirname, ".quick-output", scenario.id); - const runner = new BenchmarkRunner( - getFixtureDescriptor("balanced-incremental"), + path.join(__dirname, ".quick-output", descriptor.id); + const started = process.hrtime.bigint(); + const samples = await new BenchmarkRunner( + descriptor, outputDir, scenario - ); - const jobStart = process.hrtime.bigint(); - const samples = await runner.run(measuredSamples); - const jobMilliseconds = - Number(process.hrtime.bigint() - jobStart) / 1_000_000; - BenchmarkReporter.write(outputDir, samples, jobMilliseconds); + ).run(); + const elapsedMilliseconds = + Number(process.hrtime.bigint() - started) / 1_000_000; + const summary = BenchmarkReporter.write(outputDir, samples); - expect(samples.filter((sample) => sample.measured)).to.have.length( - measuredSamples - ); + expect(summary.measuredSamples).to.equal(8); expect( new Set(samples.map((sample) => sample.semanticDigest)).size - ).to.equal( - 1, - "fresh reconstructions must produce the same semantic digest" - ); - expect(new Set(samples.map((sample) => sample.scenarioId))).to.deep.equal( - new Set([scenario.id]) - ); - expect(jobMilliseconds).to.be.lessThan(15 * 60 * 1000); + ).to.equal(1, "every sample must observe the same fixture"); + expect(elapsedMilliseconds).to.be.lessThan(budgetMilliseconds); }); }); diff --git a/packages/performance-tests/test/quick/ScenarioCatalog.ts b/packages/performance-tests/test/quick/ScenarioCatalog.ts index eb9f5d002..a67b944a5 100644 --- a/packages/performance-tests/test/quick/ScenarioCatalog.ts +++ b/packages/performance-tests/test/quick/ScenarioCatalog.ts @@ -13,6 +13,18 @@ const scenarios = new Map([ [incrementalSynchronizationScenario.id, incrementalSynchronizationScenario], ]); +export function registerScenarioDefinition( + scenario: BenchmarkScenarioDefinition +): void { + if (scenarios.has(scenario.id)) + throw new Error(`Duplicate quick performance scenario: ${scenario.id}`); + scenarios.set(scenario.id, scenario); +} + +export function listScenarioIds(): string[] { + return [...scenarios.keys()]; +} + export function getScenarioDefinition( requestedId?: string ): BenchmarkScenarioDefinition { @@ -20,9 +32,9 @@ export function getScenarioDefinition( const scenario = scenarios.get(scenarioId); if (!scenario) throw new Error( - `Unknown quick performance scenario "${scenarioId}". Available scenarios: ${[ - ...scenarios.keys(), - ].join(", ")}` + `Unknown quick performance scenario "${scenarioId}". Available scenarios: ${listScenarioIds().join( + ", " + )}` ); return scenario; } diff --git a/packages/performance-tests/test/quick/cli.ts b/packages/performance-tests/test/quick/cli.ts index 85f2b39b0..7d5addddd 100644 --- a/packages/performance-tests/test/quick/cli.ts +++ b/packages/performance-tests/test/quick/cli.ts @@ -6,41 +6,41 @@ import * as fs from "fs"; import * as path from "path"; import { BenchmarkReporter } from "./BenchmarkReporter"; +import { resolveBenchmarkRunFromEnvironment } from "./BenchmarkResolution"; import { BenchmarkRunner, prepareBenchmarkOutputDirectory, } from "./BenchmarkRunner"; -import { balancedIncrementalDescriptor } from "./FixtureCatalog"; -import { getScenarioDefinition } from "./ScenarioCatalog"; +import { DatasetDescriptor } from "./DatasetDescriptor"; -function writeManifest(outputDir: string): void { +function writeManifest(outputDir: string, descriptor: DatasetDescriptor): void { fs.writeFileSync( path.join(outputDir, "manifest.json"), - `${JSON.stringify(balancedIncrementalDescriptor, undefined, 2)}\n` + `${JSON.stringify(descriptor, undefined, 2)}\n` ); } async function main() { const command = process.argv[2]; + const { descriptor, scenario } = resolveBenchmarkRunFromEnvironment(); const outputDir = process.env.QUICK_PERF_OUTPUT ?? - path.join(__dirname, ".quick-output", balancedIncrementalDescriptor.id); + path.join(__dirname, ".quick-output", descriptor.id); if (command === "build-fixture") { prepareBenchmarkOutputDirectory(outputDir); - writeManifest(outputDir); + writeManifest(outputDir, descriptor); return; } if (command === "verify-fixture") { - const scenario = getScenarioDefinition(process.env.QUICK_PERF_SCENARIO); const samples = await new BenchmarkRunner( - balancedIncrementalDescriptor, + descriptor, outputDir, scenario ).run(1); if (new Set(samples.map((sample) => sample.semanticDigest)).size !== 1) throw new Error("Fixture reconstruction is not deterministic"); BenchmarkReporter.write(outputDir, samples); - writeManifest(outputDir); + writeManifest(outputDir, descriptor); return; } throw new Error(`Unknown quick fixture command: ${command ?? ""}`); diff --git a/packages/performance-tests/test/quick/providers/detachedBriefcaseProvider.ts b/packages/performance-tests/test/quick/providers/detachedBriefcaseProvider.ts new file mode 100644 index 000000000..4bfe4c1d9 --- /dev/null +++ b/packages/performance-tests/test/quick/providers/detachedBriefcaseProvider.ts @@ -0,0 +1,251 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "fs"; +import * as path from "path"; +import { BriefcaseDb, BriefcaseManager } from "@itwin/core-backend"; +import { DatasetDescriptor } from "../DatasetDescriptor"; +import { + artifactBriefcaseFileName, + artifactBriefcasePath, + artifactChangesetDirectoryName, + artifactChangesetPropsFileName, + changesetArtifactFileName, + FixtureArtifactManifest, + fixtureArtifactVersion, + readChangesetFileProps, + readFixtureArtifact, + toRelativeChangesetProps, + writeFixtureArtifactManifest, +} from "../FixtureArtifact"; +import { + BuiltFixture, + FixtureProvider, + requireFixtureArtifact, +} from "../FixtureProvider"; +import { getFixtureRecipe } from "../FixtureRecipe"; +import { + PreparedDataset, + requireDetachedDataset, +} from "../FixtureMaterializer"; +import { + ReconstructedSourceHub, + reconstructSourceHub, + shutdownHubMock, +} from "../LocalHubFixture"; +import { assertFixtureDistribution } from "../validation/validateFixture"; + +/** + * Tolerant teardown for a build that may have failed at any point. Collects errors rather than + * throwing so an originating error is never masked. + * + * `briefcaseClosed` distinguishes the success path — which closes the briefcase early so its bytes + * can be copied — from a failure that left it open. + */ +async function releaseBuildHub( + hub: ReconstructedSourceHub, + briefcaseFileName: string | undefined, + briefcaseClosed: boolean +): Promise { + const errors: unknown[] = []; + if (!briefcaseClosed) { + try { + hub.sourceDb.close(); + } catch (error) { + errors.push(error); + } + } + try { + if (briefcaseFileName && fs.existsSync(briefcaseFileName)) + await BriefcaseManager.deleteBriefcaseFiles( + briefcaseFileName, + hub.accessToken + ); + } catch (error) { + errors.push(error); + } + try { + shutdownHubMock(); + } catch (error) { + errors.push(error); + } + return errors; +} + +/** + * The `source-only` topology: a source briefcase plus its pushed changeset files, with no target + * and no hub at measure time. Stage 1 does all the expensive changeset generation exactly once; + * stage 2 is a filesystem copy. + * + * The captured bytes are relocatable: `BriefcaseDb.open` makes no hub calls, and + * `ChangedInstanceIds.initialize` accepts `csFileProps` without one. + */ +export const detachedBriefcaseFixtureProvider: FixtureProvider = { + async build( + descriptor: DatasetDescriptor, + artifactDir: string + ): Promise { + const recipe = getFixtureRecipe(descriptor.layout.recipe); + const start = process.hrtime.bigint(); + fs.rmSync(artifactDir, { recursive: true, force: true }); + fs.mkdirSync(artifactDir, { recursive: true }); + + const hubDir = path.join(artifactDir, ".build-hub"); + const scratchDir = path.join(artifactDir, ".build-changesets"); + const changesetDir = path.join(artifactDir, artifactChangesetDirectoryName); + + let hub: ReconstructedSourceHub | undefined; + let briefcaseFileName: string | undefined; + let briefcaseClosed = false; + try { + let recipeState: unknown; + hub = await reconstructSourceHub( + hubDir, + `quick-artifact-${descriptor.id}`, + async (sourceSeed) => { + recipeState = await recipe.createSeed(sourceSeed, descriptor); + } + ); + briefcaseFileName = hub.sourceDb.pathName; + // The seed is uploaded as version 0, so the briefcase starts before any changeset. + const baseChangesetIndex = hub.sourceDb.changeset.index ?? 0; + + await recipe.applySourceChangesets( + hub.sourceDb, + hub.accessToken, + descriptor, + recipeState + ); + await assertFixtureDistribution(hub.sourceDb, descriptor); + + fs.mkdirSync(scratchDir, { recursive: true }); + const downloaded = await BriefcaseManager.downloadChangesets({ + accessToken: hub.accessToken, + iModelId: hub.sourceIModelId, + range: { first: baseChangesetIndex + 1 }, + targetDir: scratchDir, + }); + + const briefcaseId = hub.sourceDb.briefcaseId; + const briefcaseChangeset = hub.sourceDb.changeset; + // Close before copying so every page is flushed into the file we capture. + hub.sourceDb.close(); + briefcaseClosed = true; + const destination = artifactBriefcasePath(artifactDir); + fs.copyFileSync(briefcaseFileName, destination); + + fs.mkdirSync(changesetDir, { recursive: true }); + for (const changeset of downloaded) + fs.copyFileSync( + changeset.pathname, + path.join(changesetDir, changesetArtifactFileName(changeset)) + ); + fs.writeFileSync( + path.join(artifactDir, artifactChangesetPropsFileName), + `${JSON.stringify(toRelativeChangesetProps(downloaded), undefined, 2)}\n` + ); + + const releaseErrors = await releaseBuildHub( + hub, + briefcaseFileName, + briefcaseClosed + ); + if (releaseErrors.length > 0) + throw new AggregateError( + releaseErrors, + "Failed to release the fixture build hub" + ); + hub = undefined; + + const buildMilliseconds = + Number(process.hrtime.bigint() - start) / 1_000_000; + const indices = downloaded.map((changeset) => changeset.index); + const manifest: FixtureArtifactManifest = { + artifactVersion: fixtureArtifactVersion, + descriptor, + briefcase: { + fileName: artifactBriefcaseFileName, + briefcaseId, + changeset: { + id: briefcaseChangeset.id, + index: briefcaseChangeset.index, + }, + byteLength: fs.statSync(destination).size, + }, + changesets: { + directory: artifactChangesetDirectoryName, + propsFile: artifactChangesetPropsFileName, + count: downloaded.length, + baseChangesetIndex, + firstIndex: indices.length > 0 ? Math.min(...indices) : undefined, + lastIndex: indices.length > 0 ? Math.max(...indices) : undefined, + }, + buildMilliseconds, + builtAt: new Date().toISOString(), + }; + writeFixtureArtifactManifest(artifactDir, manifest); + return { + descriptor, + directory: artifactDir, + buildMilliseconds, + artifact: readFixtureArtifact(artifactDir), + }; + } catch (error) { + if (hub) { + const cleanupErrors = await releaseBuildHub( + hub, + briefcaseFileName, + briefcaseClosed + ); + if (cleanupErrors.length > 0) + throw new AggregateError( + [error, ...cleanupErrors], + "Fixture artifact build and cleanup both failed" + ); + } + throw error; + } finally { + // Build-time scaffolding must never reach a working copy. + fs.rmSync(hubDir, { recursive: true, force: true }); + fs.rmSync(scratchDir, { recursive: true, force: true }); + } + }, + + async materialize( + built: BuiltFixture, + sampleDir: string + ): Promise { + const artifact = requireFixtureArtifact(built); + const start = process.hrtime.bigint(); + fs.rmSync(sampleDir, { recursive: true, force: true }); + fs.mkdirSync(path.dirname(sampleDir), { recursive: true }); + fs.cpSync(artifact.directory, sampleDir, { recursive: true }); + + const csFileProps = readChangesetFileProps(sampleDir); + const sourceDb = await BriefcaseDb.open({ + fileName: artifactBriefcasePath(sampleDir), + readonly: true, + }); + return { + topology: "source-only", + descriptor: built.descriptor, + directory: sampleDir, + sourceDb, + csFileProps, + manifest: artifact.manifest, + reconstructionMilliseconds: + Number(process.hrtime.bigint() - start) / 1_000_000, + }; + }, + + async disposeSample(dataset: PreparedDataset): Promise { + // No hub, no briefcase registration to release: the working copy is just files. + requireDetachedDataset(dataset).sourceDb.close(); + }, + + async disposeBuild(built: BuiltFixture): Promise { + fs.rmSync(built.directory, { recursive: true, force: true }); + }, +}; diff --git a/packages/performance-tests/test/quick/providers/liveHubProvider.ts b/packages/performance-tests/test/quick/providers/liveHubProvider.ts new file mode 100644 index 000000000..25815be2c --- /dev/null +++ b/packages/performance-tests/test/quick/providers/liveHubProvider.ts @@ -0,0 +1,110 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { IModelTransformer } from "@itwin/imodel-transformer"; +import { DatasetDescriptor } from "../DatasetDescriptor"; +import { BuiltFixture, FixtureProvider } from "../FixtureProvider"; +import { getFixtureRecipe } from "../FixtureRecipe"; +import { PreparedDataset, requireLiveHubDataset } from "../FixtureMaterializer"; +import { + createStartedEditTxn, + disposeReconstructedHub, + ReconstructedHub, + reconstructHub, +} from "../LocalHubFixture"; +import { assertFixtureDistribution } from "../validation/validateFixture"; + +/** + * The `source-and-empty-target` topology. + * + * Its measured region re-enters the hub (changeset queries, provenance pushes), so it cannot be + * captured as relocatable bytes. Stage 1 is therefore a no-op and stage 2 performs the whole + * reconstruction — behaviour identical to the pre-two-stage runner. + */ +export const liveHubFixtureProvider: FixtureProvider = { + async build( + descriptor: DatasetDescriptor, + artifactDir: string + ): Promise { + getFixtureRecipe(descriptor.layout.recipe); + return { descriptor, directory: artifactDir, buildMilliseconds: 0 }; + }, + + async materialize( + built: BuiltFixture, + sampleDir: string, + sampleName: string + ): Promise { + const { descriptor } = built; + const recipe = getFixtureRecipe(descriptor.layout.recipe); + const start = process.hrtime.bigint(); + let recipeState: unknown; + let hub: ReconstructedHub | undefined; + try { + hub = await reconstructHub(sampleDir, sampleName, async (sourceSeed) => { + recipeState = await recipe.createSeed(sourceSeed, descriptor); + }); + + const editTxn = createStartedEditTxn(hub.targetDb); + const transformer = new IModelTransformer({ + source: hub.sourceDb, + target: editTxn, + }); + try { + await transformer.processSchemas(); + await transformer.process(); + } finally { + transformer.dispose(); + if (editTxn.isActive) editTxn.end(); + } + await hub.targetDb.pushChanges({ + accessToken: hub.accessToken, + description: "establish quick fixture provenance", + }); + await recipe.applySourceChangesets( + hub.sourceDb, + hub.accessToken, + descriptor, + recipeState + ); + await assertFixtureDistribution(hub.sourceDb, descriptor); + return { + topology: "source-and-empty-target", + descriptor, + hub, + reconstructionMilliseconds: + Number(process.hrtime.bigint() - start) / 1_000_000, + }; + } catch (error) { + if (hub) { + try { + await disposeReconstructedHub(hub); + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + "Fixture materialization and cleanup both failed" + ); + } + } + throw error; + } + }, + + async disposeSample(dataset: PreparedDataset): Promise { + await disposeReconstructedHub(requireLiveHubDataset(dataset).hub); + }, + + async disposeBuild(): Promise {}, +}; + +/** Materialize directly from a descriptor, bypassing the (no-op) stage-1 build. */ +export async function materializeLiveHubFixture( + descriptor: DatasetDescriptor, + sampleDir: string, + sampleName: string +): Promise { + const built = await liveHubFixtureProvider.build(descriptor, sampleDir); + return liveHubFixtureProvider.materialize(built, sampleDir, sampleName); +} diff --git a/packages/performance-tests/test/quick/scenarios/incrementalSynchronization.ts b/packages/performance-tests/test/quick/scenarios/incrementalSynchronization.ts index 0bae9e2ea..dff673b02 100644 --- a/packages/performance-tests/test/quick/scenarios/incrementalSynchronization.ts +++ b/packages/performance-tests/test/quick/scenarios/incrementalSynchronization.ts @@ -4,12 +4,11 @@ *--------------------------------------------------------------------------------------------*/ import { IModelTransformer } from "@itwin/imodel-transformer"; -import { HubMock } from "@itwin/core-backend/lib/cjs/internal/HubMock"; import { BenchmarkScenario, BenchmarkScenarioDefinition, } from "../BenchmarkScenario"; -import { PreparedDataset } from "../FixtureMaterializer"; +import { PreparedDataset, requireLiveHubDataset } from "../FixtureMaterializer"; import { createStartedEditTxn } from "../LocalHubFixture"; import { assertSemanticallyEqual, @@ -19,11 +18,10 @@ import { export function incrementalSynchronization( dataset: PreparedDataset ): BenchmarkScenario { - if (!HubMock.isValid) - throw new Error("Quick performance scenarios require an active HubMock"); - const editTxn = createStartedEditTxn(dataset.hub.targetDb); + const { hub } = requireLiveHubDataset(dataset); + const editTxn = createStartedEditTxn(hub.targetDb); const transformer = new IModelTransformer( - { source: dataset.hub.sourceDb, target: editTxn }, + { source: hub.sourceDb, target: editTxn }, { argsForProcessChanges: {} } ); let disposed = false; @@ -40,19 +38,18 @@ export function incrementalSynchronization( }, async finish() { dispose(); - await assertSynchronizationProvenance( - dataset.hub.sourceDb, - dataset.hub.targetDb - ); - return assertSemanticallyEqual( - dataset.hub.sourceDb, - dataset.hub.targetDb - ); + await assertSynchronizationProvenance(hub.sourceDb, hub.targetDb); + return assertSemanticallyEqual(hub.sourceDb, hub.targetDb); }, }; } export const incrementalSynchronizationScenario: BenchmarkScenarioDefinition = { id: "incremental-synchronization", + defaultFixtureId: "balanced-incremental", + capabilities: { + topology: "source-and-empty-target", + requiredClaims: ["incremental synchronization"], + }, factory: incrementalSynchronization, }; From 61c82593b6ece9baca81af57168ce6a0be7b2366 Mon Sep 17 00:00:00 2001 From: Nam Le <50554904+hl662@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:09:09 -0400 Subject: [PATCH 08/19] perf(tests): add update-heavy changeset scan recipe Adds the recipe body and schema for the changeset-scanning scenario, plus ledger serialization so the oracle's expectation can cross the fixture build/measure boundary. Elements are partitioned into four regions so every id has one unambiguous expected outcome after ChangedInstanceIds squashes the scanned range, including a region that is inserted and then deleted and must therefore cancel out entirely. Verified against a live HubMock at five shapes: the oracle matches the scan exactly across all six ChangedInstanceOps collections. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../test/quick/recipes/updateHeavyScan.ts | 447 ++++++++++++++++++ .../quick/schemas/QuickPerfScan.ecschema.xml | 11 + .../test/quick/validation/scanOracle.ts | 14 + 3 files changed, 472 insertions(+) create mode 100644 packages/performance-tests/test/quick/recipes/updateHeavyScan.ts create mode 100644 packages/performance-tests/test/quick/schemas/QuickPerfScan.ecschema.xml diff --git a/packages/performance-tests/test/quick/recipes/updateHeavyScan.ts b/packages/performance-tests/test/quick/recipes/updateHeavyScan.ts new file mode 100644 index 000000000..dc00dfa79 --- /dev/null +++ b/packages/performance-tests/test/quick/recipes/updateHeavyScan.ts @@ -0,0 +1,447 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as path from "path"; +import { AccessToken, Id64String } from "@itwin/core-bentley"; +import { + Code, + CodeScopeSpec, + CodeSpec, + ElementAspectProps, + IModel, + PhysicalElementProps, +} from "@itwin/core-common"; +import { + BriefcaseDb, + EditTxn, + ElementGroupsMembers, + ElementOwnsMultiAspects, + PhysicalModel, + PhysicalObject, + SnapshotDb, + SpatialCategory, + withEditTxn, +} from "@itwin/core-backend"; +import { DatasetDescriptor } from "../DatasetDescriptor"; +import { ScanLedger } from "../validation/scanOracle"; + +const scanAspectClass = "QuickPerfScan:ScanAspect"; + +/** + * Per-unit shape of the update-heavy scan recipe. + * + * Elements are partitioned into four regions so that every id has one unambiguous expected outcome + * after `ChangedInstanceIds` squashes the whole scanned range: + * + * - `updated` updated in every changeset -> `element.updateIds` + * - `deletedLate` updated, then deleted in the last -> `element.deleteIds` + * - `insertedThenUpdated` inserted first, updated afterwards -> `element.insertIds` + * - `insertedThenDeleted` inserted first, deleted in the last -> absent from all three sets + * + * The last region is the reason this recipe can catch squash regressions that a "the sizes match" + * assertion cannot: those ids must cancel out entirely. + */ +export interface ScanRegionSizes { + readonly updated: number; + readonly deletedLate: number; + readonly insertedThenUpdated: number; + readonly insertedThenDeleted: number; + readonly seedRelationships: number; + readonly insertedRelationships: number; + readonly updatedRelationships: number; + readonly deletedRelationships: number; +} + +const perUnit: ScanRegionSizes = { + deletedLate: 20, + deletedRelationships: 10, + insertedRelationships: 10, + insertedThenDeleted: 10, + insertedThenUpdated: 20, + seedRelationships: 40, + updated: 200, + updatedRelationships: 10, +}; + +/** Seed elements are regions A and B; the other two regions are created by the changesets. */ +const seedElementsPerUnit = perUnit.updated + perUnit.deletedLate; + +/** Fewer than four changesets collapses the schedule's distinct first/middle/penultimate/last phases. */ +const minimumChangesets = 4; + +export function scanRegionSizes( + descriptor: DatasetDescriptor +): ScanRegionSizes { + const scale = descriptor.distribution.base.elements / seedElementsPerUnit; + if (!Number.isInteger(scale) || scale < 1) + throw new Error( + `Scan fixture elements must be a positive multiple of ${seedElementsPerUnit}` + ); + if (descriptor.distribution.operations.sourceChangesets < minimumChangesets) + throw new Error( + `Scan fixture needs at least ${minimumChangesets} source changesets` + ); + return Object.fromEntries( + Object.entries(perUnit).map(([key, value]) => [key, value * scale]) + ) as unknown as ScanRegionSizes; +} + +export interface ScanRecipeState { + readonly categoryId: Id64String; + readonly modelId: Id64String; + /** Region A: updated in every changeset. */ + readonly updatedIds: readonly Id64String[]; + /** Region B: updated throughout, then deleted in the final changeset. */ + readonly deletedLateIds: readonly Id64String[]; + /** One owned multi-aspect per region A element, in the same order. */ + readonly updatedAspectIds: readonly Id64String[]; + /** One owned multi-aspect per region B element, in the same order. */ + readonly deletedLateAspectIds: readonly Id64String[]; + readonly relationshipIds: readonly Id64String[]; +} + +function scanElementProps( + modelId: Id64String, + categoryId: Id64String, + region: string, + index: number, + label: string +): PhysicalElementProps { + return { + category: categoryId, + classFullName: PhysicalObject.classFullName, + code: new Code({ + scope: IModel.rootSubjectId, + spec: IModel.rootSubjectId, + value: `scan-${region}-${index}`, + }), + model: modelId, + userLabel: label, + }; +} + +function scanAspectProps( + ownerId: Id64String, + sequence: number, + payload: string, + aspectId?: Id64String +): ElementAspectProps { + return { + classFullName: scanAspectClass, + element: new ElementOwnsMultiAspects(ownerId), + id: aspectId, + payload, + sequence, + } as ElementAspectProps; +} + +export async function createScanSeed( + fileName: string, + descriptor: DatasetDescriptor +): Promise { + const sizes = scanRegionSizes(descriptor); + const db = SnapshotDb.createEmpty(fileName, { + rootSubject: { name: descriptor.id }, + }); + try { + await db.importSchemas([ + path.join(__dirname, "../schemas/QuickPerfScan.ecschema.xml"), + ]); + + const { categoryId, modelId } = withEditTxn( + db, + "create scan model and category", + (txn) => ({ + categoryId: SpatialCategory.insert( + txn, + IModel.dictionaryId, + "ScanCategory", + {} + ), + modelId: PhysicalModel.insert(txn, IModel.rootSubjectId, "ScanModel"), + }) + ); + + const insertRegion = (region: string, count: number) => + withEditTxn(db, `insert scan region ${region}`, (txn) => { + const elementIds: Id64String[] = []; + const aspectIds: Id64String[] = []; + for (let index = 0; index < count; index++) { + const elementId = txn.insertElement( + scanElementProps( + modelId, + categoryId, + region, + index, + `${region}-base-${index}` + ) + ); + elementIds.push(elementId); + aspectIds.push( + txn.insertAspect( + scanAspectProps(elementId, index, `${region}-aspect-${index}`) + ) + ); + } + return { aspectIds, elementIds }; + }); + + const regionA = insertRegion("a", sizes.updated); + const regionB = insertRegion("b", sizes.deletedLate); + + // Relationships live entirely within region A, which is never deleted, so no relationship is + // ever removed by an element cascade. Every relationship delete in the ledger is explicit. + const relationshipIds = withEditTxn( + db, + "insert scan relationships", + (txn) => + Array.from({ length: sizes.seedRelationships }, (_, index) => + txn.insertRelationship( + ElementGroupsMembers.create( + db, + regionA.elementIds[index], + regionA.elementIds[index + 1], + index + ).toJSON() + ) + ) + ); + + return { + categoryId, + deletedLateAspectIds: regionB.aspectIds, + deletedLateIds: regionB.elementIds, + modelId, + relationshipIds, + updatedAspectIds: regionA.aspectIds, + updatedIds: regionA.elementIds, + }; + } finally { + db.close(); + } +} + +/** + * Runs the changeset schedule and returns the ledger of every operation performed. + * + * The schedule is: + * - changeset 1: structural inserts (regions C and D, a model, a CodeSpec, relationships) + * plus the first round of region A/B updates + * - changesets 2..n-2: region A/B/C updates + * - changeset n-1: the same updates plus relationship updates and deletes + * - changeset n: region A/C updates, then region B and D element deletes + */ +export async function applyScanChangesets( + db: BriefcaseDb, + accessToken: AccessToken, + descriptor: DatasetDescriptor, + state: ScanRecipeState +): Promise { + const sizes = scanRegionSizes(descriptor); + const changesetCount = descriptor.distribution.operations.sourceChangesets; + const ledger = new ScanLedger(); + + const insertedThenUpdatedIds: Id64String[] = []; + const insertedThenUpdatedAspectIds: Id64String[] = []; + const insertedThenDeletedIds: Id64String[] = []; + const insertedRelationshipIds: Id64String[] = []; + + const updateElements = ( + txn: EditTxn, + ids: readonly Id64String[], + region: string, + changeset: number + ) => { + ids.forEach((id, index) => + txn.updateElement({ + ...scanElementProps( + state.modelId, + state.categoryId, + region, + index, + `${region}-updated-${changeset}-${index}` + ), + id, + } as PhysicalElementProps) + ); + ledger.record("element", "Updated", ids); + }; + + const updateAspects = ( + txn: EditTxn, + ownerIds: readonly Id64String[], + aspectIds: readonly Id64String[], + region: string, + changeset: number + ) => { + aspectIds.forEach((aspectId, index) => + txn.updateAspect( + scanAspectProps( + ownerIds[index], + index, + `${region}-aspect-${changeset}-${index}`, + aspectId + ) + ) + ); + ledger.record("aspect", "Updated", aspectIds); + }; + + for (let changeset = 1; changeset <= changesetCount; changeset++) { + const isFirst = changeset === 1; + const isLast = changeset === changesetCount; + const isPenultimate = changeset === changesetCount - 1; + + withEditTxn(db, `scan changeset ${changeset}`, (txn) => { + if (isFirst) { + for (let index = 0; index < sizes.insertedThenUpdated; index++) { + const elementId = txn.insertElement( + scanElementProps( + state.modelId, + state.categoryId, + "c", + index, + `c-inserted-${index}` + ) + ); + insertedThenUpdatedIds.push(elementId); + insertedThenUpdatedAspectIds.push( + txn.insertAspect( + scanAspectProps(elementId, index, `c-aspect-${index}`) + ) + ); + } + ledger.record("element", "Inserted", insertedThenUpdatedIds); + ledger.record("aspect", "Inserted", insertedThenUpdatedAspectIds); + + // Region D carries no aspects. Whether cascade-deleting an aspect that was inserted within + // the same scanned range emits a row at all is a second unknown, and this region already + // exists to test one thing: that insert-then-delete cancels. + for (let index = 0; index < sizes.insertedThenDeleted; index++) + insertedThenDeletedIds.push( + txn.insertElement( + scanElementProps( + state.modelId, + state.categoryId, + "d", + index, + `d-inserted-${index}` + ) + ) + ); + ledger.record("element", "Inserted", insertedThenDeletedIds); + + // Inserting elements into a model bumps that model's row, so the model the recipe fills is + // expected to appear as an update even though the recipe never edits the model itself. + ledger.record("model", "Updated", state.modelId); + + // A model insert also inserts its modeled partition element under the same id, so this one + // call is expected to show up in two collections. + const extraModelId = PhysicalModel.insert( + txn, + IModel.rootSubjectId, + "ScanExtraModel" + ); + ledger.record("model", "Inserted", extraModelId); + ledger.record("element", "Inserted", extraModelId); + // ...and that partition element lands in the repository model, bumping it in turn. + ledger.record("model", "Updated", IModel.repositoryModelId); + + ledger.record( + "codeSpec", + "Inserted", + db.codeSpecs.insert( + txn, + CodeSpec.create(db, "ScanCodeSpec", CodeScopeSpec.Type.Repository) + ) + ); + + for (let index = 0; index < sizes.insertedRelationships; index++) + insertedRelationshipIds.push( + txn.insertRelationship( + ElementGroupsMembers.create( + db, + state.updatedIds[index], + state.updatedIds[index + sizes.seedRelationships + 1], + index + ).toJSON() + ) + ); + ledger.record("relationship", "Inserted", insertedRelationshipIds); + } + + updateElements(txn, state.updatedIds, "a", changeset); + updateAspects( + txn, + state.updatedIds, + state.updatedAspectIds, + "a", + changeset + ); + + if (!isLast) { + updateElements(txn, state.deletedLateIds, "b", changeset); + updateAspects( + txn, + state.deletedLateIds, + state.deletedLateAspectIds, + "b", + changeset + ); + } + + if (!isFirst) updateElements(txn, insertedThenUpdatedIds, "c", changeset); + + if (isPenultimate) { + const updated = state.relationshipIds.slice( + 0, + sizes.updatedRelationships + ); + for (const relationshipId of updated) { + const relationship = + db.relationships.getInstance( + ElementGroupsMembers.classFullName, + relationshipId + ); + relationship.memberPriority += 1000; + txn.updateRelationship(relationship.toJSON()); + } + ledger.record("relationship", "Updated", updated); + + const deleted = state.relationshipIds.slice( + sizes.updatedRelationships, + sizes.updatedRelationships + sizes.deletedRelationships + ); + for (const relationshipId of deleted) + txn.deleteRelationship( + db.relationships + .getInstance( + ElementGroupsMembers.classFullName, + relationshipId + ) + .toJSON() + ); + ledger.record("relationship", "Deleted", deleted); + } + + if (isLast) { + txn.deleteElement([...state.deletedLateIds]); + ledger.record("element", "Deleted", state.deletedLateIds); + // Deleting an element cascades to the aspects it owns. + ledger.record("aspect", "Deleted", state.deletedLateAspectIds); + + txn.deleteElement(insertedThenDeletedIds); + ledger.record("element", "Deleted", insertedThenDeletedIds); + } + }); + + await db.pushChanges({ + accessToken, + description: `scan changeset ${changeset} of ${changesetCount}`, + }); + } + + return ledger; +} diff --git a/packages/performance-tests/test/quick/schemas/QuickPerfScan.ecschema.xml b/packages/performance-tests/test/quick/schemas/QuickPerfScan.ecschema.xml new file mode 100644 index 000000000..bd043ea9d --- /dev/null +++ b/packages/performance-tests/test/quick/schemas/QuickPerfScan.ecschema.xml @@ -0,0 +1,11 @@ + + + + + + bis:ElementMultiAspect + + + + diff --git a/packages/performance-tests/test/quick/validation/scanOracle.ts b/packages/performance-tests/test/quick/validation/scanOracle.ts index e57241637..cb1adb458 100644 --- a/packages/performance-tests/test/quick/validation/scanOracle.ts +++ b/packages/performance-tests/test/quick/validation/scanOracle.ts @@ -66,6 +66,20 @@ export class ScanLedger { public get entries(): readonly ScanLedgerEntry[] { return this._entries; } + + /** + * Rebuilds a ledger from serialized entries. + * + * The recipe runs while the fixture artifact is built; the oracle runs later, against a copy. The + * ledger therefore has to survive a round trip through JSON. Replaying through `record` keeps the + * deduplication invariant even if the serialized form was hand-written or concatenated. + */ + public static fromEntries(entries: readonly ScanLedgerEntry[]): ScanLedger { + const ledger = new ScanLedger(); + for (const entry of entries) + ledger.record(entry.collection, entry.op, entry.id); + return ledger; + } } interface MutableScanOps { From 8f74ff7277e42b358ae0999c7795cae5179a0ccb Mon Sep 17 00:00:00 2001 From: Nam Le <50554904+hl662@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:10:01 -0400 Subject: [PATCH 09/19] Guard the detached-open invariant with a dedicated regression test Stage 2 is only a filesystem copy because a briefcase's bytes are portable and BriefcaseDb.open consults no hub. That property was asserted incidentally inside a broader test, so a regression would have surfaced as a confusing benchmark failure rather than a clear one. Assert it directly: relocate the briefcase to a new path, open it readonly with HubMock shut down, and confirm it still advertises the briefcase id and parent changeset of an iModel that no longer exists anywhere. openDgnDb makes no hub calls today, but nothing in core-backend's contract promises it never will. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../test/quick/FixtureArtifact.quick-unit.ts | 48 ++++++++++++++++++- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/packages/performance-tests/test/quick/FixtureArtifact.quick-unit.ts b/packages/performance-tests/test/quick/FixtureArtifact.quick-unit.ts index f715d360e..809a68b69 100644 --- a/packages/performance-tests/test/quick/FixtureArtifact.quick-unit.ts +++ b/packages/performance-tests/test/quick/FixtureArtifact.quick-unit.ts @@ -7,7 +7,7 @@ import { expect } from "chai"; import * as fs from "fs"; import * as os from "os"; import * as path from "path"; -import { IModelHost } from "@itwin/core-backend"; +import { BriefcaseDb, IModelHost } from "@itwin/core-backend"; import { HubMock } from "@itwin/core-backend/lib/cjs/internal/HubMock"; import { ChangedInstanceIds } from "@itwin/imodel-transformer"; import { @@ -18,7 +18,11 @@ import { } from "./FixtureArtifact"; import { balancedIncrementalSourceOnlyDescriptor } from "./FixtureCatalog"; import { requireDetachedDataset } from "./FixtureMaterializer"; -import { BuiltFixture, getFixtureProvider } from "./FixtureProvider"; +import { + BuiltFixture, + getFixtureProvider, + requireFixtureArtifact, +} from "./FixtureProvider"; import { detachedBriefcaseFixtureProvider } from "./providers/detachedBriefcaseProvider"; describe("detached fixture artifact", () => { @@ -48,6 +52,46 @@ describe("detached fixture artifact", () => { ); }); + /** + * The load-bearing invariant of the whole two-stage design: a briefcase's bytes are portable and + * `BriefcaseDb.open` consults no hub. Stage 2 is only a filesystem copy because this holds. + * + * `openDgnDb` makes no hub calls today, but nothing in core-backend's API contract promises it + * never will. If a future version validates a briefcase id or parent changeset against a hub, + * this fails here with a clear cause instead of surfacing as an inscrutable benchmark failure. + */ + it("opens a relocated briefcase readonly with no hub of any kind", async () => { + expect(HubMock.isValid).to.equal( + false, + "stage 1 must release its build hub before any working copy is opened" + ); + const manifest = requireFixtureArtifact(built).manifest; + const relocated = path.join(root, "relocated", "renamed.bim"); + fs.mkdirSync(path.dirname(relocated), { recursive: true }); + fs.copyFileSync(artifactBriefcasePath(built.directory), relocated); + + const sourceDb = await BriefcaseDb.open({ + fileName: relocated, + readonly: true, + }); + try { + // The copy still advertises a briefcase id and a parent changeset belonging to an iModel + // that no longer exists anywhere. Retaining them is harmless precisely because open is + // hub-free; if either were ever resolved remotely, the open above would have thrown. + expect(sourceDb.briefcaseId).to.equal(manifest.briefcase.briefcaseId); + expect(sourceDb.changeset.id).to.equal(manifest.briefcase.changeset.id); + // realpath because macOS resolves /var to /private/var on open. + expect(fs.realpathSync(sourceDb.pathName)).to.equal( + fs.realpathSync(relocated) + ); + // Readable, not merely openable. + expect(sourceDb.elements.getRootSubject()).to.not.be.undefined; + } finally { + sourceDb.close(); + } + fs.rmSync(path.dirname(relocated), { recursive: true, force: true }); + }); + it("emits a self-describing artifact and leaves no build scaffolding", () => { const entries = fs.readdirSync(built.directory).sort(); expect(entries).to.deep.equal([ From 508c8066d317614517dc100fe53c0eb7b44d405b Mon Sep 17 00:00:00 2001 From: Nam Le <50554904+hl662@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:21:00 -0400 Subject: [PATCH 10/19] Let recipes carry data across the fixture stage boundary A scanning oracle needs the exact id sets its recipe operated on, and there was no path for recipe-produced data to reach a scenario. The ids cannot be recovered later: the artifact briefcase is pinned at tip, so every element the recipe deleted is already gone from it, and deriving them from the changeset files would be circular for a scenario whose job is to verify those same files. applySourceChangesets may now return a value. Stage 1 serializes it to recipe.json in the artifact and records the file in the manifest; stage 2 surfaces it as PreparedDetachedDataset.recipe, typed unknown so the framework never interprets it. Recipes that return nothing are untouched and the key stays absent, so the artifact version stays at 1. Validation walks the returned value and rejects anything JSON cannot carry. A round-trip comparison would not do: JSON.stringify(new Set()) is "{}", and both sides then have zero enumerable keys, so a lossy Set would compare equal and an oracle would later read an empty id list as a pass. Because the artifact is built once and copied, recipe.json is byte-identical across both arms of an A/B run, so an expectation cannot drift between arms and turn a correctness disagreement into an unexplained verdict. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../test/quick/FixtureArtifact.quick-unit.ts | 168 ++++++++++++++++++ .../test/quick/FixtureArtifact.ts | 103 +++++++++++ .../test/quick/FixtureMaterializer.ts | 6 + .../test/quick/FixtureRecipe.ts | 24 ++- .../providers/detachedBriefcaseProvider.ts | 11 +- 5 files changed, 305 insertions(+), 7 deletions(-) diff --git a/packages/performance-tests/test/quick/FixtureArtifact.quick-unit.ts b/packages/performance-tests/test/quick/FixtureArtifact.quick-unit.ts index 809a68b69..2642cdf9d 100644 --- a/packages/performance-tests/test/quick/FixtureArtifact.quick-unit.ts +++ b/packages/performance-tests/test/quick/FixtureArtifact.quick-unit.ts @@ -10,13 +10,20 @@ import * as path from "path"; import { BriefcaseDb, IModelHost } from "@itwin/core-backend"; import { HubMock } from "@itwin/core-backend/lib/cjs/internal/HubMock"; import { ChangedInstanceIds } from "@itwin/imodel-transformer"; +import { DatasetDescriptor } from "./DatasetDescriptor"; import { artifactBriefcasePath, artifactManifestFileName, readChangesetFileProps, readFixtureArtifact, + readFixtureRecipeData, + writeFixtureRecipeData, } from "./FixtureArtifact"; import { balancedIncrementalSourceOnlyDescriptor } from "./FixtureCatalog"; +import { + balancedIncrementalRecipe, + registerFixtureRecipe, +} from "./FixtureRecipe"; import { requireDetachedDataset } from "./FixtureMaterializer"; import { BuiltFixture, @@ -112,6 +119,19 @@ describe("detached fixture artifact", () => { ).to.have.length(artifact.manifest.changesets.count); }); + it("omits recipe data when the recipe returns nothing", () => { + // `balanced-incremental` opts out, so the key must be absent rather than null. + const raw = JSON.parse( + fs.readFileSync( + path.join(built.directory, artifactManifestFileName), + "utf8" + ) + ) as Record; + expect(raw).to.not.have.property("recipeDataFile"); + expect(fs.existsSync(path.join(built.directory, "recipe.json"))).to.be + .false; + }); + it("stores relocatable changeset pathnames", () => { const raw = JSON.parse( fs.readFileSync(path.join(built.directory, "csFileProps.json"), "utf8") @@ -208,4 +228,152 @@ describe("detached fixture artifact", () => { expect(() => readChangesetFileProps(corrupt)).to.throw(); fs.rmSync(corrupt, { recursive: true, force: true }); }); + + it("carries recipe data across the stage boundary unchanged", () => { + const dir = path.join(root, "recipe-data"); + fs.mkdirSync(dir, { recursive: true }); + // The shape a scanning oracle needs: deleted ids cannot be recovered from a tip-pinned + // briefcase, so they only exist if the recipe hands them over here. + const data = { + elements: { insertIds: ["0x20", "0x21"], deleteIds: ["0x22"] }, + counts: { changesets: 8 }, + nested: [{ ok: true }, { ok: false }], + }; + expect(writeFixtureRecipeData(dir, data)).to.equal("recipe.json"); + const manifest = { + ...readFixtureArtifact(built.directory).manifest, + recipeDataFile: "recipe.json", + }; + expect(readFixtureRecipeData(dir, manifest)).to.deep.equal(data); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + /** + * Values that `JSON.stringify` mangles rather than rejecting are the dangerous ones: a `Set` + * serializes to `{}`, so an oracle would read an empty id list and report a false pass. These + * must fail at build time, where the message points at the recipe. + */ + it("rejects recipe data that JSON cannot represent", () => { + const dir = path.join(root, "bad-recipe-data"); + fs.mkdirSync(dir, { recursive: true }); + const circular: Record = {}; + circular.self = circular; + const cases: [string, unknown][] = [ + ["Set", { ids: new Set(["0x20"]) }], + ["Map", { ids: new Map([["a", 1]]) }], + ["Date", { builtAt: new Date() }], + ["BigInt", { count: BigInt(1) }], + ["NaN", { ratio: NaN }], + ["Infinity", { ratio: Infinity }], + ["undefined", { ids: undefined }], + ["function", { make: () => 1 }], + ["class instance", { at: new (class Point {})() }], + ["circular", circular], + ]; + for (const [label, value] of cases) + expect(() => writeFixtureRecipeData(dir, value), label).to.throw( + /Recipe data at/ + ); + expect( + fs.existsSync(path.join(dir, "recipe.json")), + "a rejected value must not leave a partial artifact" + ).to.be.false; + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it("fails loudly when declared recipe data is missing", () => { + const artifact = readFixtureArtifact(built.directory); + expect(() => + readFixtureRecipeData(built.directory, { + ...artifact.manifest, + recipeDataFile: "recipe.json", + }) + ).to.throw(/missing/); + }); +}); + +/** + * The end-to-end contract W3's oracle depends on: whatever a recipe returns must survive stage 1, + * the filesystem copy, and stage 2 — identically for every sample, and therefore identically for + * both arms of an A/B comparison. + */ +describe("recipe data across the stage boundary", () => { + const emitted = { + elements: { insertIds: ["0x20", "0x21"], deleteIds: ["0x22"] }, + changesets: 8, + }; + const recipeId = "balanced-incremental-with-data"; + const descriptor: DatasetDescriptor = { + ...balancedIncrementalSourceOnlyDescriptor, + id: "balanced-incremental-emitting-data", + layout: { + ...balancedIncrementalSourceOnlyDescriptor.layout, + recipe: recipeId, + }, + }; + let root: string; + let built: BuiltFixture; + + before(async function () { + this.timeout(600_000); + registerFixtureRecipe({ + id: recipeId, + createSeed: async (fileName, forDescriptor) => + balancedIncrementalRecipe.createSeed(fileName, forDescriptor), + applySourceChangesets: async (db, token, forDescriptor, state) => { + await balancedIncrementalRecipe.applySourceChangesets( + db, + token, + forDescriptor, + state + ); + return emitted; + }, + }); + root = fs.mkdtempSync(path.join(os.tmpdir(), "quick-recipe-data-")); + await IModelHost.startup(); + built = await detachedBriefcaseFixtureProvider.build( + descriptor, + path.join(root, "fixture-artifact") + ); + }); + + after(async () => { + await detachedBriefcaseFixtureProvider.disposeBuild(built); + if (IModelHost.isValid) await IModelHost.shutdown(); + fs.rmSync(root, { recursive: true, force: true }); + }); + + it("records the data file in the manifest rather than requiring a probe", () => { + const artifact = readFixtureArtifact(built.directory); + expect(artifact.manifest.recipeDataFile).to.equal("recipe.json"); + expect(fs.existsSync(path.join(built.directory, "recipe.json"))).to.be.true; + }); + + it("surfaces identical data on every working copy", async function () { + this.timeout(300_000); + const datasets = []; + for (const name of ["sample-0", "sample-1"]) + datasets.push( + requireDetachedDataset( + await detachedBriefcaseFixtureProvider.materialize( + built, + path.join(root, "samples", name), + name + ) + ) + ); + try { + for (const dataset of datasets) + expect(dataset.recipe).to.deep.equal(emitted); + // Byte-identical, not merely deep-equal: this is what makes an A/B verdict trustworthy. + const [first, second] = datasets.map((dataset) => + fs.readFileSync(path.join(dataset.directory, "recipe.json")) + ); + expect(first.equals(second)).to.be.true; + } finally { + for (const dataset of datasets) + await detachedBriefcaseFixtureProvider.disposeSample(dataset); + } + }); }); diff --git a/packages/performance-tests/test/quick/FixtureArtifact.ts b/packages/performance-tests/test/quick/FixtureArtifact.ts index a9d50670c..7c13d19b4 100644 --- a/packages/performance-tests/test/quick/FixtureArtifact.ts +++ b/packages/performance-tests/test/quick/FixtureArtifact.ts @@ -19,6 +19,7 @@ export const artifactBriefcaseFileName = "briefcase.bim"; export const artifactChangesetDirectoryName = "changesets"; export const artifactChangesetPropsFileName = "csFileProps.json"; export const artifactManifestFileName = "manifest.json"; +export const artifactRecipeDataFileName = "recipe.json"; export interface FixtureArtifactManifest { readonly artifactVersion: number; @@ -38,6 +39,11 @@ export interface FixtureArtifactManifest { readonly firstIndex?: number; readonly lastIndex?: number; }; + /** + * Present only when the recipe returned data to carry across the stage boundary. Absent means + * the recipe emitted nothing — stage 2 reads this key rather than probing the filesystem. + */ + readonly recipeDataFile?: string; readonly buildMilliseconds: number; readonly builtAt: string; } @@ -48,6 +54,89 @@ export interface FixtureArtifact { readonly manifest: FixtureArtifactManifest; } +function describeValue(value: unknown): string { + if (value === null) return "null"; + if (typeof value !== "object") return typeof value; + const name = value.constructor?.name; + return name && name !== "Object" ? name : "object"; +} + +/** + * Reject anything that would not survive `JSON.stringify`/`parse` intact. + * + * A plain round-trip comparison is not enough: `JSON.stringify(new Set())` yields `{}`, and both + * sides then have zero enumerable keys, so a lossy `Set` would compare equal. Walking the original + * and requiring JSON-native values at every node catches that, plus `Map`, `Date`, `BigInt`, `NaN`, + * `undefined`, functions and class instances — and names the path that is wrong. + */ +function assertJsonNative(value: unknown, at: string, seen: Set): void { + if (value === null || typeof value === "string" || typeof value === "boolean") + return; + if (typeof value === "number") { + if (!Number.isFinite(value)) + throw new Error( + `Recipe data at ${at} is ${String( + value + )}, which JSON serializes as null` + ); + return; + } + if (typeof value !== "object") + throw new Error( + `Recipe data at ${at} is a ${describeValue( + value + )}, which JSON cannot represent` + ); + if (seen.has(value)) throw new Error(`Recipe data at ${at} is circular`); + seen.add(value); + if (Array.isArray(value)) { + value.forEach((entry, index) => + assertJsonNative(entry, `${at}[${index}]`, seen) + ); + } else { + const prototype = Object.getPrototypeOf(value) as object | null; + if (prototype !== Object.prototype && prototype !== null) + throw new Error( + `Recipe data at ${at} is a ${describeValue( + value + )}; only plain objects and arrays survive JSON` + ); + for (const [key, entry] of Object.entries(value)) + assertJsonNative(entry, `${at}.${key}`, seen); + } + seen.delete(value); +} + +/** + * Serialize a recipe's returned data into the artifact, failing at build time rather than + * producing a silently lossy artifact a scenario would misread much later. + */ +export function writeFixtureRecipeData( + directory: string, + data: unknown +): string { + assertJsonNative(data, "", new Set()); + fs.writeFileSync( + path.join(directory, artifactRecipeDataFileName), + `${JSON.stringify(data, undefined, 2)}\n` + ); + return artifactRecipeDataFileName; +} + +/** Read recipe data from an artifact or working copy; `undefined` when the recipe emitted none. */ +export function readFixtureRecipeData( + directory: string, + manifest: FixtureArtifactManifest +): unknown { + if (manifest.recipeDataFile === undefined) return undefined; + const file = path.join(directory, manifest.recipeDataFile); + if (!fs.existsSync(file)) + throw new Error( + `Fixture artifact manifest declares recipe data at ${manifest.recipeDataFile} but the file is missing` + ); + return JSON.parse(fs.readFileSync(file, "utf8")); +} + export function changesetArtifactFileName( changeset: Pick ): string { @@ -139,6 +228,13 @@ export function validateFixtureArtifactManifest( typeof manifest.builtAt !== "string" ) throw new Error("Fixture artifact manifest has an invalid shape"); + if ( + manifest.recipeDataFile !== undefined && + typeof manifest.recipeDataFile !== "string" + ) + throw new Error( + "Fixture artifact manifest has an invalid recipeDataFile entry" + ); validateDescriptor(manifest.descriptor); return manifest as FixtureArtifactManifest; } @@ -166,5 +262,12 @@ export function readFixtureArtifact(directory: string): FixtureArtifact { throw new Error( `Fixture artifact has ${changesets.length} changesets but its manifest declares ${manifest.changesets.count}` ); + if ( + manifest.recipeDataFile !== undefined && + !fs.existsSync(path.join(directory, manifest.recipeDataFile)) + ) + throw new Error( + `Fixture artifact is missing the recipe data its manifest declares: ${manifest.recipeDataFile}` + ); return { directory, manifest }; } diff --git a/packages/performance-tests/test/quick/FixtureMaterializer.ts b/packages/performance-tests/test/quick/FixtureMaterializer.ts index 9a898a7d8..d5630bcb3 100644 --- a/packages/performance-tests/test/quick/FixtureMaterializer.ts +++ b/packages/performance-tests/test/quick/FixtureMaterializer.ts @@ -31,6 +31,12 @@ export interface PreparedDetachedDataset extends PreparedDatasetBase { /** Changeset props whose `pathname` values are absolute and point into `directory`. */ readonly csFileProps: ChangesetFileProps[]; readonly manifest: FixtureArtifactManifest; + /** + * Whatever the recipe returned from `applySourceChangesets`, or `undefined` if it returned + * nothing. Deliberately `unknown`: the framework never interprets it, and it is byte-identical + * across samples and across A/B arms because it is captured once at stage 1. + */ + readonly recipe?: unknown; } export type PreparedDataset = PreparedLiveHubDataset | PreparedDetachedDataset; diff --git a/packages/performance-tests/test/quick/FixtureRecipe.ts b/packages/performance-tests/test/quick/FixtureRecipe.ts index 173996307..6154e04c5 100644 --- a/packages/performance-tests/test/quick/FixtureRecipe.ts +++ b/packages/performance-tests/test/quick/FixtureRecipe.ts @@ -16,18 +16,30 @@ import { * A recipe produces the *change mix* for a fixture: it seeds the source iModel and then applies a * deterministic series of pushed changesets. It never touches HubMock; the fixture provider owns * the hub lifecycle. + * + * `TArtifactData` is anything the recipe must tell the scenario that cannot be recovered from the + * artifact afterwards — most importantly the exact ids it operated on. Deleted ids are gone from + * the tip-pinned briefcase, and deriving them from the changeset files would be circular for a + * scenario whose job is to verify those same files. Stage 1 captures the returned value once, so + * every sample and every A/B arm reads byte-identical expectations. */ -export interface FixtureRecipe { +export interface FixtureRecipe { readonly id: string; /** Create the source seed file. Returns state carried into {@link applySourceChangesets}. */ createSeed(fileName: string, descriptor: DatasetDescriptor): Promise; - /** Apply and push the recipe's changesets to an open source briefcase. */ + /** + * Apply and push the recipe's changesets to an open source briefcase. + * + * Any returned value is serialized into the artifact as `recipe.json` and surfaced to the + * scenario as `PreparedDetachedDataset.recipe`. It must round-trip through JSON; returning + * nothing is the normal case. + */ applySourceChangesets( db: BriefcaseDb, accessToken: AccessToken, descriptor: DatasetDescriptor, state: TState - ): Promise; + ): Promise; } export const balancedIncrementalRecipe: FixtureRecipe = { @@ -38,17 +50,17 @@ export const balancedIncrementalRecipe: FixtureRecipe = { applyBalancedChangesets(db, accessToken, descriptor, state), }; -const recipes = new Map>([ +const recipes = new Map>([ [balancedIncrementalRecipe.id, balancedIncrementalRecipe], ]); -export function registerFixtureRecipe(recipe: FixtureRecipe): void { +export function registerFixtureRecipe(recipe: FixtureRecipe): void { if (recipes.has(recipe.id)) throw new Error(`Duplicate quick performance recipe: ${recipe.id}`); recipes.set(recipe.id, recipe); } -export function getFixtureRecipe(id: string): FixtureRecipe { +export function getFixtureRecipe(id: string): FixtureRecipe { const recipe = recipes.get(id); if (!recipe) throw new Error( diff --git a/packages/performance-tests/test/quick/providers/detachedBriefcaseProvider.ts b/packages/performance-tests/test/quick/providers/detachedBriefcaseProvider.ts index 4bfe4c1d9..327a4ac98 100644 --- a/packages/performance-tests/test/quick/providers/detachedBriefcaseProvider.ts +++ b/packages/performance-tests/test/quick/providers/detachedBriefcaseProvider.ts @@ -17,8 +17,10 @@ import { fixtureArtifactVersion, readChangesetFileProps, readFixtureArtifact, + readFixtureRecipeData, toRelativeChangesetProps, writeFixtureArtifactManifest, + writeFixtureRecipeData, } from "../FixtureArtifact"; import { BuiltFixture, @@ -112,7 +114,7 @@ export const detachedBriefcaseFixtureProvider: FixtureProvider = { // The seed is uploaded as version 0, so the briefcase starts before any changeset. const baseChangesetIndex = hub.sourceDb.changeset.index ?? 0; - await recipe.applySourceChangesets( + const recipeData = await recipe.applySourceChangesets( hub.sourceDb, hub.accessToken, descriptor, @@ -159,6 +161,11 @@ export const detachedBriefcaseFixtureProvider: FixtureProvider = { ); hub = undefined; + const recipeDataFile = + recipeData === undefined || recipeData === null + ? undefined + : writeFixtureRecipeData(artifactDir, recipeData); + const buildMilliseconds = Number(process.hrtime.bigint() - start) / 1_000_000; const indices = downloaded.map((changeset) => changeset.index); @@ -182,6 +189,7 @@ export const detachedBriefcaseFixtureProvider: FixtureProvider = { firstIndex: indices.length > 0 ? Math.min(...indices) : undefined, lastIndex: indices.length > 0 ? Math.max(...indices) : undefined, }, + recipeDataFile, buildMilliseconds, builtAt: new Date().toISOString(), }; @@ -235,6 +243,7 @@ export const detachedBriefcaseFixtureProvider: FixtureProvider = { sourceDb, csFileProps, manifest: artifact.manifest, + recipe: readFixtureRecipeData(sampleDir, artifact.manifest), reconstructionMilliseconds: Number(process.hrtime.bigint() - start) / 1_000_000, }; From fa0006f3e4a6652a716be83ec4a047fb8c7ef54a Mon Sep 17 00:00:00 2001 From: Nam Le <50554904+hl662@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:27:10 -0400 Subject: [PATCH 11/19] perf(tests): add changeset-scanning scenario Adds a second quick-performance scenario measuring ChangedInstanceIds.initialize({iModel, csFileProps}) over local changeset files, with no target iModel and no hub on the measured path. The measured region is deliberately narrow. IModelTransformer.initialize also does provenance init, clone-context and exporter setup and deleted entity remapping; scanning is roughly a third of it, so at ~3% run-to-run variation a 20% scan regression would move that larger total by ~6% and vanish into the noise. Measured alone it sits at 2.4% CV over 8 samples. Two hooks were added to FixtureRecipe because a second recipe made two implicit assumptions visible: - applySourceChangesets may now return JSON-serializable recipe data, captured as recipe.json in the artifact and handed back to the scenario. A recipe runs while the artifact is built and a scenario runs later against a copy, so the scan oracle ledger had no way across that boundary. Deleted ids in particular cannot be recovered by querying the tip briefcase, and three of the six collections are delete sets. - validate() is now recipe-owned. assertFixtureDistribution queries QuickPerf.Balanced* classes and updated-% labels, so it was silently specific to one recipe while being called for all of them. Verification is a recipe oracle rather than a digest: the recipe records every operation it performs and the scenario asserts exact set equality across all six ChangedInstanceOps collections. A digest can only detect that behaviour changed, not that it was wrong from the start. The digest is still returned, as a cross-arm behaviour gate. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../test/quick/FixtureArtifact.ts | 21 ++++ .../test/quick/FixtureCatalog.ts | 95 ++++++++++++++++++ .../test/quick/FixtureMaterializer.ts | 2 + .../test/quick/FixtureRecipe.ts | 39 +++++++- .../test/quick/ScenarioCatalog.ts | 2 + .../providers/detachedBriefcaseProvider.ts | 9 +- .../test/quick/providers/liveHubProvider.ts | 3 +- .../test/quick/recipes/updateHeavyScan.ts | 96 ++++++++++++++++++- .../test/quick/scenarios/changesetScanning.ts | 85 ++++++++++++++++ .../test/quick/validation/scanOracle.ts | 16 +++- .../test/quick/validation/validateFixture.ts | 5 +- 11 files changed, 359 insertions(+), 14 deletions(-) create mode 100644 packages/performance-tests/test/quick/scenarios/changesetScanning.ts diff --git a/packages/performance-tests/test/quick/FixtureArtifact.ts b/packages/performance-tests/test/quick/FixtureArtifact.ts index a9d50670c..d691c50d2 100644 --- a/packages/performance-tests/test/quick/FixtureArtifact.ts +++ b/packages/performance-tests/test/quick/FixtureArtifact.ts @@ -19,6 +19,7 @@ export const artifactBriefcaseFileName = "briefcase.bim"; export const artifactChangesetDirectoryName = "changesets"; export const artifactChangesetPropsFileName = "csFileProps.json"; export const artifactManifestFileName = "manifest.json"; +export const artifactRecipeDataFileName = "recipe.json"; export interface FixtureArtifactManifest { readonly artifactVersion: number; @@ -100,6 +101,26 @@ export function readChangesetFileProps( }); } +/** + * Recipe-specific data captured at build time, if the recipe produced any. + * + * Absent for recipes that return nothing, which is why this reads as "undefined" rather than + * throwing: an artifact without recipe data is a valid artifact. + */ +export function readRecipeData(directory: string): unknown { + const file = path.join(directory, artifactRecipeDataFileName); + if (!fs.existsSync(file)) return undefined; + return JSON.parse(fs.readFileSync(file, "utf-8")); +} + +export function writeRecipeData(directory: string, data: unknown): void { + if (data === undefined) return; + fs.writeFileSync( + path.join(directory, artifactRecipeDataFileName), + `${JSON.stringify(data)}\n` + ); +} + export function artifactBriefcasePath(directory: string): string { return path.join(directory, artifactBriefcaseFileName); } diff --git a/packages/performance-tests/test/quick/FixtureCatalog.ts b/packages/performance-tests/test/quick/FixtureCatalog.ts index 489762d6b..649253378 100644 --- a/packages/performance-tests/test/quick/FixtureCatalog.ts +++ b/packages/performance-tests/test/quick/FixtureCatalog.ts @@ -107,12 +107,107 @@ export const balancedIncrementalSourceOnlyDescriptor: DatasetDescriptor = { recipeHash: canonicalSha256(recipeIdentity("source-only")), }; +/** + * Update-heavy multi-changeset source, sized so that scanning it dominates timer noise. + * + * Region sizes are derived from `base.elements` by `scanRegionSizes`; see `recipes/updateHeavyScan` + * for what each region proves. + * + * Calibrated at 2,420 elements x 20 changesets: the scan measures ~2.15 s with a coefficient of + * variation of 2.4% over 8 samples, against ~44 ms of per-sample copy, verification and teardown. + * Scan cost is linear in changed rows at roughly 48 ms per (1,000 elements x changeset), so the + * shape is a single `scanScale` knob. + * + * The size is bounded by comparison mode rather than by a single run. One run is 9 executions and + * finishes in well under a minute, but an A/B comparison at its escalated width is 128 executions, + * where the build cost amortizes away and the per-execution cost is all that matters. This shape + * keeps that case inside a 15 minute budget with roughly 3x headroom for slower CI hardware; a + * larger shape would measure the same thing no better and fit the escalated case worse. + */ +const scanScale = 11; +const scanDistribution = { + base: { + // Region A (updated throughout) plus region B (updated, then deleted last). + aspects: 220 * scanScale, + elements: 220 * scanScale, + geometricElements: 0, + relationships: 40 * scanScale, + }, + operations: { + elements: { + // Regions C and D, both inserted in the first changeset. + inserts: 30 * scanScale, + // Every seeded element is updated, plus region C after its insert. + updates: 240 * scanScale, + // Region B at the end, region D at the end. + deletes: 30 * scanScale, + }, + aspects: { + inserts: 20 * scanScale, + updates: 220 * scanScale, + // Region B's owned aspects, cascade-deleted with their elements. + deletes: 20 * scanScale, + }, + relationships: { + inserts: 10 * scanScale, + updates: 10 * scanScale, + deletes: 10 * scanScale, + }, + geometryUpdates: 0, + sourceChangesets: 20, + }, +} as const; + +const scanRecipeIdentity = { + schema: "QuickPerfScan.01.00.00", + seed: 328, + topology: "source-only", + distribution: scanDistribution, + inputs: { + recipe: fs.readFileSync( + path.join(__dirname, "recipes/updateHeavyScan.ts"), + "utf8" + ), + schema: fs.readFileSync( + path.join(__dirname, "schemas/QuickPerfScan.ecschema.xml"), + "utf8" + ), + lockfile: fs.readFileSync( + path.join(__dirname, "../../../../pnpm-lock.yaml"), + "utf8" + ), + }, + versions: generator, +}; + +export const updateHeavyScanDescriptor: DatasetDescriptor = { + id: "update-heavy-scan", + version: 1, + label: "update-heavy scan", + scenarioClaims: [ + "changeset scanning", + "changed-instance squashing", + "aspect lifecycle", + "relationship lifecycle", + ], + layout: { + kind: "reconstructed", + topology: "source-only", + recipe: "update-heavy-scan", + seed: 328, + }, + distribution: scanDistribution, + generator, + recipeHash: canonicalSha256(scanRecipeIdentity), +}; + const fixtures = new Map([ [balancedIncrementalDescriptor.id, balancedIncrementalDescriptor], [ balancedIncrementalSourceOnlyDescriptor.id, balancedIncrementalSourceOnlyDescriptor, ], + [updateHeavyScanDescriptor.id, updateHeavyScanDescriptor], ]); export function registerFixtureDescriptor(descriptor: DatasetDescriptor): void { diff --git a/packages/performance-tests/test/quick/FixtureMaterializer.ts b/packages/performance-tests/test/quick/FixtureMaterializer.ts index 9a898a7d8..f6be5811f 100644 --- a/packages/performance-tests/test/quick/FixtureMaterializer.ts +++ b/packages/performance-tests/test/quick/FixtureMaterializer.ts @@ -31,6 +31,8 @@ export interface PreparedDetachedDataset extends PreparedDatasetBase { /** Changeset props whose `pathname` values are absolute and point into `directory`. */ readonly csFileProps: ChangesetFileProps[]; readonly manifest: FixtureArtifactManifest; + /** Whatever the recipe captured at build time. Scenario-interpreted; the framework ignores it. */ + readonly recipeData?: unknown; } export type PreparedDataset = PreparedLiveHubDataset | PreparedDetachedDataset; diff --git a/packages/performance-tests/test/quick/FixtureRecipe.ts b/packages/performance-tests/test/quick/FixtureRecipe.ts index 173996307..f68f1643e 100644 --- a/packages/performance-tests/test/quick/FixtureRecipe.ts +++ b/packages/performance-tests/test/quick/FixtureRecipe.ts @@ -11,6 +11,13 @@ import { BalancedRecipeState, createBalancedSeed, } from "./recipes/balancedIncremental"; +import { + applyScanChangesets, + createScanSeed, + ScanRecipeState, + validateScanFixture, +} from "./recipes/updateHeavyScan"; +import { assertFixtureDistribution } from "./validation/validateFixture"; /** * A recipe produces the *change mix* for a fixture: it seeds the source iModel and then applies a @@ -21,13 +28,30 @@ export interface FixtureRecipe { readonly id: string; /** Create the source seed file. Returns state carried into {@link applySourceChangesets}. */ createSeed(fileName: string, descriptor: DatasetDescriptor): Promise; - /** Apply and push the recipe's changesets to an open source briefcase. */ + /** + * Apply and push the recipe's changesets to an open source briefcase. + * + * Any returned value is recipe-specific data captured into the fixture artifact and handed back + * to the scenario at measure time. This exists because a recipe runs while the artifact is built + * but a scenario runs later against a copy, so anything the recipe knows and the scenario needs + * has to survive that boundary. A recipe that needs nothing returns `undefined`. + * + * The value must be JSON-serializable. The framework never inspects it. + */ applySourceChangesets( db: BriefcaseDb, accessToken: AccessToken, descriptor: DatasetDescriptor, state: TState - ): Promise; + ): Promise; + /** + * Assert the built source iModel matches what the descriptor promises. + * + * Validation is recipe-owned because it is inherently recipe-specific: it queries the classes the + * recipe created. A single shared check would either be limited to what every recipe has in + * common, or silently wrong for every recipe but one. + */ + validate(db: BriefcaseDb, descriptor: DatasetDescriptor): Promise; } export const balancedIncrementalRecipe: FixtureRecipe = { @@ -36,10 +60,21 @@ export const balancedIncrementalRecipe: FixtureRecipe = { createBalancedSeed(fileName, descriptor), applySourceChangesets: async (db, accessToken, descriptor, state) => applyBalancedChangesets(db, accessToken, descriptor, state), + validate: async (db, descriptor) => assertFixtureDistribution(db, descriptor), +}; + +export const updateHeavyScanRecipe: FixtureRecipe = { + id: "update-heavy-scan", + createSeed: async (fileName, descriptor) => + createScanSeed(fileName, descriptor), + applySourceChangesets: async (db, accessToken, descriptor, state) => + applyScanChangesets(db, accessToken, descriptor, state), + validate: async (db, descriptor) => validateScanFixture(db, descriptor), }; const recipes = new Map>([ [balancedIncrementalRecipe.id, balancedIncrementalRecipe], + [updateHeavyScanRecipe.id, updateHeavyScanRecipe], ]); export function registerFixtureRecipe(recipe: FixtureRecipe): void { diff --git a/packages/performance-tests/test/quick/ScenarioCatalog.ts b/packages/performance-tests/test/quick/ScenarioCatalog.ts index a67b944a5..eb20a4c81 100644 --- a/packages/performance-tests/test/quick/ScenarioCatalog.ts +++ b/packages/performance-tests/test/quick/ScenarioCatalog.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { BenchmarkScenarioDefinition } from "./BenchmarkScenario"; +import { changesetScanningScenario } from "./scenarios/changesetScanning"; import { incrementalSynchronizationScenario } from "./scenarios/incrementalSynchronization"; export const defaultQuickPerformanceScenarioId = @@ -11,6 +12,7 @@ export const defaultQuickPerformanceScenarioId = const scenarios = new Map([ [incrementalSynchronizationScenario.id, incrementalSynchronizationScenario], + [changesetScanningScenario.id, changesetScanningScenario], ]); export function registerScenarioDefinition( diff --git a/packages/performance-tests/test/quick/providers/detachedBriefcaseProvider.ts b/packages/performance-tests/test/quick/providers/detachedBriefcaseProvider.ts index 4bfe4c1d9..71c1aff94 100644 --- a/packages/performance-tests/test/quick/providers/detachedBriefcaseProvider.ts +++ b/packages/performance-tests/test/quick/providers/detachedBriefcaseProvider.ts @@ -17,8 +17,10 @@ import { fixtureArtifactVersion, readChangesetFileProps, readFixtureArtifact, + readRecipeData, toRelativeChangesetProps, writeFixtureArtifactManifest, + writeRecipeData, } from "../FixtureArtifact"; import { BuiltFixture, @@ -35,7 +37,6 @@ import { reconstructSourceHub, shutdownHubMock, } from "../LocalHubFixture"; -import { assertFixtureDistribution } from "../validation/validateFixture"; /** * Tolerant teardown for a build that may have failed at any point. Collects errors rather than @@ -112,13 +113,13 @@ export const detachedBriefcaseFixtureProvider: FixtureProvider = { // The seed is uploaded as version 0, so the briefcase starts before any changeset. const baseChangesetIndex = hub.sourceDb.changeset.index ?? 0; - await recipe.applySourceChangesets( + const recipeData = await recipe.applySourceChangesets( hub.sourceDb, hub.accessToken, descriptor, recipeState ); - await assertFixtureDistribution(hub.sourceDb, descriptor); + await recipe.validate(hub.sourceDb, descriptor); fs.mkdirSync(scratchDir, { recursive: true }); const downloaded = await BriefcaseManager.downloadChangesets({ @@ -146,6 +147,7 @@ export const detachedBriefcaseFixtureProvider: FixtureProvider = { path.join(artifactDir, artifactChangesetPropsFileName), `${JSON.stringify(toRelativeChangesetProps(downloaded), undefined, 2)}\n` ); + writeRecipeData(artifactDir, recipeData); const releaseErrors = await releaseBuildHub( hub, @@ -235,6 +237,7 @@ export const detachedBriefcaseFixtureProvider: FixtureProvider = { sourceDb, csFileProps, manifest: artifact.manifest, + recipeData: readRecipeData(sampleDir), reconstructionMilliseconds: Number(process.hrtime.bigint() - start) / 1_000_000, }; diff --git a/packages/performance-tests/test/quick/providers/liveHubProvider.ts b/packages/performance-tests/test/quick/providers/liveHubProvider.ts index 25815be2c..f036a808f 100644 --- a/packages/performance-tests/test/quick/providers/liveHubProvider.ts +++ b/packages/performance-tests/test/quick/providers/liveHubProvider.ts @@ -14,7 +14,6 @@ import { ReconstructedHub, reconstructHub, } from "../LocalHubFixture"; -import { assertFixtureDistribution } from "../validation/validateFixture"; /** * The `source-and-empty-target` topology. @@ -69,7 +68,7 @@ export const liveHubFixtureProvider: FixtureProvider = { descriptor, recipeState ); - await assertFixtureDistribution(hub.sourceDb, descriptor); + await recipe.validate(hub.sourceDb, descriptor); return { topology: "source-and-empty-target", descriptor, diff --git a/packages/performance-tests/test/quick/recipes/updateHeavyScan.ts b/packages/performance-tests/test/quick/recipes/updateHeavyScan.ts index dc00dfa79..253280c35 100644 --- a/packages/performance-tests/test/quick/recipes/updateHeavyScan.ts +++ b/packages/performance-tests/test/quick/recipes/updateHeavyScan.ts @@ -25,7 +25,8 @@ import { withEditTxn, } from "@itwin/core-backend"; import { DatasetDescriptor } from "../DatasetDescriptor"; -import { ScanLedger } from "../validation/scanOracle"; +import { ScanLedger, ScanLedgerEntry } from "../validation/scanOracle"; +import { queryCount } from "../validation/validateFixture"; const scanAspectClass = "QuickPerfScan:ScanAspect"; @@ -42,6 +43,18 @@ const scanAspectClass = "QuickPerfScan:ScanAspect"; * * The last region is the reason this recipe can catch squash regressions that a "the sizes match" * assertion cannot: those ids must cancel out entirely. + * + * Note there is no region for the fourth squash rule, delete-then-insert reinstating an insert. + * That rule is encoded in the oracle and unit-tested directly, but no region exercises it against + * a real changeset because it is not producible here: ids are assigned by the briefcase and a + * deleted ElementId, aspect id or relationship id is never handed out again, so a delete can never + * be followed by an insert of the same id within one scanned range. + * + * Elements carry no geometry. An unchanged GeometryStream column never appears in a SQLite + * changeset, so geometry would cost build time, disk and determinism while contributing nothing to + * scan cost. This recipe is tuned as a sensitive regression detector, not as a representative + * production workload; answering the representativeness question is a separate, geometry-bearing + * recipe. */ export interface ScanRegionSizes { readonly updated: number; @@ -238,7 +251,7 @@ export async function applyScanChangesets( accessToken: AccessToken, descriptor: DatasetDescriptor, state: ScanRecipeState -): Promise { +): Promise { const sizes = scanRegionSizes(descriptor); const changesetCount = descriptor.distribution.operations.sourceChangesets; const ledger = new ScanLedger(); @@ -443,5 +456,82 @@ export async function applyScanChangesets( }); } - return ledger; + return ledger.entries; +} + +/** + * Assert the built source iModel matches the region structure the recipe promised. + * + * This checks the tip state, which is what a copied briefcase actually contains. It is deliberately + * expressed in terms of regions rather than raw totals: a regression that deleted the wrong region + * would still produce plausible totals, but cannot produce the right per-region counts. + */ +export async function validateScanFixture( + db: BriefcaseDb, + descriptor: DatasetDescriptor +): Promise { + const sizes = scanRegionSizes(descriptor); + const regionCount = async (region: string) => + queryCount( + db, + `SELECT count(*) cnt FROM ${PhysicalObject.classFullName.replace( + ":", + "." + )} WHERE CodeValue LIKE 'scan-${region}-%'` + ); + + const expectations: Array<[string, number, number]> = [ + [ + "region A elements (updated throughout)", + await regionCount("a"), + sizes.updated, + ], + [ + "region B elements (deleted in the last changeset)", + await regionCount("b"), + 0, + ], + [ + "region C elements (inserted then updated)", + await regionCount("c"), + sizes.insertedThenUpdated, + ], + ["region D elements (inserted then deleted)", await regionCount("d"), 0], + [ + "scan aspects", + await queryCount(db, "SELECT count(*) cnt FROM QuickPerfScan.ScanAspect"), + sizes.updated + sizes.insertedThenUpdated, + ], + [ + "relationships", + await queryCount( + db, + "SELECT count(*) cnt FROM BisCore.ElementGroupsMembers" + ), + sizes.seedRelationships + + sizes.insertedRelationships - + sizes.deletedRelationships, + ], + [ + "updated relationships", + await queryCount( + db, + "SELECT count(*) cnt FROM BisCore.ElementGroupsMembers WHERE MemberPriority >= 1000" + ), + sizes.updatedRelationships, + ], + ]; + + const failures = expectations + .filter(([, actual, expected]) => actual !== expected) + .map( + ([what, actual, expected]) => + `${what}: expected ${expected}, got ${actual}` + ); + if (failures.length > 0) + throw new Error( + `Fixture "${descriptor.id}" does not match its recipe:\n ${failures.join( + "\n " + )}` + ); } diff --git a/packages/performance-tests/test/quick/scenarios/changesetScanning.ts b/packages/performance-tests/test/quick/scenarios/changesetScanning.ts new file mode 100644 index 000000000..294c362d4 --- /dev/null +++ b/packages/performance-tests/test/quick/scenarios/changesetScanning.ts @@ -0,0 +1,85 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { ChangedInstanceIds } from "@itwin/imodel-transformer"; +import { BenchmarkScenarioDefinition } from "../BenchmarkScenario"; +import { + PreparedDataset, + requireDetachedDataset, +} from "../FixtureMaterializer"; +import { updateHeavyScanDescriptor } from "../FixtureCatalog"; +import { + assertScanMatchesOracle, + scanDigest, + ScanExpectation, + ScanLedger, + ScanLedgerEntry, + squashLedger, +} from "../validation/scanOracle"; + +/** + * Measures {@link ChangedInstanceIds.initialize} over a set of local changeset files. + * + * The measured region is deliberately narrow. `IModelTransformer.initialize` also performs + * provenance initialization, clone-context and exporter setup and deleted-entity remapping, so a + * regression in any of those would surface here as a scanning regression, and — more importantly — + * a scanning regression would be diluted by everything else. Scanning is roughly a third of that + * larger region, so at the ~3% run-to-run variation this framework sees, a 20% scan regression + * would move the total by about 6% and disappear into the noise. Narrowing the region is what makes + * the regression detectable. + * + * `csFileProps` is the only one of the four `initialize` input modes that touches no hub: the other + * three re-enter `BriefcaseManager` to query or download changesets. That is why this scenario needs + * no hub and no target iModel, and why `HubMock` is deliberately not running during measurement. + */ +class ChangesetScanningScenario { + private _result?: ScanExpectation; + private _aborted = false; + + constructor(private readonly _dataset: PreparedDataset) {} + + public async measure(): Promise { + const dataset = requireDetachedDataset(this._dataset); + this._result = await ChangedInstanceIds.initialize({ + csFileProps: dataset.csFileProps, + iModel: dataset.sourceDb, + }); + } + + public abort(): void { + this._aborted = true; + } + + public async finish(): Promise { + if (this._aborted) throw new Error("Changeset scanning scenario aborted"); + const dataset = requireDetachedDataset(this._dataset); + if (!this._result) + throw new Error("Changeset scanning scenario finished before measuring"); + + const entries = dataset.recipeData as ScanLedgerEntry[] | undefined; + if (!entries) + throw new Error( + `Fixture "${dataset.descriptor.id}" carries no recipe ledger, so the scan result cannot be verified` + ); + + // Verify against what the recipe says it did, not against a previous digest. A digest can only + // detect that behaviour changed; it cannot detect behaviour that was wrong from the start. + assertScanMatchesOracle( + this._result, + squashLedger(ScanLedger.fromEntries(entries)) + ); + return scanDigest(this._result); + } +} + +export const changesetScanningScenario: BenchmarkScenarioDefinition = { + id: "changeset-scanning", + defaultFixtureId: updateHeavyScanDescriptor.id, + capabilities: { + topology: "source-only", + requiredClaims: ["changeset scanning"], + }, + factory: (dataset) => new ChangesetScanningScenario(dataset), +}; diff --git a/packages/performance-tests/test/quick/validation/scanOracle.ts b/packages/performance-tests/test/quick/validation/scanOracle.ts index cb1adb458..d9db7a96d 100644 --- a/packages/performance-tests/test/quick/validation/scanOracle.ts +++ b/packages/performance-tests/test/quick/validation/scanOracle.ts @@ -91,9 +91,19 @@ interface MutableScanOps { /** * Squash semantics, written independently of `ChangedInstanceIds.handleChange`. * - * Reusing the transformer's implementation would check the code against itself. The rules are: - * an insert clears a pending delete; an update is dropped when the instance is already an insert; - * a delete cancels a pending insert outright and otherwise supersedes a pending update. + * Reusing the transformer's implementation would check the code against itself. There are four + * rules: + * + * 1. an insert clears a pending delete; + * 2. an update is dropped when the instance is already an insert; + * 3. a delete cancels a pending insert outright; + * 4. a delete otherwise supersedes a pending update. + * + * Rule 1 is encoded and unit-tested here but no fixture region exercises it against a real + * changeset, because it is not producible: instance ids are assigned by the briefcase and a deleted + * id is never handed out again, so within one scanned range a delete can never be followed by an + * insert of the same id. It is implemented anyway rather than omitted, because an unimplemented + * rule would silently disagree with the transformer if a future fixture ever did produce it. */ function applyOp(ops: MutableScanOps, op: ScanOp, id: Id64String): void { switch (op) { diff --git a/packages/performance-tests/test/quick/validation/validateFixture.ts b/packages/performance-tests/test/quick/validation/validateFixture.ts index 464f2e377..d80a458fe 100644 --- a/packages/performance-tests/test/quick/validation/validateFixture.ts +++ b/packages/performance-tests/test/quick/validation/validateFixture.ts @@ -108,7 +108,10 @@ async function queryGeometryRecords(db: BriefcaseDb): Promise { return values; } -async function queryCount(db: BriefcaseDb, ecsql: string): Promise { +export async function queryCount( + db: BriefcaseDb, + ecsql: string +): Promise { const reader = db.createQueryReader(ecsql, undefined, { usePrimaryConn: true, }); From 29e54c0d35d9994b26838b426c98b6f150642f38 Mon Sep 17 00:00:00 2001 From: Nam Le <50554904+hl662@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:32:40 -0400 Subject: [PATCH 12/19] perf(tests): reject repeated deletes in the scan ledger The ledger deduplicated repeated ops on the stated grounds that all three squash rules are idempotent under repetition. That is false for Deleted. Its branch in ChangedInstanceIds.handleChange is conditional on insertIds.has(id), and the first delete falsifies that condition: Inserted -> Deleted => id in no set (the two cancel) Inserted -> Deleted -> Deleted => id in deleteIds So deduplicating a repeated delete silently produces the wrong expectation. The current recipe never emits one, but the justification was unsound, which would have mislead the next recipe author. Repeated deletes are not producible from a valid changeset sequence, so the ledger now rejects a second Deleted for an id as a recipe authoring error rather than deduplicating it. Rejecting is strictly more informative than deduping here. Inserts and updates are genuinely idempotent and are still deduplicated; the scale argument for that is unaffected. squashLedger now also accepts raw entries so that squash semantics can be pinned directly by tests, including sequences record() refuses. Adds tests pinning the non-idempotent case and both double-membership sequences the transformer produces. See the accompanying finding on handleChange's incomplete reconciliation: the oracle mirrors that behaviour deliberately rather than normalizing it. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../test/quick/scanOracle.quick-unit.ts | 50 ++++++++++++++++++- .../test/quick/validation/scanOracle.ts | 39 ++++++++++++--- 2 files changed, 82 insertions(+), 7 deletions(-) diff --git a/packages/performance-tests/test/quick/scanOracle.quick-unit.ts b/packages/performance-tests/test/quick/scanOracle.quick-unit.ts index 6a241d74b..fe6e2a224 100644 --- a/packages/performance-tests/test/quick/scanOracle.quick-unit.ts +++ b/packages/performance-tests/test/quick/scanOracle.quick-unit.ts @@ -91,7 +91,7 @@ describe("changeset scan oracle", () => { expect(squashed.font.insertIds.size).to.equal(0); }); - it("drops repeated ops without changing the squashed outcome", () => { + it("drops repeated inserts and updates without changing the outcome", () => { const ledger = new ScanLedger(); for (let index = 0; index < 50; index++) ledger.record("element", "Updated", "0x40"); @@ -100,6 +100,54 @@ describe("changeset scan oracle", () => { expect([...squashLedger(ledger).element.deleteIds]).to.deep.equal(["0x40"]); }); + it("rejects a repeated delete instead of deduplicating it", () => { + const ledger = new ScanLedger(); + ledger.record("element", "Inserted", "0x41"); + ledger.record("element", "Deleted", "0x41"); + expect(() => ledger.record("element", "Deleted", "0x41")).to.throw( + /second "Deleted"/ + ); + }); + + // Pins the reason the recorder rejects repeated deletes rather than deduplicating them: the + // Deleted branch is conditional on insertIds.has(id), and the first delete falsifies that + // condition. Deduplicating would collapse this to the empty result, which is wrong. + it("does not treat a repeated delete as idempotent", () => { + const insertThenDelete = squashLedger([ + { collection: "element", id: "0x42", op: "Inserted" }, + { collection: "element", id: "0x42", op: "Deleted" }, + ]); + expect(insertThenDelete.element.insertIds.size).to.equal(0); + expect(insertThenDelete.element.deleteIds.size).to.equal(0); + + const withSecondDelete = squashLedger([ + { collection: "element", id: "0x42", op: "Inserted" }, + { collection: "element", id: "0x42", op: "Deleted" }, + { collection: "element", id: "0x42", op: "Deleted" }, + ]); + expect([...withSecondDelete.element.deleteIds]).to.deep.equal(["0x42"]); + }); + + // The transformer's Inserted branch clears deleteIds but not updateIds, and its Updated branch + // guards on insertIds but not deleteIds, so each leaves an id in two collections at once. Neither + // sequence is producible from a changeset, but both are reachable through addCustomElementChange. + // The oracle mirrors the behaviour rather than normalizing it, so that it keeps agreeing with the + it("mirrors the transformer's incomplete reconciliation rather than normalizing it", () => { + const updateThenInsert = squashLedger([ + { collection: "element", id: "0x43", op: "Updated" }, + { collection: "element", id: "0x43", op: "Inserted" }, + ]); + expect([...updateThenInsert.element.insertIds]).to.deep.equal(["0x43"]); + expect([...updateThenInsert.element.updateIds]).to.deep.equal(["0x43"]); + + const deleteThenUpdate = squashLedger([ + { collection: "element", id: "0x44", op: "Deleted" }, + { collection: "element", id: "0x44", op: "Updated" }, + ]); + expect([...deleteThenUpdate.element.updateIds]).to.deep.equal(["0x44"]); + expect([...deleteThenUpdate.element.deleteIds]).to.deep.equal(["0x44"]); + }); + it("records batches of ids", () => { const ledger = new ScanLedger(); ledger.record("aspect", "Inserted", ["0x50", "0x51"]); diff --git a/packages/performance-tests/test/quick/validation/scanOracle.ts b/packages/performance-tests/test/quick/validation/scanOracle.ts index d9db7a96d..529ba545f 100644 --- a/packages/performance-tests/test/quick/validation/scanOracle.ts +++ b/packages/performance-tests/test/quick/validation/scanOracle.ts @@ -41,13 +41,23 @@ export type ScanExpectation = Readonly>; /** * Ordered record of every change a recipe performs, used to predict the scan result. * - * Repeating an op for the same instance is idempotent under all three squash rules, so the recorder - * drops repeats. That keeps an update-heavy recipe's ledger proportional to touched instances rather - * than to touched instances times changesets, without changing the squashed outcome. + * An update-heavy recipe touches the same instances in every changeset, so recording every sighting + * would make the ledger proportional to instances times changesets. The recorder drops a repeat of + * the op it last saw for that instance, which is safe because `Inserted` and `Updated` are both + * idempotent under repetition: `Inserted` unconditionally adds to `insertIds`, and `Updated` either + * adds to `updateIds` or is suppressed by a state a second sighting cannot change. + * + * `Deleted` is *not* idempotent, so it is rejected rather than deduplicated. Its branch is + * conditional on `insertIds.has(id)` and the first delete falsifies that condition, so + * insert-then-delete cancels to nothing while insert-then-delete-then-delete leaves the id in + * `deleteIds`. Deduplicating it would silently produce the wrong expectation for any recipe that + * repeated one. A repeated delete is not producible from a valid changeset sequence anyway - the + * row is already gone - so rejecting it reports a recipe authoring error instead of hiding it. */ export class ScanLedger { private readonly _entries: ScanLedgerEntry[] = []; private readonly _lastOp = new Map(); + private readonly _deleted = new Set(); public record( collection: ScanCollection, @@ -57,7 +67,13 @@ export class ScanLedger { const iterable = typeof ids === "string" ? [ids] : ids; for (const id of iterable) { const key = `${collection}\u0000${id}`; - if (this._lastOp.get(key) === op) continue; + if (op === "Deleted") { + if (this._deleted.has(key)) + throw new Error( + `Ledger recorded a second "Deleted" for ${collection} ${id}. Deletes do not squash idempotently, and a valid changeset sequence cannot delete a row twice.` + ); + this._deleted.add(key); + } else if (this._lastOp.get(key) === op) continue; this._lastOp.set(key, op); this._entries.push({ collection, id, op }); } @@ -125,7 +141,15 @@ function applyOp(ops: MutableScanOps, op: ScanOp, id: Id64String): void { } } -export function squashLedger(ledger: ScanLedger): ScanExpectation { +/** + * Replays a ledger through the squash rules. + * + * Accepts raw entries as well as a {@link ScanLedger} so that squash semantics can be pinned + * directly by tests, including sequences {@link ScanLedger.record} deliberately refuses to accept. + */ +export function squashLedger( + ledger: ScanLedger | readonly ScanLedgerEntry[] +): ScanExpectation { const result = Object.fromEntries( scanCollections.map((collection) => [ collection, @@ -136,7 +160,10 @@ export function squashLedger(ledger: ScanLedger): ScanExpectation { }, ]) ) as Record; - for (const entry of ledger.entries) + const entries = Array.isArray(ledger) + ? (ledger as readonly ScanLedgerEntry[]) + : (ledger as ScanLedger).entries; + for (const entry of entries) applyOp(result[entry.collection], entry.op, entry.id); return result; } From ca5af8241258b6fbbaa4b710cf07fe47150c1e7c Mon Sep 17 00:00:00 2001 From: Nam Le <50554904+hl662@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:35:08 -0400 Subject: [PATCH 13/19] perf(tests): recalibrate changeset-scanning shape at scanScale 16 Verified against a real 8-sample run (varianceStatus stable, CV 1.0%): build 19.3s, median scan 3.46s, ~51ms per-sample overhead. Updates the doc comment's numbers, which were still describing scanScale 11, and corrects the 128-execution headroom claim from ~3x to ~2x against the 15-minute budget at this shape. --- .../test/quick/FixtureCatalog.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/performance-tests/test/quick/FixtureCatalog.ts b/packages/performance-tests/test/quick/FixtureCatalog.ts index 649253378..26d16e281 100644 --- a/packages/performance-tests/test/quick/FixtureCatalog.ts +++ b/packages/performance-tests/test/quick/FixtureCatalog.ts @@ -113,18 +113,19 @@ export const balancedIncrementalSourceOnlyDescriptor: DatasetDescriptor = { * Region sizes are derived from `base.elements` by `scanRegionSizes`; see `recipes/updateHeavyScan` * for what each region proves. * - * Calibrated at 2,420 elements x 20 changesets: the scan measures ~2.15 s with a coefficient of - * variation of 2.4% over 8 samples, against ~44 ms of per-sample copy, verification and teardown. - * Scan cost is linear in changed rows at roughly 48 ms per (1,000 elements x changeset), so the + * Calibrated at 3,520 elements x 20 changesets: the scan measures ~3.46 s with a coefficient of + * variation of 1.0% over 8 samples, against ~51 ms of per-sample copy, verification and teardown. + * Scan cost is linear in changed rows at roughly 49 ms per (1,000 elements x changeset), so the * shape is a single `scanScale` knob. * * The size is bounded by comparison mode rather than by a single run. One run is 9 executions and - * finishes in well under a minute, but an A/B comparison at its escalated width is 128 executions, - * where the build cost amortizes away and the per-execution cost is all that matters. This shape - * keeps that case inside a 15 minute budget with roughly 3x headroom for slower CI hardware; a - * larger shape would measure the same thing no better and fit the escalated case worse. + * finishes in well under a minute (measured: 52 s including the 19.3 s build), but an A/B + * comparison at its escalated width is 128 executions, where the build cost amortizes away and the + * per-execution cost is all that matters. This shape keeps that case inside a 15 minute budget with + * roughly 2x headroom for slower CI hardware; a larger shape would measure the same thing no better + * and fit the escalated case worse. */ -const scanScale = 11; +const scanScale = 16; const scanDistribution = { base: { // Region A (updated throughout) plus region B (updated, then deleted last). From 0f23183508c24108b7eb2f9f1231cb962f9202c7 Mon Sep 17 00:00:00 2001 From: Nam Le <50554904+hl662@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:37:04 -0400 Subject: [PATCH 14/19] perf(tests): guard the vacuous-pass Set trap, isolate quick-unit IModelHost profiles Adds a regression test proving a recipe returning a Set (which JSON.stringify silently collapses to "{}") fails the build loudly instead of emitting a half-written artifact that reads back as valid. Wires in startIsolatedHost/shutdownIsolatedHost so these correctness tests run on a private IModelHost profile instead of the default one, which is exclusively locked and shared by every iTwin.js process on the machine -- concurrent runs otherwise fail with an unrelated-looking "Db is busy" error. --- .../test/quick/FixtureArtifact.quick-unit.ts | 76 +++++++++++++++++-- .../test/quick/isolatedHost.ts | 35 +++++++++ 2 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 packages/performance-tests/test/quick/isolatedHost.ts diff --git a/packages/performance-tests/test/quick/FixtureArtifact.quick-unit.ts b/packages/performance-tests/test/quick/FixtureArtifact.quick-unit.ts index 2642cdf9d..3040e866a 100644 --- a/packages/performance-tests/test/quick/FixtureArtifact.quick-unit.ts +++ b/packages/performance-tests/test/quick/FixtureArtifact.quick-unit.ts @@ -7,7 +7,7 @@ import { expect } from "chai"; import * as fs from "fs"; import * as os from "os"; import * as path from "path"; -import { BriefcaseDb, IModelHost } from "@itwin/core-backend"; +import { BriefcaseDb } from "@itwin/core-backend"; import { HubMock } from "@itwin/core-backend/lib/cjs/internal/HubMock"; import { ChangedInstanceIds } from "@itwin/imodel-transformer"; import { DatasetDescriptor } from "./DatasetDescriptor"; @@ -31,6 +31,7 @@ import { requireFixtureArtifact, } from "./FixtureProvider"; import { detachedBriefcaseFixtureProvider } from "./providers/detachedBriefcaseProvider"; +import { shutdownIsolatedHost, startIsolatedHost } from "./isolatedHost"; describe("detached fixture artifact", () => { const descriptor = balancedIncrementalSourceOnlyDescriptor; @@ -40,7 +41,7 @@ describe("detached fixture artifact", () => { before(async function () { this.timeout(600_000); root = fs.mkdtempSync(path.join(os.tmpdir(), "quick-artifact-")); - await IModelHost.startup(); + await startIsolatedHost(); built = await detachedBriefcaseFixtureProvider.build( descriptor, path.join(root, "fixture-artifact") @@ -49,7 +50,7 @@ describe("detached fixture artifact", () => { after(async () => { await detachedBriefcaseFixtureProvider.disposeBuild(built); - if (IModelHost.isValid) await IModelHost.shutdown(); + await shutdownIsolatedHost(); fs.rmSync(root, { recursive: true, force: true }); }); @@ -331,7 +332,7 @@ describe("recipe data across the stage boundary", () => { }, }); root = fs.mkdtempSync(path.join(os.tmpdir(), "quick-recipe-data-")); - await IModelHost.startup(); + await startIsolatedHost(); built = await detachedBriefcaseFixtureProvider.build( descriptor, path.join(root, "fixture-artifact") @@ -340,7 +341,7 @@ describe("recipe data across the stage boundary", () => { after(async () => { await detachedBriefcaseFixtureProvider.disposeBuild(built); - if (IModelHost.isValid) await IModelHost.shutdown(); + await shutdownIsolatedHost(); fs.rmSync(root, { recursive: true, force: true }); }); @@ -377,3 +378,68 @@ describe("recipe data across the stage boundary", () => { } }); }); + +/** + * The vacuous-pass guard, exercised through the real build rather than the writer alone. + * + * A recipe returning a `Set` of ids is the realistic mistake: `JSON.stringify(new Set([...]))` is + * `"{}"`, so without validation stage 1 would happily emit an artifact whose ledger is an empty + * object. A scanning oracle would then compare its results against nothing and report green. The + * build must fail instead, and must not leave behind a directory that later reads as a valid + * artifact. + */ +describe("recipe data that cannot survive JSON", () => { + const recipeId = "balanced-incremental-emitting-a-set"; + const descriptor: DatasetDescriptor = { + ...balancedIncrementalSourceOnlyDescriptor, + id: "balanced-incremental-emitting-a-set", + layout: { + ...balancedIncrementalSourceOnlyDescriptor.layout, + recipe: recipeId, + }, + }; + let root: string; + + before(async function () { + this.timeout(600_000); + registerFixtureRecipe({ + id: recipeId, + createSeed: async (fileName, forDescriptor) => + balancedIncrementalRecipe.createSeed(fileName, forDescriptor), + applySourceChangesets: async (db, token, forDescriptor, state) => { + await balancedIncrementalRecipe.applySourceChangesets( + db, + token, + forDescriptor, + state + ); + return { elements: { deleteIds: new Set(["0x20", "0x21"]) } }; + }, + }); + root = fs.mkdtempSync(path.join(os.tmpdir(), "quick-bad-recipe-")); + await startIsolatedHost(); + }); + + after(async () => { + await shutdownIsolatedHost(); + fs.rmSync(root, { recursive: true, force: true }); + }); + + it("fails the build instead of emitting an empty ledger", async function () { + this.timeout(600_000); + const artifactDir = path.join(root, "fixture-artifact"); + let built: BuiltFixture | undefined; + try { + built = await detachedBriefcaseFixtureProvider.build( + descriptor, + artifactDir + ); + } catch (error) { + expect((error as Error).message).to.match(/Recipe data at .* is a Set/); + } + expect(built, "the build must not succeed").to.equal(undefined); + // A half-written artifact must never read back as usable. + expect(() => readFixtureArtifact(artifactDir)).to.throw(); + expect(fs.existsSync(path.join(artifactDir, "recipe.json"))).to.be.false; + }); +}); diff --git a/packages/performance-tests/test/quick/isolatedHost.ts b/packages/performance-tests/test/quick/isolatedHost.ts new file mode 100644 index 000000000..d5e262a29 --- /dev/null +++ b/packages/performance-tests/test/quick/isolatedHost.ts @@ -0,0 +1,35 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "fs"; +import { IModelHost } from "@itwin/core-backend"; + +/** + * Start `IModelHost` on a profile private to this process. + * + * `IModelHost.startup()` takes an exclusive lock on its profile directory, and the default profile + * is shared by every iTwin.js process on the machine. Two concurrent runs therefore fail with + * `Db is busy: Profile [...] is already in use by another process` — which surfaces far from its + * cause, as an unrelated seed or teardown error. + * + * The benchmark itself deliberately keeps the default profile: its cache state is part of what is + * being measured, and changing that would move published numbers. These are correctness tests, so + * isolation is free and makes them safe to run alongside a benchmark. + */ +export async function startIsolatedHost(): Promise { + await IModelHost.startup({ + profileName: `quick-unit-${process.pid}`, + }); +} + +/** Shut down and remove the private profile, so runs do not accumulate directories. */ +export async function shutdownIsolatedHost(): Promise { + if (!IModelHost.isValid) return; + const profileDir = IModelHost.profileDir; + await IModelHost.shutdown(); + // Only ever remove a profile this helper created. + if (/quick-unit-\d+/.test(profileDir)) + fs.rmSync(profileDir, { recursive: true, force: true }); +} From 6d539dbd12fb188f4e328d052686dbe3ae95e503 Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Date: Wed, 29 Jul 2026 15:39:15 -0400 Subject: [PATCH 15/19] Scope environment setup to weekly tests Keep credential loading in the Hub-backed regression suite so local helper unit tests run without CI or a .env file. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- packages/performance-tests/vitest.config.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/performance-tests/vitest.config.ts b/packages/performance-tests/vitest.config.ts index 38c500a75..4c42d9f66 100644 --- a/packages/performance-tests/vitest.config.ts +++ b/packages/performance-tests/vitest.config.ts @@ -10,7 +10,6 @@ export default defineConfig({ globals: true, environment: "node", include: ["test/TransformerRegression.test.ts", "test/unit/**/*.test.ts"], - setupFiles: ["./test/setup.ts"], // Transformations and worker-owned startup/teardown may legitimately run for hours. testTimeout: 0, hookTimeout: 0, From 479dc31a426b62841264f743a3e70671aca6ba5e Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Date: Wed, 29 Jul 2026 15:44:39 -0400 Subject: [PATCH 16/19] Scope weekly performance lifecycle settings Use IModelHost validity for partial-startup cleanup and keep unlimited timeouts local to the credentialed weekly regression file. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../test/TransformerRegression.test.ts | 12 +++--------- packages/performance-tests/vitest.config.ts | 3 --- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/packages/performance-tests/test/TransformerRegression.test.ts b/packages/performance-tests/test/TransformerRegression.test.ts index 799550459..66be0c493 100644 --- a/packages/performance-tests/test/TransformerRegression.test.ts +++ b/packages/performance-tests/test/TransformerRegression.test.ts @@ -73,26 +73,21 @@ const testCasesMap = new Map>([ const loggerCategory = "Transformer Performance Regression Tests"; const outputDir = path.join(__dirname, ".output"); +vi.setConfig({ testTimeout: 0, hookTimeout: 0 }); + class WorkerLifecycle { private _authClient?: AuthorizationClient; - private _hostStarted = false; public setAuthClient(authClient: AuthorizationClient): void { this._authClient = authClient; } - public markHostStarted(): void { - this._hostStarted = true; - } - public async shutdown(): Promise { - const hostStarted = this._hostStarted; const authClient = this._authClient; - this._hostStarted = false; this._authClient = undefined; const cleanupTasks: CleanupTask[] = []; - if (hostStarted) { + if (IModelHost.isValid) { cleanupTasks.push({ name: "IModelHost shutdown", run: async () => IModelHost.shutdown(), @@ -191,7 +186,6 @@ async function setupTestData( }); hostConfig.hubAccess = new BackendIModelsAccess(hubClient); await IModelHost.startup(hostConfig); - workerLifecycle.markHostStarted(); return preFetchAsyncIterator(getTestIModels(filterIModels)); } diff --git a/packages/performance-tests/vitest.config.ts b/packages/performance-tests/vitest.config.ts index 4c42d9f66..b42dd75d1 100644 --- a/packages/performance-tests/vitest.config.ts +++ b/packages/performance-tests/vitest.config.ts @@ -10,9 +10,6 @@ export default defineConfig({ globals: true, environment: "node", include: ["test/TransformerRegression.test.ts", "test/unit/**/*.test.ts"], - // Transformations and worker-owned startup/teardown may legitimately run for hours. - testTimeout: 0, - hookTimeout: 0, pool: "forks", maxWorkers: 1, fileParallelism: false, From 96c981d20985bbc5ad7229f2a01832d6903d967d Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Date: Wed, 29 Jul 2026 15:55:45 -0400 Subject: [PATCH 17/19] Remove redundant performance test log Rely on Vitest's per-test reporting instead of emitting an unstructured completion message. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- packages/performance-tests/test/TransformerRegression.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/performance-tests/test/TransformerRegression.test.ts b/packages/performance-tests/test/TransformerRegression.test.ts index 66be0c493..32b71a720 100644 --- a/packages/performance-tests/test/TransformerRegression.test.ts +++ b/packages/performance-tests/test/TransformerRegression.test.ts @@ -306,8 +306,6 @@ describe("Transformer Regression Tests", () => { transformerModule: definition.transformerModule, addReport, }); - // eslint-disable-next-line no-console - console.log("Finished the test"); }); } }); From 33bf668915c30b7a8a871165da5cf0836aa8284b Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Date: Wed, 29 Jul 2026 16:11:11 -0400 Subject: [PATCH 18/19] Document performance test architecture Explain the weekly regression lifecycle, helper unit tests, registration matrix, cleanup, reporting contract, and extension boundaries. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- packages/performance-tests/ARCHITECTURE.md | 118 +++++++++++++++++++++ packages/performance-tests/README.md | 3 + 2 files changed, 121 insertions(+) create mode 100644 packages/performance-tests/ARCHITECTURE.md diff --git a/packages/performance-tests/ARCHITECTURE.md b/packages/performance-tests/ARCHITECTURE.md new file mode 100644 index 000000000..9b59e0b58 --- /dev/null +++ b/packages/performance-tests/ARCHITECTURE.md @@ -0,0 +1,118 @@ +# Performance Test Architecture + +This package contains the transformer's performance regression tests and the +unit tests for their supporting infrastructure. Both use Vitest, but they have +different runtime requirements. + +## Test categories + +| Category | Location | Purpose | External requirements | +| --- | --- | --- | --- | +| Weekly regression | `test/TransformerRegression.test.ts` | Measure transformer implementations against selected Hub iModels and a generated local iModel | Hub credentials, OIDC configuration, and network access | +| Infrastructure unit | `test/unit/**/*.test.ts` | Verify registration and cleanup behavior | None | + +The unit tests do not measure transformer performance. They validate code used +to construct and tear down the weekly regression suite. + +## Vitest configuration + +`vitest.config.ts` includes both categories and uses one forked worker with file +parallelism disabled. The weekly tests share process-wide resources such as +`IModelHost`, authentication, downloaded briefcases, and the CSV reporter, so +they must run serially. + +The weekly regression file opts out of test and hook timeouts because individual +transformations and downloads can run for hours. Infrastructure unit tests keep +Vitest's default timeouts so a broken unit test fails instead of hanging the +worker. + +Environment loading is scoped to `TransformerRegression.test.ts`. Unit tests can +therefore run locally without a `.env` file by running +`pnpm exec vitest run test/unit`. + +The repository's root `pnpm test` excludes this package. Run its tests explicitly +from `packages/performance-tests`. + +## Weekly regression lifecycle + +The weekly suite has two phases. + +### Collection + +Before running any tests, Vitest imports `TransformerRegression.test.ts` to +discover them. During this setup step, the module: + +1. Load environment configuration and authenticate. +2. Start `IModelHost` with Hub access. +3. Discover and filter the configured Hub iModels. +4. Add the generated local iModel. +5. Load the built-in and optional comparison transformer modules. +6. Build the supported test-case/module combinations. + +If collection fails after authentication or host startup begins, the worker +lifecycle attempts to shut down every initialized resource before preserving +and rethrowing the original error. + +### Execution + +For each selected iModel, the suite: + +1. Download or generate a local `.bim` source database and record its report + metadata. +2. Opens a fresh read-only source database for each test. +3. Runs every supported test-case/module combination. +4. Closes the source database after each test. + +The raw-insert comparison runs after the per-iModel transform cases. Once all +tests finish, the suite exports `test/.output/report.csv`, shuts down +`IModelHost`, and signs out. Cleanup tasks are all attempted even if an earlier +task fails. + +## Test registration + +`RegressionTestRegistration.ts` creates the execution matrix; it does not run +tests or store results. + +Each test case names the factory function it requires from +`TestTransformerModule`. Each loaded transformer module is paired only with the +test cases it supports. `TransformerRegression.test.ts` consumes those +definitions and registers the corresponding Vitest tests. + +The module name is the human-readable identifier used in test names and report +entries. The module object contains the implementation that the test executes. +Additional implementations can be loaded through `EXTRA_TRANSFORMERS`. + +## Inputs and authentication + +`template.env` documents the weekly suite's environment variables. The important +input controls are: + +- `ITWIN_IDS`: iTwins from which test iModels are discovered. +- `IMODEL_IDS`: specific iModels to include, or `*` for every iModel in the + configured iTwins. +- `EXTRA_TRANSFORMERS`: optional module paths for comparison implementations. +- `LOG_LEVEL`: verbosity for the iTwin logger. + +CI uses headless authentication. Local weekly runs use the CLI authorization +client and still require the OIDC and Hub configuration described in +`template.env`. Never commit a populated `.env` file. + +## Reporting + +Test cases report measurements through the callback in `TestCaseContext`. +`@itwin/perf-tools` combines those measurements with iModel and branch metadata, +then writes `test/.output/report.csv`. + +That path and CSV format are the artifact contract consumed by the hosted weekly +performance pipeline. Changes to either require coordinated pipeline validation. + +## Extending the package + +- Put credential-free tests of support code under `test/unit`. +- Add a weekly performance case under `test/cases`, declare its required + transformer factory in `TestTransformerModule`, and add it to `testCasesMap`. +- Add a built-in transformer implementation under `test/transformers`, or load a + comparison implementation with `EXTRA_TRANSFORMERS`. +- Give future performance-test categories their own entry file, setup, timeout, + and reporting policy. Do not make Hub credentials or unlimited timeouts global + merely because the weekly suite needs them. diff --git a/packages/performance-tests/README.md b/packages/performance-tests/README.md index b520154e2..b971251a1 100644 --- a/packages/performance-tests/README.md +++ b/packages/performance-tests/README.md @@ -2,6 +2,9 @@ A package containing performance tests for the [`@itwin/imodel-transformer` library](../../README.md). +See [ARCHITECTURE.md](./ARCHITECTURE.md) for the test categories, weekly suite +lifecycle, registration model, and extension guidance. + ## Tests Here are tests we need but don't have: From 7686109ad129678c1c183c3c284ca5a877506117 Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Date: Thu, 30 Jul 2026 17:59:47 -0400 Subject: [PATCH 19/19] Remove scan behavior oracle from benchmark Keep benchmark validation focused on exercising changeset scanning while leaving transformer semantics to transformer tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../test/quick/src/fixtures/FixtureRecipe.ts | 6 +- .../src/fixtures/recipes/updateHeavyScan.ts | 69 +---- .../src/fixtures/validation/scanOracle.ts | 250 ------------------ .../quick/src/scenarios/changesetScanning.ts | 78 ++++-- .../test/quick/tests/unit/scanOracle.test.ts | 229 ---------------- 5 files changed, 68 insertions(+), 564 deletions(-) delete mode 100644 packages/performance-tests/test/quick/src/fixtures/validation/scanOracle.ts delete mode 100644 packages/performance-tests/test/quick/tests/unit/scanOracle.test.ts diff --git a/packages/performance-tests/test/quick/src/fixtures/FixtureRecipe.ts b/packages/performance-tests/test/quick/src/fixtures/FixtureRecipe.ts index 9469ccd5b..35e91b040 100644 --- a/packages/performance-tests/test/quick/src/fixtures/FixtureRecipe.ts +++ b/packages/performance-tests/test/quick/src/fixtures/FixtureRecipe.ts @@ -17,7 +17,6 @@ import { ScanRecipeState, validateScanFixture, } from "./recipes/updateHeavyScan.js"; -import { ScanLedgerEntry } from "./validation/scanOracle.js"; import { assertFixtureDistribution } from "./validation/validateFixture.js"; /** @@ -65,10 +64,7 @@ export const balancedIncrementalRecipe: FixtureRecipe = { validate: async (db, descriptor) => assertFixtureDistribution(db, descriptor), }; -export const updateHeavyScanRecipe: FixtureRecipe< - ScanRecipeState, - readonly ScanLedgerEntry[] -> = { +export const updateHeavyScanRecipe: FixtureRecipe = { id: "update-heavy-scan", createSeed: async (fileName, descriptor) => createScanSeed(fileName, descriptor), diff --git a/packages/performance-tests/test/quick/src/fixtures/recipes/updateHeavyScan.ts b/packages/performance-tests/test/quick/src/fixtures/recipes/updateHeavyScan.ts index ddd38d31e..eefb512c0 100644 --- a/packages/performance-tests/test/quick/src/fixtures/recipes/updateHeavyScan.ts +++ b/packages/performance-tests/test/quick/src/fixtures/recipes/updateHeavyScan.ts @@ -24,7 +24,6 @@ import { withEditTxn, } from "@itwin/core-backend"; import { FixtureDescriptor } from "../FixtureDescriptor.js"; -import { ScanLedger, ScanLedgerEntry } from "../validation/scanOracle.js"; import { queryCount } from "../validation/validateFixture.js"; import { quickPath } from "../../support/paths.js"; @@ -251,15 +250,12 @@ export async function applyScanChangesets( accessToken: AccessToken, descriptor: FixtureDescriptor, state: ScanRecipeState -): Promise { +): Promise { const sizes = scanRegionSizes(descriptor); const changesetCount = descriptor.distribution.operations.sourceChangesets; - const ledger = new ScanLedger(); const insertedThenUpdatedIds: Id64String[] = []; - const insertedThenUpdatedAspectIds: Id64String[] = []; const insertedThenDeletedIds: Id64String[] = []; - const insertedRelationshipIds: Id64String[] = []; const updateElements = ( txn: EditTxn, @@ -279,7 +275,6 @@ export async function applyScanChangesets( id, } as PhysicalElementProps) ); - ledger.record("element", "Updated", ids); }; const updateAspects = ( @@ -299,7 +294,6 @@ export async function applyScanChangesets( ) ) ); - ledger.record("aspect", "Updated", aspectIds); }; for (let changeset = 1; changeset <= changesetCount; changeset++) { @@ -320,14 +314,10 @@ export async function applyScanChangesets( ) ); insertedThenUpdatedIds.push(elementId); - insertedThenUpdatedAspectIds.push( - txn.insertAspect( - scanAspectProps(elementId, index, `c-aspect-${index}`) - ) + txn.insertAspect( + scanAspectProps(elementId, index, `c-aspect-${index}`) ); } - ledger.record("element", "Inserted", insertedThenUpdatedIds); - ledger.record("aspect", "Inserted", insertedThenUpdatedAspectIds); // Region D carries no aspects. Whether cascade-deleting an aspect that was inserted within // the same scanned range emits a row at all is a second unknown, and this region already @@ -344,45 +334,22 @@ export async function applyScanChangesets( ) ) ); - ledger.record("element", "Inserted", insertedThenDeletedIds); - - // Inserting elements into a model bumps that model's row, so the model the recipe fills is - // expected to appear as an update even though the recipe never edits the model itself. - ledger.record("model", "Updated", state.modelId); + PhysicalModel.insert(txn, IModel.rootSubjectId, "ScanExtraModel"); - // A model insert also inserts its modeled partition element under the same id, so this one - // call is expected to show up in two collections. - const extraModelId = PhysicalModel.insert( + db.codeSpecs.insert( txn, - IModel.rootSubjectId, - "ScanExtraModel" - ); - ledger.record("model", "Inserted", extraModelId); - ledger.record("element", "Inserted", extraModelId); - // ...and that partition element lands in the repository model, bumping it in turn. - ledger.record("model", "Updated", IModel.repositoryModelId); - - ledger.record( - "codeSpec", - "Inserted", - db.codeSpecs.insert( - txn, - CodeSpec.create(db, "ScanCodeSpec", CodeScopeSpec.Type.Repository) - ) + CodeSpec.create(db, "ScanCodeSpec", CodeScopeSpec.Type.Repository) ); for (let index = 0; index < sizes.insertedRelationships; index++) - insertedRelationshipIds.push( - txn.insertRelationship( - ElementGroupsMembers.create( - db, - state.updatedIds[index], - state.updatedIds[index + sizes.seedRelationships + 1], - index - ).toJSON() - ) + txn.insertRelationship( + ElementGroupsMembers.create( + db, + state.updatedIds[index], + state.updatedIds[index + sizes.seedRelationships + 1], + index + ).toJSON() ); - ledger.record("relationship", "Inserted", insertedRelationshipIds); } updateElements(txn, state.updatedIds, "a", changeset); @@ -421,8 +388,6 @@ export async function applyScanChangesets( relationship.memberPriority += 1000; txn.updateRelationship(relationship.toJSON()); } - ledger.record("relationship", "Updated", updated); - const deleted = state.relationshipIds.slice( sizes.updatedRelationships, sizes.updatedRelationships + sizes.deletedRelationships @@ -436,17 +401,11 @@ export async function applyScanChangesets( ) .toJSON() ); - ledger.record("relationship", "Deleted", deleted); } if (isLast) { txn.deleteElement([...state.deletedLateIds]); - ledger.record("element", "Deleted", state.deletedLateIds); - // Deleting an element cascades to the aspects it owns. - ledger.record("aspect", "Deleted", state.deletedLateAspectIds); - txn.deleteElement(insertedThenDeletedIds); - ledger.record("element", "Deleted", insertedThenDeletedIds); } }); @@ -455,8 +414,6 @@ export async function applyScanChangesets( description: `scan changeset ${changeset} of ${changesetCount}`, }); } - - return ledger.entries; } /** diff --git a/packages/performance-tests/test/quick/src/fixtures/validation/scanOracle.ts b/packages/performance-tests/test/quick/src/fixtures/validation/scanOracle.ts deleted file mode 100644 index 6bb12ebd3..000000000 --- a/packages/performance-tests/test/quick/src/fixtures/validation/scanOracle.ts +++ /dev/null @@ -1,250 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Bentley Systems, Incorporated. All rights reserved. - * See LICENSE.md in the project root for license terms and full copyright notice. - *--------------------------------------------------------------------------------------------*/ - -import { Id64String } from "@itwin/core-bentley"; -import { canonicalSha256 } from "../FixtureDescriptor.js"; - -/** - * The six collections carried by `ChangedInstanceIds`. Verifying only `element` would let aspect and - * relationship regressions pass silently, so the scan oracle covers all of them. - */ -export const scanCollections = [ - "aspect", - "codeSpec", - "element", - "font", - "model", - "relationship", -] as const; - -export type ScanCollection = (typeof scanCollections)[number]; - -/** Mirrors `SqliteChangeOp` without importing it, so the oracle stays independent of transformer internals. */ -export type ScanOp = "Deleted" | "Inserted" | "Updated"; - -export interface ScanLedgerEntry { - readonly collection: ScanCollection; - readonly id: Id64String; - readonly op: ScanOp; -} - -export interface ScanOps { - readonly insertIds: ReadonlySet; - readonly updateIds: ReadonlySet; - readonly deleteIds: ReadonlySet; -} - -export type ScanExpectation = Readonly>; - -/** - * Ordered record of every change a recipe performs, used to predict the scan result. - * - * An update-heavy recipe touches the same instances in every changeset, so recording every sighting - * would make the ledger proportional to instances times changesets. The recorder drops a repeat of - * the op it last saw for that instance, which is safe because `Inserted` and `Updated` are both - * idempotent under repetition: `Inserted` unconditionally adds to `insertIds`, and `Updated` either - * adds to `updateIds` or is suppressed by a state a second sighting cannot change. - * - * `Deleted` is *not* idempotent, so it is rejected rather than deduplicated. Its branch is - * conditional on `insertIds.has(id)` and the first delete falsifies that condition, so - * insert-then-delete cancels to nothing while insert-then-delete-then-delete leaves the id in - * `deleteIds`. Deduplicating it would silently produce the wrong expectation for any recipe that - * repeated one. A repeated delete is not producible from a valid changeset sequence anyway - the - * row is already gone - so rejecting it reports a recipe authoring error instead of hiding it. - */ -export class ScanLedger { - private readonly _entries: ScanLedgerEntry[] = []; - private readonly _lastOp = new Map(); - private readonly _deleted = new Set(); - - public record( - collection: ScanCollection, - op: ScanOp, - ids: Id64String | Iterable - ): void { - const iterable = typeof ids === "string" ? [ids] : ids; - for (const id of iterable) { - const key = `${collection}\u0000${id}`; - if (op === "Deleted") { - if (this._deleted.has(key)) - throw new Error( - `Ledger recorded a second "Deleted" for ${collection} ${id}. Deletes do not squash idempotently, and a valid changeset sequence cannot delete a row twice.` - ); - this._deleted.add(key); - } else if (this._lastOp.get(key) === op) continue; - this._lastOp.set(key, op); - this._entries.push({ collection, id, op }); - } - } - - public get entries(): readonly ScanLedgerEntry[] { - return this._entries; - } - - /** - * Rebuilds a ledger from serialized entries. - * - * The recipe runs while the fixture artifact is built; the oracle runs later, against a copy. The - * ledger therefore has to survive a round trip through JSON. Replaying through `record` keeps the - * deduplication invariant even if the serialized form was hand-written or concatenated. - */ - public static fromEntries(entries: readonly ScanLedgerEntry[]): ScanLedger { - const ledger = new ScanLedger(); - for (const entry of entries) - ledger.record(entry.collection, entry.op, entry.id); - return ledger; - } -} - -interface MutableScanOps { - insertIds: Set; - updateIds: Set; - deleteIds: Set; -} - -/** - * Squash semantics, written independently of `ChangedInstanceIds.handleChange`. - * - * Reusing the transformer's implementation would check the code against itself. There are four - * rules: - * - * 1. an insert clears a pending delete; - * 2. an update is dropped when the instance is already an insert; - * 3. a delete cancels a pending insert outright; - * 4. a delete otherwise supersedes a pending update. - * - * Rule 1 is encoded and unit-tested here but no fixture region exercises it against a real - * changeset, because it is not producible: instance ids are assigned by the briefcase and a deleted - * id is never handed out again, so within one scanned range a delete can never be followed by an - * insert of the same id. It is implemented anyway rather than omitted, because an unimplemented - * rule would silently disagree with the transformer if a future fixture ever did produce it. - */ -function applyOp(ops: MutableScanOps, op: ScanOp, id: Id64String): void { - switch (op) { - case "Inserted": - ops.insertIds.add(id); - ops.deleteIds.delete(id); - return; - case "Updated": - if (!ops.insertIds.has(id)) ops.updateIds.add(id); - return; - case "Deleted": - if (ops.insertIds.has(id)) { - ops.insertIds.delete(id); - return; - } - ops.updateIds.delete(id); - ops.deleteIds.add(id); - return; - } -} - -/** - * Replays a ledger through the squash rules. - * - * Accepts raw entries as well as a {@link ScanLedger} so that squash semantics can be pinned - * directly by tests, including sequences {@link ScanLedger.record} deliberately refuses to accept. - */ -export function squashLedger( - ledger: ScanLedger | readonly ScanLedgerEntry[] -): ScanExpectation { - const result = Object.fromEntries( - scanCollections.map((collection) => [ - collection, - { - insertIds: new Set(), - updateIds: new Set(), - deleteIds: new Set(), - }, - ]) - ) as Record; - const entries = Array.isArray(ledger) - ? (ledger as readonly ScanLedgerEntry[]) - : (ledger as ScanLedger).entries; - for (const entry of entries) - applyOp(result[entry.collection], entry.op, entry.id); - return result; -} - -/** Numeric ordering for hex `Id64String`s, so the digest does not depend on string collation. */ -export function compareIds(left: Id64String, right: Id64String): number { - const leftValue = BigInt(left); - const rightValue = BigInt(right); - return leftValue < rightValue ? -1 : leftValue > rightValue ? 1 : 0; -} - -function sortedIds(ids: ReadonlySet): Id64String[] { - return [...ids].sort(compareIds); -} - -export function normalizeScanResult( - result: ScanExpectation -): Record> { - return Object.fromEntries( - scanCollections.map((collection) => [ - collection, - { - delete: sortedIds(result[collection].deleteIds), - insert: sortedIds(result[collection].insertIds), - update: sortedIds(result[collection].updateIds), - }, - ]) - ) as Record>; -} - -function difference( - actual: ReadonlySet, - expected: ReadonlySet -): { missing: Id64String[]; unexpected: Id64String[] } { - return { - missing: sortedIds(new Set([...expected].filter((id) => !actual.has(id)))), - unexpected: sortedIds( - new Set([...actual].filter((id) => !expected.has(id))) - ), - }; -} - -/** - * Asserts the scan result against what the recipe says it produced. This is strictly stronger than - * comparing a digest to its own previous value, which can only detect a change of behaviour and not - * a behaviour that was wrong from the start. - */ -export function assertScanMatchesOracle( - actual: ScanExpectation, - expected: ScanExpectation -): void { - const mismatches: string[] = []; - for (const collection of scanCollections) { - for (const op of ["insertIds", "updateIds", "deleteIds"] as const) { - const { missing, unexpected } = difference( - actual[collection][op], - expected[collection][op] - ); - if (missing.length === 0 && unexpected.length === 0) continue; - mismatches.push( - `${collection}.${op}: expected ${ - expected[collection][op].size - }, got ${actual[collection][op].size}; missing=[${missing.join( - "," - )}], unexpected=[${unexpected.join(",")}]` - ); - } - } - if (mismatches.length > 0) - throw new Error( - `Changeset scan did not match the recipe oracle:\n ${mismatches.join( - "\n " - )}` - ); -} - -/** - * Stable fingerprint over all six collections. Every sample scans a copy of one immutable artifact, - * so ids are identical across samples and equality is meaningful; it also serves as a cross-arm - * behaviour gate for A/B comparison. - */ -export function scanDigest(result: ScanExpectation): string { - return canonicalSha256(normalizeScanResult(result)); -} diff --git a/packages/performance-tests/test/quick/src/scenarios/changesetScanning.ts b/packages/performance-tests/test/quick/src/scenarios/changesetScanning.ts index 2a8c9fd40..1838db88f 100644 --- a/packages/performance-tests/test/quick/src/scenarios/changesetScanning.ts +++ b/packages/performance-tests/test/quick/src/scenarios/changesetScanning.ts @@ -5,20 +5,57 @@ import { ChangedInstanceIds } from "@itwin/imodel-transformer"; import { updateHeavyScanDescriptor } from "../catalogs/FixtureCatalog.js"; +import { canonicalSha256 } from "../fixtures/FixtureDescriptor.js"; import { PreparedDataset, requireDetachedDataset, } from "../fixtures/FixtureProvider.js"; -import { - assertScanMatchesOracle, - scanDigest, - ScanExpectation, - ScanLedger, - ScanLedgerEntry, - squashLedger, -} from "../fixtures/validation/scanOracle.js"; import { BenchmarkScenarioDefinition } from "../framework/BenchmarkScenario.js"; +type ScanResult = NonNullable< + Awaited> +>; + +function scanDigest(result: ScanResult): string { + const collections = { + aspect: result.aspect, + codeSpec: result.codeSpec, + element: result.element, + font: result.font, + model: result.model, + relationship: result.relationship, + }; + const normalized = Object.fromEntries( + Object.entries(collections).map(([collection, operations]) => [ + collection, + { + delete: [...operations.deleteIds].sort(), + insert: [...operations.insertIds].sort(), + update: [...operations.updateIds].sort(), + }, + ]) + ); + return canonicalSha256(normalized); +} + +function changedIdCount(result: ScanResult): number { + return [ + result.aspect, + result.codeSpec, + result.element, + result.font, + result.model, + result.relationship, + ].reduce( + (total, operations) => + total + + operations.insertIds.size + + operations.updateIds.size + + operations.deleteIds.size, + 0 + ); +} + /** * Measures {@link ChangedInstanceIds.initialize} over a set of local changeset files. * @@ -35,17 +72,22 @@ import { BenchmarkScenarioDefinition } from "../framework/BenchmarkScenario.js"; * no hub and no target iModel, and why `HubMock` is deliberately not running during measurement. */ class ChangesetScanningScenario { - private _result?: ScanExpectation; + private _result?: ScanResult; private _aborted = false; constructor(private readonly _dataset: PreparedDataset) {} public async measure(): Promise { const dataset = requireDetachedDataset(this._dataset); - this._result = await ChangedInstanceIds.initialize({ + if (dataset.csFileProps.length === 0) + throw new Error("Changeset scanning fixture contains no changeset files"); + const result = await ChangedInstanceIds.initialize({ csFileProps: dataset.csFileProps, iModel: dataset.sourceDb, }); + if (!result) + throw new Error("Changeset scanning did not produce a scan result"); + this._result = result; } public abort(): void { @@ -54,22 +96,10 @@ class ChangesetScanningScenario { public async finish(): Promise { if (this._aborted) throw new Error("Changeset scanning scenario aborted"); - const dataset = requireDetachedDataset(this._dataset); if (!this._result) throw new Error("Changeset scanning scenario finished before measuring"); - - const entries = dataset.recipe as ScanLedgerEntry[] | undefined; - if (!entries) - throw new Error( - `Fixture "${dataset.descriptor.id}" carries no recipe ledger, so the scan result cannot be verified` - ); - - // Verify against what the recipe says it did, not against a previous digest. A digest can only - // detect that behaviour changed; it cannot detect behaviour that was wrong from the start. - assertScanMatchesOracle( - this._result, - squashLedger(ScanLedger.fromEntries(entries)) - ); + if (changedIdCount(this._result) === 0) + throw new Error("Changeset scanning produced no changed instance ids"); return scanDigest(this._result); } } diff --git a/packages/performance-tests/test/quick/tests/unit/scanOracle.test.ts b/packages/performance-tests/test/quick/tests/unit/scanOracle.test.ts deleted file mode 100644 index d1fbf673b..000000000 --- a/packages/performance-tests/test/quick/tests/unit/scanOracle.test.ts +++ /dev/null @@ -1,229 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Bentley Systems, Incorporated. All rights reserved. - * See LICENSE.md in the project root for license terms and full copyright notice. - *--------------------------------------------------------------------------------------------*/ - -import { describe, expect, it } from "vitest"; -import { - assertScanMatchesOracle, - normalizeScanResult, - ScanCollection, - scanCollections, - scanDigest, - ScanLedger, - squashLedger, -} from "../../src/fixtures/validation/scanOracle.js"; - -function ledgerOf( - ...entries: readonly [ - ScanCollection, - "Deleted" | "Inserted" | "Updated", - string, - ][] -): ScanLedger { - const ledger = new ScanLedger(); - for (const [collection, op, id] of entries) ledger.record(collection, op, id); - return ledger; -} - -describe("changeset scan oracle", () => { - it("keeps a repeatedly updated instance in updateIds", () => { - const squashed = squashLedger( - ledgerOf( - ["element", "Updated", "0x20"], - ["element", "Updated", "0x20"], - ["element", "Updated", "0x20"] - ) - ); - expect([...squashed.element.updateIds]).to.deep.equal(["0x20"]); - expect(squashed.element.insertIds.size).to.equal(0); - expect(squashed.element.deleteIds.size).to.equal(0); - }); - - it("turns update-then-delete into a delete", () => { - const squashed = squashLedger( - ledgerOf(["element", "Updated", "0x21"], ["element", "Deleted", "0x21"]) - ); - expect([...squashed.element.deleteIds]).to.deep.equal(["0x21"]); - expect(squashed.element.updateIds.size).to.equal(0); - }); - - it("keeps insert-then-update as an insert", () => { - const squashed = squashLedger( - ledgerOf(["element", "Inserted", "0x22"], ["element", "Updated", "0x22"]) - ); - expect([...squashed.element.insertIds]).to.deep.equal(["0x22"]); - expect(squashed.element.updateIds.size).to.equal(0); - }); - - it("cancels insert-then-delete entirely", () => { - const squashed = squashLedger( - ledgerOf(["element", "Inserted", "0x23"], ["element", "Deleted", "0x23"]) - ); - for (const op of ["insertIds", "updateIds", "deleteIds"] as const) - expect(squashed.element[op].size, op).to.equal(0); - }); - - it("reinstates an insert after a delete", () => { - const squashed = squashLedger( - ledgerOf( - ["relationship", "Deleted", "0x24"], - ["relationship", "Inserted", "0x24"] - ) - ); - expect([...squashed.relationship.insertIds]).to.deep.equal(["0x24"]); - expect(squashed.relationship.deleteIds.size).to.equal(0); - }); - - it("keeps collections independent", () => { - const squashed = squashLedger( - ledgerOf( - ["element", "Inserted", "0x30"], - ["model", "Inserted", "0x30"], - ["aspect", "Deleted", "0x30"], - ["codeSpec", "Updated", "0x30"] - ) - ); - expect([...squashed.element.insertIds]).to.deep.equal(["0x30"]); - expect([...squashed.model.insertIds]).to.deep.equal(["0x30"]); - expect([...squashed.aspect.deleteIds]).to.deep.equal(["0x30"]); - expect([...squashed.codeSpec.updateIds]).to.deep.equal(["0x30"]); - expect(squashed.font.insertIds.size).to.equal(0); - }); - - it("drops repeated inserts and updates without changing the outcome", () => { - const ledger = new ScanLedger(); - for (let index = 0; index < 50; index++) - ledger.record("element", "Updated", "0x40"); - ledger.record("element", "Deleted", "0x40"); - expect(ledger.entries).to.have.length(2); - expect([...squashLedger(ledger).element.deleteIds]).to.deep.equal(["0x40"]); - }); - - it("rejects a repeated delete instead of deduplicating it", () => { - const ledger = new ScanLedger(); - ledger.record("element", "Inserted", "0x41"); - ledger.record("element", "Deleted", "0x41"); - expect(() => ledger.record("element", "Deleted", "0x41")).to.throw( - /second "Deleted"/ - ); - }); - - // Pins the reason the recorder rejects repeated deletes rather than deduplicating them: the - // Deleted branch is conditional on insertIds.has(id), and the first delete falsifies that - // condition. Deduplicating would collapse this to the empty result, which is wrong. - it("does not treat a repeated delete as idempotent", () => { - const insertThenDelete = squashLedger([ - { collection: "element", id: "0x42", op: "Inserted" }, - { collection: "element", id: "0x42", op: "Deleted" }, - ]); - expect(insertThenDelete.element.insertIds.size).to.equal(0); - expect(insertThenDelete.element.deleteIds.size).to.equal(0); - - const withSecondDelete = squashLedger([ - { collection: "element", id: "0x42", op: "Inserted" }, - { collection: "element", id: "0x42", op: "Deleted" }, - { collection: "element", id: "0x42", op: "Deleted" }, - ]); - expect([...withSecondDelete.element.deleteIds]).to.deep.equal(["0x42"]); - }); - - // The transformer's Inserted branch clears deleteIds but not updateIds, and its Updated branch - // guards on insertIds but not deleteIds, so each leaves an id in two collections at once. Neither - // sequence is producible from a changeset, but both are reachable through addCustomElementChange. - // The oracle mirrors the behaviour rather than normalizing it, so that it keeps agreeing with the - it("mirrors the transformer's incomplete reconciliation rather than normalizing it", () => { - const updateThenInsert = squashLedger([ - { collection: "element", id: "0x43", op: "Updated" }, - { collection: "element", id: "0x43", op: "Inserted" }, - ]); - expect([...updateThenInsert.element.insertIds]).to.deep.equal(["0x43"]); - expect([...updateThenInsert.element.updateIds]).to.deep.equal(["0x43"]); - - const deleteThenUpdate = squashLedger([ - { collection: "element", id: "0x44", op: "Deleted" }, - { collection: "element", id: "0x44", op: "Updated" }, - ]); - expect([...deleteThenUpdate.element.updateIds]).to.deep.equal(["0x44"]); - expect([...deleteThenUpdate.element.deleteIds]).to.deep.equal(["0x44"]); - }); - - it("records batches of ids", () => { - const ledger = new ScanLedger(); - ledger.record("aspect", "Inserted", ["0x50", "0x51"]); - expect([...squashLedger(ledger).aspect.insertIds]).to.deep.equal([ - "0x50", - "0x51", - ]); - }); - - it("orders digest ids numerically rather than lexically", () => { - const ledger = new ScanLedger(); - ledger.record("element", "Inserted", ["0x100", "0x9", "0x20"]); - expect( - normalizeScanResult(squashLedger(ledger)).element.insert - ).to.deep.equal(["0x9", "0x20", "0x100"]); - }); - - it("produces an insertion-order-independent digest", () => { - const forward = new ScanLedger(); - forward.record("element", "Inserted", ["0x60", "0x61"]); - const reverse = new ScanLedger(); - reverse.record("element", "Inserted", ["0x61", "0x60"]); - expect(scanDigest(squashLedger(forward))).to.equal( - scanDigest(squashLedger(reverse)) - ); - }); - - it("covers all six ChangedInstanceIds collections", () => { - expect([...scanCollections].sort()).to.deep.equal([ - "aspect", - "codeSpec", - "element", - "font", - "model", - "relationship", - ]); - }); - - it("accepts a scan that matches the recipe", () => { - const expected = squashLedger(ledgerOf(["element", "Updated", "0x70"])); - expect(() => - assertScanMatchesOracle( - squashLedger(ledgerOf(["element", "Updated", "0x70"])), - expected - ) - ).to.not.throw(); - }); - - it("reports missing and unexpected ids per collection", () => { - const expected = squashLedger( - ledgerOf(["aspect", "Updated", "0x80"], ["aspect", "Updated", "0x81"]) - ); - const actual = squashLedger( - ledgerOf(["aspect", "Updated", "0x80"], ["aspect", "Updated", "0x82"]) - ); - let message = ""; - try { - assertScanMatchesOracle(actual, expected); - } catch (error) { - message = (error as Error).message; - } - expect(message).to.match(/aspect\.updateIds/); - expect(message).to.match(/missing=\[0x81\]/); - expect(message).to.match(/unexpected=\[0x82\]/); - }); - - it("fails when a collection outside element regresses", () => { - const expected = squashLedger(ledgerOf(["element", "Updated", "0x90"])); - const actual = squashLedger( - ledgerOf( - ["element", "Updated", "0x90"], - ["relationship", "Deleted", "0x91"] - ) - ); - expect(() => assertScanMatchesOracle(actual, expected)).to.throw( - /relationship\.deleteIds/ - ); - }); -});