diff --git a/.github/workflows/quick-performance.yml b/.github/workflows/quick-performance.yml index 0de88035..84f7202f 100644 --- a/.github/workflows/quick-performance.yml +++ b/.github/workflows/quick-performance.yml @@ -10,6 +10,7 @@ on: type: choice options: - incremental-synchronization + - changeset-scanning fixture: description: Fixture id (blank uses the scenario's default) required: false @@ -78,6 +79,7 @@ jobs: "### Quick performance reliability", `Scenario: **${summary.scenarioId}**`, `Status: **${summary.varianceStatus}**`, + `Transformer: ${summary.versions?.transformer} / core-backend: ${summary.versions?.coreBackend} / node: ${summary.versions?.node}`, `CV: ${summary.wallMilliseconds.coefficientOfVariation}`, `Normalized MAD: ${summary.wallMilliseconds.normalizedMad}`, `Unstable samples: ${summary.unstableSamples.join(", ")}`, diff --git a/packages/performance-tests/test/quick/QuickPerformance.test.ts b/packages/performance-tests/test/quick/QuickPerformance.test.ts index 7f4229ef..ec5c02cd 100644 --- a/packages/performance-tests/test/quick/QuickPerformance.test.ts +++ b/packages/performance-tests/test/quick/QuickPerformance.test.ts @@ -13,33 +13,46 @@ import { BenchmarkRunner } from "./src/framework/BenchmarkRunner.js"; import { scenarioBudgetMilliseconds } from "./src/framework/BenchmarkScenario.js"; import { quickPath } from "./src/support/paths.js"; +/** + * Headroom above the scenario budget so the budget assertion reports the overrun before Vitest + * kills the test. + */ +const budgetHeadroomMilliseconds = 60 * 1000; + describe("quick performance", () => { const { descriptor, scenario } = resolveBenchmarkRunFromEnvironment(); const measuredSamples = resolveMeasuredSamplesFromEnvironment(); const budgetMilliseconds = scenarioBudgetMilliseconds(scenario); - it(`${scenario.id} completes within budget on ${descriptor.id}`, async () => { - const outputDir = - process.env.QUICK_PERF_OUTPUT ?? - quickPath(".quick-output", descriptor.id); - const started = process.hrtime.bigint(); - const samples = await new BenchmarkRunner( - descriptor, - outputDir, - scenario - ).run(measuredSamples); - const elapsedMilliseconds = - Number(process.hrtime.bigint() - started) / 1_000_000; - const summary = BenchmarkReporter.write( - outputDir, - samples, - elapsedMilliseconds - ); + it( + `${scenario.id} completes within budget on ${descriptor.id}`, + async () => { + const outputDir = + process.env.QUICK_PERF_OUTPUT ?? + quickPath(".quick-output", descriptor.id); + const started = process.hrtime.bigint(); + const samples = await new BenchmarkRunner( + descriptor, + outputDir, + scenario + ).run(measuredSamples); + const elapsedMilliseconds = + Number(process.hrtime.bigint() - started) / 1_000_000; + const summary = BenchmarkReporter.write( + outputDir, + samples, + elapsedMilliseconds + ); - expect(summary.measuredSamples).to.equal(measuredSamples); - expect( - new Set(samples.map((sample) => sample.semanticDigest)).size - ).to.equal(1, "every sample must observe the same fixture"); - expect(elapsedMilliseconds).to.be.lessThan(budgetMilliseconds); - }); + expect(summary.measuredSamples).to.equal(measuredSamples); + expect( + new Set(samples.map((sample) => sample.semanticDigest)).size + ).to.equal(1, "every sample must observe the same fixture"); + expect(new Set(samples.map((sample) => sample.scenarioId))).to.deep.equal( + new Set([scenario.id]) + ); + expect(elapsedMilliseconds).to.be.lessThan(budgetMilliseconds); + }, + budgetMilliseconds + budgetHeadroomMilliseconds + ); }); diff --git a/packages/performance-tests/test/quick/assets/schemas/QuickPerfScan.ecschema.xml b/packages/performance-tests/test/quick/assets/schemas/QuickPerfScan.ecschema.xml new file mode 100644 index 00000000..bd043ea9 --- /dev/null +++ b/packages/performance-tests/test/quick/assets/schemas/QuickPerfScan.ecschema.xml @@ -0,0 +1,11 @@ + + + + + + bis:ElementMultiAspect + + + + diff --git a/packages/performance-tests/test/quick/src/catalogs/FixtureCatalog.ts b/packages/performance-tests/test/quick/src/catalogs/FixtureCatalog.ts index 4c64d336..9428736f 100644 --- a/packages/performance-tests/test/quick/src/catalogs/FixtureCatalog.ts +++ b/packages/performance-tests/test/quick/src/catalogs/FixtureCatalog.ts @@ -4,15 +4,14 @@ *--------------------------------------------------------------------------------------------*/ import * as fs from "node:fs"; -import { createRequire } from "node:module"; import * as path from "node:path"; import { canonicalSha256, FixtureDescriptor, } from "../fixtures/FixtureDescriptor.js"; import { quickPath, quickRootDirectory } from "../support/paths.js"; +import { resolvedVersions } from "../support/versions.js"; -const localRequire = createRequire(import.meta.url); const scale = 25; const distribution = { base: { @@ -42,18 +41,7 @@ const distribution = { }, } as const; -function packageVersion(packageName: string): string { - const packageJson = JSON.parse( - fs.readFileSync(localRequire.resolve(`${packageName}/package.json`), "utf8") - ) as { version: string }; - return packageJson.version; -} - -const generator = { - coreBackend: packageVersion("@itwin/core-backend"), - node: process.version, - transformer: packageVersion("@itwin/imodel-transformer"), -}; +const generator = resolvedVersions(); const recipeIdentity = (topology: string) => ({ schema: "QuickPerf.01.00.00", @@ -129,12 +117,108 @@ export const balancedIncrementalSourceOnlyDescriptor: FixtureDescriptor = { recipeHash: canonicalSha256(recipeIdentity("source-only")), }; +/** + * Update-heavy multi-changeset source, sized so that scanning it dominates timer noise. + * + * Region sizes are derived from `base.elements` by `scanRegionSizes`; see + * `fixtures/recipes/updateHeavyScan` for what each region proves. + * + * Calibrated at 3,520 elements x 20 changesets: the scan measures ~3.46 s with a coefficient of + * variation of 1.0% over 8 samples, against ~51 ms of per-sample copy, verification and teardown. + * Scan cost is linear in changed rows at roughly 49 ms per (1,000 elements x changeset), so the + * shape is a single `scanScale` knob. + */ +const scanScale = 16; +const scanDistribution = { + base: { + // Region A (updated throughout) plus region B (updated, then deleted last). + aspects: 220 * scanScale, + elements: 220 * scanScale, + geometricElements: 0, + relationships: 40 * scanScale, + }, + operations: { + elements: { + // Regions C and D, both inserted in the first changeset. + inserts: 30 * scanScale, + // Every seeded element is updated, plus region C after its insert. + updates: 240 * scanScale, + // Region B at the end, region D at the end. + deletes: 30 * scanScale, + }, + aspects: { + inserts: 20 * scanScale, + updates: 220 * scanScale, + // Region B's owned aspects, cascade-deleted with their elements. + deletes: 20 * scanScale, + }, + relationships: { + inserts: 10 * scanScale, + updates: 10 * scanScale, + deletes: 10 * scanScale, + }, + geometryUpdates: 0, + sourceChangesets: 20, + }, +} as const; + +const scanRecipeIdentity = { + schema: "QuickPerfScan.01.00.00", + seed: 328, + topology: "source-only", + distribution: scanDistribution, + inputs: { + recipe: fs.readFileSync( + quickPath("src", "fixtures", "recipes", "updateHeavyScan.ts"), + "utf8" + ), + schema: fs.readFileSync( + quickPath("assets", "schemas", "QuickPerfScan.ecschema.xml"), + "utf8" + ), + lockfile: fs.readFileSync( + path.resolve( + quickRootDirectory, + "..", + "..", + "..", + "..", + "pnpm-lock.yaml" + ), + "utf8" + ), + }, + versions: generator, +}; + +export const updateHeavyScanDescriptor: FixtureDescriptor = { + id: "update-heavy-scan", + version: 1, + label: "update-heavy scan", + scenarioClaims: [ + "changeset scanning", + "changed-instance squashing", + "aspect lifecycle", + "relationship lifecycle", + ], + layout: { + kind: "reconstructed", + topology: "source-only", + recipe: "update-heavy-scan", + seed: 328, + }, + distribution: scanDistribution, + generator, + recipeHash: canonicalSha256(scanRecipeIdentity), +}; + const fixtures = new Map([ [balancedIncrementalDescriptor.id, balancedIncrementalDescriptor], [ balancedIncrementalSourceOnlyDescriptor.id, balancedIncrementalSourceOnlyDescriptor, ], + [updateHeavyScanDescriptor.id, updateHeavyScanDescriptor], ]); export function registerFixtureDescriptor(descriptor: FixtureDescriptor): void { diff --git a/packages/performance-tests/test/quick/src/catalogs/ScenarioCatalog.ts b/packages/performance-tests/test/quick/src/catalogs/ScenarioCatalog.ts index b82d2b36..d81af203 100644 --- a/packages/performance-tests/test/quick/src/catalogs/ScenarioCatalog.ts +++ b/packages/performance-tests/test/quick/src/catalogs/ScenarioCatalog.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { BenchmarkScenarioDefinition } from "../framework/BenchmarkScenario.js"; +import { changesetScanningScenario } from "../scenarios/changesetScanning.js"; import { incrementalSynchronizationScenario } from "../scenarios/incrementalSynchronization.js"; export const defaultQuickPerformanceScenarioId = @@ -11,6 +12,7 @@ export const defaultQuickPerformanceScenarioId = const scenarios = new Map([ [incrementalSynchronizationScenario.id, incrementalSynchronizationScenario], + [changesetScanningScenario.id, changesetScanningScenario], ]); export function registerScenarioDefinition( diff --git a/packages/performance-tests/test/quick/src/fixtures/FixtureRecipe.ts b/packages/performance-tests/test/quick/src/fixtures/FixtureRecipe.ts index 7f9ac92b..35e91b04 100644 --- a/packages/performance-tests/test/quick/src/fixtures/FixtureRecipe.ts +++ b/packages/performance-tests/test/quick/src/fixtures/FixtureRecipe.ts @@ -11,6 +11,13 @@ import { BalancedRecipeState, createBalancedSeed, } from "./recipes/balancedIncremental.js"; +import { + applyScanChangesets, + createScanSeed, + ScanRecipeState, + validateScanFixture, +} from "./recipes/updateHeavyScan.js"; +import { assertFixtureDistribution } from "./validation/validateFixture.js"; /** * A recipe produces the *change mix* for a fixture: it seeds the source iModel and then applies a @@ -40,6 +47,12 @@ export interface FixtureRecipe { descriptor: FixtureDescriptor, state: TState ): Promise; + /** + * Assert the built source iModel matches what the descriptor promises. + * + * Validation is recipe-owned because it queries the classes that recipe created. + */ + validate(db: BriefcaseDb, descriptor: FixtureDescriptor): Promise; } export const balancedIncrementalRecipe: FixtureRecipe = { @@ -48,10 +61,21 @@ export const balancedIncrementalRecipe: FixtureRecipe = { createBalancedSeed(fileName, descriptor), applySourceChangesets: async (db, accessToken, descriptor, state) => applyBalancedChangesets(db, accessToken, descriptor, state), + validate: async (db, descriptor) => assertFixtureDistribution(db, descriptor), +}; + +export const updateHeavyScanRecipe: FixtureRecipe = { + id: "update-heavy-scan", + createSeed: async (fileName, descriptor) => + createScanSeed(fileName, descriptor), + applySourceChangesets: async (db, accessToken, descriptor, state) => + applyScanChangesets(db, accessToken, descriptor, state), + validate: async (db, descriptor) => validateScanFixture(db, descriptor), }; const recipes = new Map>([ [balancedIncrementalRecipe.id, balancedIncrementalRecipe], + [updateHeavyScanRecipe.id, updateHeavyScanRecipe], ]); export function registerFixtureRecipe(recipe: FixtureRecipe): void { diff --git a/packages/performance-tests/test/quick/src/fixtures/providers/detachedBriefcaseProvider.ts b/packages/performance-tests/test/quick/src/fixtures/providers/detachedBriefcaseProvider.ts index 74ecbc4c..5649929f 100644 --- a/packages/performance-tests/test/quick/src/fixtures/providers/detachedBriefcaseProvider.ts +++ b/packages/performance-tests/test/quick/src/fixtures/providers/detachedBriefcaseProvider.ts @@ -35,7 +35,6 @@ import { reconstructSourceHub, shutdownHubMock, } from "../LocalHubFixture.js"; -import { assertFixtureDistribution } from "../validation/validateFixture.js"; /** * Tolerant teardown for a build that may have failed at any point. Collects errors rather than @@ -118,7 +117,7 @@ export const detachedBriefcaseFixtureProvider: FixtureProvider = { descriptor, recipeState ); - await assertFixtureDistribution(hub.sourceDb, descriptor); + await recipe.validate(hub.sourceDb, descriptor); fs.mkdirSync(scratchDir, { recursive: true }); const downloaded = await BriefcaseManager.downloadChangesets({ diff --git a/packages/performance-tests/test/quick/src/fixtures/providers/liveHubProvider.ts b/packages/performance-tests/test/quick/src/fixtures/providers/liveHubProvider.ts index c1ed86ac..c52b6160 100644 --- a/packages/performance-tests/test/quick/src/fixtures/providers/liveHubProvider.ts +++ b/packages/performance-tests/test/quick/src/fixtures/providers/liveHubProvider.ts @@ -18,7 +18,6 @@ import { ReconstructedHub, reconstructHub, } from "../LocalHubFixture.js"; -import { assertFixtureDistribution } from "../validation/validateFixture.js"; /** * The `source-and-empty-target` topology. @@ -73,7 +72,7 @@ export const liveHubFixtureProvider: FixtureProvider = { descriptor, recipeState ); - await assertFixtureDistribution(hub.sourceDb, descriptor); + await recipe.validate(hub.sourceDb, descriptor); return { topology: "source-and-empty-target", descriptor, diff --git a/packages/performance-tests/test/quick/src/fixtures/recipes/updateHeavyScan.ts b/packages/performance-tests/test/quick/src/fixtures/recipes/updateHeavyScan.ts new file mode 100644 index 00000000..eefb512c --- /dev/null +++ b/packages/performance-tests/test/quick/src/fixtures/recipes/updateHeavyScan.ts @@ -0,0 +1,494 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { AccessToken, Id64String } from "@itwin/core-bentley"; +import { + Code, + CodeScopeSpec, + CodeSpec, + ElementAspectProps, + IModel, + PhysicalElementProps, +} from "@itwin/core-common"; +import { + BriefcaseDb, + EditTxn, + ElementGroupsMembers, + ElementOwnsMultiAspects, + PhysicalModel, + PhysicalObject, + SnapshotDb, + SpatialCategory, + withEditTxn, +} from "@itwin/core-backend"; +import { FixtureDescriptor } from "../FixtureDescriptor.js"; +import { queryCount } from "../validation/validateFixture.js"; +import { quickPath } from "../../support/paths.js"; + +const scanAspectClass = "QuickPerfScan:ScanAspect"; + +/** + * Per-unit shape of the update-heavy scan recipe. + * + * Elements are partitioned into four regions so that every id has one unambiguous expected outcome + * after `ChangedInstanceIds` squashes the whole scanned range: + * + * - `updated` updated in every changeset -> `element.updateIds` + * - `deletedLate` updated, then deleted in the last -> `element.deleteIds` + * - `insertedThenUpdated` inserted first, updated afterwards -> `element.insertIds` + * - `insertedThenDeleted` inserted first, deleted in the last -> absent from all three sets + * + * The last region is the reason this recipe can catch squash regressions that a "the sizes match" + * assertion cannot: those ids must cancel out entirely. + * + * Note there is no region for the fourth squash rule, delete-then-insert reinstating an insert. + * That rule is encoded in the oracle and unit-tested directly, but no region exercises it against + * a real changeset because it is not producible here: ids are assigned by the briefcase and a + * deleted ElementId, aspect id or relationship id is never handed out again, so a delete can never + * be followed by an insert of the same id within one scanned range. + * + * Elements carry no geometry. An unchanged GeometryStream column never appears in a SQLite + * changeset, so geometry would cost build time, disk and determinism while contributing nothing to + * scan cost. This recipe is tuned as a sensitive regression detector, not as a representative + * production workload; answering the representativeness question is a separate, geometry-bearing + * recipe. + */ +export interface ScanRegionSizes { + readonly updated: number; + readonly deletedLate: number; + readonly insertedThenUpdated: number; + readonly insertedThenDeleted: number; + readonly seedRelationships: number; + readonly insertedRelationships: number; + readonly updatedRelationships: number; + readonly deletedRelationships: number; +} + +const perUnit: ScanRegionSizes = { + deletedLate: 20, + deletedRelationships: 10, + insertedRelationships: 10, + insertedThenDeleted: 10, + insertedThenUpdated: 20, + seedRelationships: 40, + updated: 200, + updatedRelationships: 10, +}; + +/** Seed elements are regions A and B; the other two regions are created by the changesets. */ +const seedElementsPerUnit = perUnit.updated + perUnit.deletedLate; + +/** Fewer than four changesets collapses the schedule's distinct first/middle/penultimate/last phases. */ +const minimumChangesets = 4; + +export function scanRegionSizes( + descriptor: FixtureDescriptor +): ScanRegionSizes { + const scale = descriptor.distribution.base.elements / seedElementsPerUnit; + if (!Number.isInteger(scale) || scale < 1) + throw new Error( + `Scan fixture elements must be a positive multiple of ${seedElementsPerUnit}` + ); + if (descriptor.distribution.operations.sourceChangesets < minimumChangesets) + throw new Error( + `Scan fixture needs at least ${minimumChangesets} source changesets` + ); + return Object.fromEntries( + Object.entries(perUnit).map(([key, value]) => [key, value * scale]) + ) as unknown as ScanRegionSizes; +} + +export interface ScanRecipeState { + readonly categoryId: Id64String; + readonly modelId: Id64String; + /** Region A: updated in every changeset. */ + readonly updatedIds: readonly Id64String[]; + /** Region B: updated throughout, then deleted in the final changeset. */ + readonly deletedLateIds: readonly Id64String[]; + /** One owned multi-aspect per region A element, in the same order. */ + readonly updatedAspectIds: readonly Id64String[]; + /** One owned multi-aspect per region B element, in the same order. */ + readonly deletedLateAspectIds: readonly Id64String[]; + readonly relationshipIds: readonly Id64String[]; +} + +function scanElementProps( + modelId: Id64String, + categoryId: Id64String, + region: string, + index: number, + label: string +): PhysicalElementProps { + return { + category: categoryId, + classFullName: PhysicalObject.classFullName, + code: new Code({ + scope: IModel.rootSubjectId, + spec: IModel.rootSubjectId, + value: `scan-${region}-${index}`, + }), + model: modelId, + userLabel: label, + }; +} + +function scanAspectProps( + ownerId: Id64String, + sequence: number, + payload: string, + aspectId?: Id64String +): ElementAspectProps { + return { + classFullName: scanAspectClass, + element: new ElementOwnsMultiAspects(ownerId), + id: aspectId, + payload, + sequence, + } as ElementAspectProps; +} + +export async function createScanSeed( + fileName: string, + descriptor: FixtureDescriptor +): Promise { + const sizes = scanRegionSizes(descriptor); + const db = SnapshotDb.createEmpty(fileName, { + rootSubject: { name: descriptor.id }, + }); + try { + await db.importSchemas([ + quickPath("assets", "schemas", "QuickPerfScan.ecschema.xml"), + ]); + + const { categoryId, modelId } = withEditTxn( + db, + "create scan model and category", + (txn) => ({ + categoryId: SpatialCategory.insert( + txn, + IModel.dictionaryId, + "ScanCategory", + {} + ), + modelId: PhysicalModel.insert(txn, IModel.rootSubjectId, "ScanModel"), + }) + ); + + const insertRegion = (region: string, count: number) => + withEditTxn(db, `insert scan region ${region}`, (txn) => { + const elementIds: Id64String[] = []; + const aspectIds: Id64String[] = []; + for (let index = 0; index < count; index++) { + const elementId = txn.insertElement( + scanElementProps( + modelId, + categoryId, + region, + index, + `${region}-base-${index}` + ) + ); + elementIds.push(elementId); + aspectIds.push( + txn.insertAspect( + scanAspectProps(elementId, index, `${region}-aspect-${index}`) + ) + ); + } + return { aspectIds, elementIds }; + }); + + const regionA = insertRegion("a", sizes.updated); + const regionB = insertRegion("b", sizes.deletedLate); + + // Relationships live entirely within region A, which is never deleted, so no relationship is + // ever removed by an element cascade. Every relationship delete in the ledger is explicit. + const relationshipIds = withEditTxn( + db, + "insert scan relationships", + (txn) => + Array.from({ length: sizes.seedRelationships }, (_, index) => + txn.insertRelationship( + ElementGroupsMembers.create( + db, + regionA.elementIds[index], + regionA.elementIds[index + 1], + index + ).toJSON() + ) + ) + ); + + return { + categoryId, + deletedLateAspectIds: regionB.aspectIds, + deletedLateIds: regionB.elementIds, + modelId, + relationshipIds, + updatedAspectIds: regionA.aspectIds, + updatedIds: regionA.elementIds, + }; + } finally { + db.close(); + } +} + +/** + * Runs the changeset schedule and returns the ledger of every operation performed. + * + * The schedule is: + * - changeset 1: structural inserts (regions C and D, a model, a CodeSpec, relationships) + * plus the first round of region A/B updates + * - changesets 2..n-2: region A/B/C updates + * - changeset n-1: the same updates plus relationship updates and deletes + * - changeset n: region A/C updates, then region B and D element deletes + */ +export async function applyScanChangesets( + db: BriefcaseDb, + accessToken: AccessToken, + descriptor: FixtureDescriptor, + state: ScanRecipeState +): Promise { + const sizes = scanRegionSizes(descriptor); + const changesetCount = descriptor.distribution.operations.sourceChangesets; + + const insertedThenUpdatedIds: Id64String[] = []; + const insertedThenDeletedIds: Id64String[] = []; + + const updateElements = ( + txn: EditTxn, + ids: readonly Id64String[], + region: string, + changeset: number + ) => { + ids.forEach((id, index) => + txn.updateElement({ + ...scanElementProps( + state.modelId, + state.categoryId, + region, + index, + `${region}-updated-${changeset}-${index}` + ), + id, + } as PhysicalElementProps) + ); + }; + + const updateAspects = ( + txn: EditTxn, + ownerIds: readonly Id64String[], + aspectIds: readonly Id64String[], + region: string, + changeset: number + ) => { + aspectIds.forEach((aspectId, index) => + txn.updateAspect( + scanAspectProps( + ownerIds[index], + index, + `${region}-aspect-${changeset}-${index}`, + aspectId + ) + ) + ); + }; + + for (let changeset = 1; changeset <= changesetCount; changeset++) { + const isFirst = changeset === 1; + const isLast = changeset === changesetCount; + const isPenultimate = changeset === changesetCount - 1; + + withEditTxn(db, `scan changeset ${changeset}`, (txn) => { + if (isFirst) { + for (let index = 0; index < sizes.insertedThenUpdated; index++) { + const elementId = txn.insertElement( + scanElementProps( + state.modelId, + state.categoryId, + "c", + index, + `c-inserted-${index}` + ) + ); + insertedThenUpdatedIds.push(elementId); + txn.insertAspect( + scanAspectProps(elementId, index, `c-aspect-${index}`) + ); + } + + // Region D carries no aspects. Whether cascade-deleting an aspect that was inserted within + // the same scanned range emits a row at all is a second unknown, and this region already + // exists to test one thing: that insert-then-delete cancels. + for (let index = 0; index < sizes.insertedThenDeleted; index++) + insertedThenDeletedIds.push( + txn.insertElement( + scanElementProps( + state.modelId, + state.categoryId, + "d", + index, + `d-inserted-${index}` + ) + ) + ); + PhysicalModel.insert(txn, IModel.rootSubjectId, "ScanExtraModel"); + + db.codeSpecs.insert( + txn, + CodeSpec.create(db, "ScanCodeSpec", CodeScopeSpec.Type.Repository) + ); + + for (let index = 0; index < sizes.insertedRelationships; index++) + txn.insertRelationship( + ElementGroupsMembers.create( + db, + state.updatedIds[index], + state.updatedIds[index + sizes.seedRelationships + 1], + index + ).toJSON() + ); + } + + updateElements(txn, state.updatedIds, "a", changeset); + updateAspects( + txn, + state.updatedIds, + state.updatedAspectIds, + "a", + changeset + ); + + if (!isLast) { + updateElements(txn, state.deletedLateIds, "b", changeset); + updateAspects( + txn, + state.deletedLateIds, + state.deletedLateAspectIds, + "b", + changeset + ); + } + + if (!isFirst) updateElements(txn, insertedThenUpdatedIds, "c", changeset); + + if (isPenultimate) { + const updated = state.relationshipIds.slice( + 0, + sizes.updatedRelationships + ); + for (const relationshipId of updated) { + const relationship = + db.relationships.getInstance( + ElementGroupsMembers.classFullName, + relationshipId + ); + relationship.memberPriority += 1000; + txn.updateRelationship(relationship.toJSON()); + } + const deleted = state.relationshipIds.slice( + sizes.updatedRelationships, + sizes.updatedRelationships + sizes.deletedRelationships + ); + for (const relationshipId of deleted) + txn.deleteRelationship( + db.relationships + .getInstance( + ElementGroupsMembers.classFullName, + relationshipId + ) + .toJSON() + ); + } + + if (isLast) { + txn.deleteElement([...state.deletedLateIds]); + txn.deleteElement(insertedThenDeletedIds); + } + }); + + await db.pushChanges({ + accessToken, + description: `scan changeset ${changeset} of ${changesetCount}`, + }); + } +} + +/** + * Assert the built source iModel matches the region structure the recipe promised. + * + * This checks the tip state, which is what a copied briefcase actually contains. It is deliberately + * expressed in terms of regions rather than raw totals: a regression that deleted the wrong region + * would still produce plausible totals, but cannot produce the right per-region counts. + */ +export async function validateScanFixture( + db: BriefcaseDb, + descriptor: FixtureDescriptor +): Promise { + const sizes = scanRegionSizes(descriptor); + const regionCount = async (region: string) => + queryCount( + db, + `SELECT count(*) cnt FROM ${PhysicalObject.classFullName.replace( + ":", + "." + )} WHERE CodeValue LIKE 'scan-${region}-%'` + ); + + const expectations: Array<[string, number, number]> = [ + [ + "region A elements (updated throughout)", + await regionCount("a"), + sizes.updated, + ], + [ + "region B elements (deleted in the last changeset)", + await regionCount("b"), + 0, + ], + [ + "region C elements (inserted then updated)", + await regionCount("c"), + sizes.insertedThenUpdated, + ], + ["region D elements (inserted then deleted)", await regionCount("d"), 0], + [ + "scan aspects", + await queryCount(db, "SELECT count(*) cnt FROM QuickPerfScan.ScanAspect"), + sizes.updated + sizes.insertedThenUpdated, + ], + [ + "relationships", + await queryCount( + db, + "SELECT count(*) cnt FROM BisCore.ElementGroupsMembers" + ), + sizes.seedRelationships + + sizes.insertedRelationships - + sizes.deletedRelationships, + ], + [ + "updated relationships", + await queryCount( + db, + "SELECT count(*) cnt FROM BisCore.ElementGroupsMembers WHERE MemberPriority >= 1000" + ), + sizes.updatedRelationships, + ], + ]; + + const failures = expectations + .filter(([, actual, expected]) => actual !== expected) + .map( + ([what, actual, expected]) => + `${what}: expected ${expected}, got ${actual}` + ); + if (failures.length > 0) + throw new Error( + `Fixture "${descriptor.id}" does not match its recipe:\n ${failures.join( + "\n " + )}` + ); +} diff --git a/packages/performance-tests/test/quick/src/fixtures/validation/validateFixture.ts b/packages/performance-tests/test/quick/src/fixtures/validation/validateFixture.ts index f849743c..5be35a34 100644 --- a/packages/performance-tests/test/quick/src/fixtures/validation/validateFixture.ts +++ b/packages/performance-tests/test/quick/src/fixtures/validation/validateFixture.ts @@ -107,7 +107,10 @@ async function queryGeometryRecords(db: BriefcaseDb): Promise { return values; } -async function queryCount(db: BriefcaseDb, ecsql: string): Promise { +export async function queryCount( + db: BriefcaseDb, + ecsql: string +): Promise { const reader = db.createQueryReader(ecsql, undefined, { usePrimaryConn: true, }); diff --git a/packages/performance-tests/test/quick/src/framework/BenchmarkScenario.ts b/packages/performance-tests/test/quick/src/framework/BenchmarkScenario.ts index 03fe307d..b700680c 100644 --- a/packages/performance-tests/test/quick/src/framework/BenchmarkScenario.ts +++ b/packages/performance-tests/test/quick/src/framework/BenchmarkScenario.ts @@ -32,14 +32,23 @@ export interface BenchmarkScenarioDefinition { readonly defaultFixtureId: string; readonly capabilities: BenchmarkScenarioCapabilities; readonly factory: BenchmarkScenarioFactory; - /** Wall-clock budget for the whole run, in milliseconds. */ + /** + * Wall time allowed for the measured run, excluding checkout, install and build. + * Defaults to {@link defaultScenarioBudgetMilliseconds}. + */ readonly budgetMilliseconds?: number; } export const defaultScenarioBudgetMilliseconds = 15 * 60 * 1000; export function scenarioBudgetMilliseconds( - scenario: BenchmarkScenarioDefinition + definition: BenchmarkScenarioDefinition ): number { - return scenario.budgetMilliseconds ?? defaultScenarioBudgetMilliseconds; + const budget = + definition.budgetMilliseconds ?? defaultScenarioBudgetMilliseconds; + if (!Number.isFinite(budget) || budget <= 0) + throw new Error( + `Scenario "${definition.id}" declares an invalid budget: ${budget}` + ); + return budget; } diff --git a/packages/performance-tests/test/quick/src/reporting/BenchmarkReporter.ts b/packages/performance-tests/test/quick/src/reporting/BenchmarkReporter.ts index 35b5c456..2b584822 100644 --- a/packages/performance-tests/test/quick/src/reporting/BenchmarkReporter.ts +++ b/packages/performance-tests/test/quick/src/reporting/BenchmarkReporter.ts @@ -12,6 +12,7 @@ import { medianAbsoluteDeviation, percentile, } from "./statistics.js"; +import { resolvedVersions } from "../support/versions.js"; export const maximumCoefficientOfVariation = 0.05; export const maximumNormalizedMad = 0.05; @@ -88,6 +89,7 @@ export class BenchmarkReporter { measuredSamples: measured.length, reportSchemaVersion: identity.reportSchemaVersion, scenarioId: samples[0].scenarioId, + versions: resolvedVersions(), varianceStatus: classifyVariance( measured.length, wallCoefficientOfVariation, @@ -133,7 +135,7 @@ export class BenchmarkReporter { fs.writeFileSync( path.join(outputDir, "summary.csv"), [ - "reportSchemaVersion,scenario,fixture,fixtureVersion,fixtureRecipeHash,nodeVersion,coreBackendVersion,transformerVersion,measuredSamples,jobMs,fixtureBuildMs,medianMs,p90Ms,p95Ms,madMs,cv,reconstructionTotalMs,verificationTotalMs,teardownTotalMs", + "reportSchemaVersion,scenario,fixture,fixtureVersion,fixtureRecipeHash,fixtureNodeVersion,fixtureCoreBackendVersion,fixtureTransformerVersion,measuringNodeVersion,measuringCoreBackendVersion,measuringTransformerVersion,measuredSamples,jobMs,fixtureBuildMs,medianMs,p90Ms,p95Ms,madMs,cv,reconstructionTotalMs,verificationTotalMs,teardownTotalMs", [ summary.reportSchemaVersion, summary.scenarioId, @@ -143,6 +145,9 @@ export class BenchmarkReporter { summary.fixtureGenerator.node, summary.fixtureGenerator.coreBackend, summary.fixtureGenerator.transformer, + summary.versions.node, + summary.versions.coreBackend, + summary.versions.transformer, summary.measuredSamples, summary.jobMilliseconds ?? "", summary.fixtureBuildMilliseconds, diff --git a/packages/performance-tests/test/quick/src/scenarios/changesetScanning.ts b/packages/performance-tests/test/quick/src/scenarios/changesetScanning.ts new file mode 100644 index 00000000..1838db88 --- /dev/null +++ b/packages/performance-tests/test/quick/src/scenarios/changesetScanning.ts @@ -0,0 +1,115 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { ChangedInstanceIds } from "@itwin/imodel-transformer"; +import { updateHeavyScanDescriptor } from "../catalogs/FixtureCatalog.js"; +import { canonicalSha256 } from "../fixtures/FixtureDescriptor.js"; +import { + PreparedDataset, + requireDetachedDataset, +} from "../fixtures/FixtureProvider.js"; +import { BenchmarkScenarioDefinition } from "../framework/BenchmarkScenario.js"; + +type ScanResult = NonNullable< + Awaited> +>; + +function scanDigest(result: ScanResult): string { + const collections = { + aspect: result.aspect, + codeSpec: result.codeSpec, + element: result.element, + font: result.font, + model: result.model, + relationship: result.relationship, + }; + const normalized = Object.fromEntries( + Object.entries(collections).map(([collection, operations]) => [ + collection, + { + delete: [...operations.deleteIds].sort(), + insert: [...operations.insertIds].sort(), + update: [...operations.updateIds].sort(), + }, + ]) + ); + return canonicalSha256(normalized); +} + +function changedIdCount(result: ScanResult): number { + return [ + result.aspect, + result.codeSpec, + result.element, + result.font, + result.model, + result.relationship, + ].reduce( + (total, operations) => + total + + operations.insertIds.size + + operations.updateIds.size + + operations.deleteIds.size, + 0 + ); +} + +/** + * Measures {@link ChangedInstanceIds.initialize} over a set of local changeset files. + * + * The measured region is deliberately narrow. `IModelTransformer.initialize` also performs + * provenance initialization, clone-context and exporter setup and deleted-entity remapping, so a + * regression in any of those would surface here as a scanning regression, and — more importantly — + * a scanning regression would be diluted by everything else. Scanning is roughly a third of that + * larger region, so at the ~3% run-to-run variation this framework sees, a 20% scan regression + * would move the total by about 6% and disappear into the noise. Narrowing the region is what makes + * the regression detectable. + * + * `csFileProps` is the only one of the four `initialize` input modes that touches no hub: the other + * three re-enter `BriefcaseManager` to query or download changesets. That is why this scenario needs + * no hub and no target iModel, and why `HubMock` is deliberately not running during measurement. + */ +class ChangesetScanningScenario { + private _result?: ScanResult; + private _aborted = false; + + constructor(private readonly _dataset: PreparedDataset) {} + + public async measure(): Promise { + const dataset = requireDetachedDataset(this._dataset); + if (dataset.csFileProps.length === 0) + throw new Error("Changeset scanning fixture contains no changeset files"); + const result = await ChangedInstanceIds.initialize({ + csFileProps: dataset.csFileProps, + iModel: dataset.sourceDb, + }); + if (!result) + throw new Error("Changeset scanning did not produce a scan result"); + this._result = result; + } + + public abort(): void { + this._aborted = true; + } + + public async finish(): Promise { + if (this._aborted) throw new Error("Changeset scanning scenario aborted"); + if (!this._result) + throw new Error("Changeset scanning scenario finished before measuring"); + if (changedIdCount(this._result) === 0) + throw new Error("Changeset scanning produced no changed instance ids"); + return scanDigest(this._result); + } +} + +export const changesetScanningScenario: BenchmarkScenarioDefinition = { + id: "changeset-scanning", + defaultFixtureId: updateHeavyScanDescriptor.id, + capabilities: { + topology: "source-only", + requiredClaims: ["changeset scanning"], + }, + factory: (dataset) => new ChangesetScanningScenario(dataset), +}; diff --git a/packages/performance-tests/test/quick/src/support/versions.ts b/packages/performance-tests/test/quick/src/support/versions.ts new file mode 100644 index 00000000..5c6b130a --- /dev/null +++ b/packages/performance-tests/test/quick/src/support/versions.ts @@ -0,0 +1,34 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "node:fs"; +import { createRequire } from "node:module"; + +const localRequire = createRequire(import.meta.url); + +export interface ResolvedVersions { + readonly coreBackend: string; + readonly node: string; + readonly transformer: string; +} + +export function packageVersion(packageName: string): string { + const packageJson = JSON.parse( + fs.readFileSync(localRequire.resolve(`${packageName}/package.json`), "utf8") + ) as { version: string }; + return packageJson.version; +} + +/** + * Versions actually loaded by the measuring process. A fixture descriptor records the versions that + * *generated* it, which are not necessarily the versions that measure it. + */ +export function resolvedVersions(): ResolvedVersions { + return { + coreBackend: packageVersion("@itwin/core-backend"), + node: process.version, + transformer: packageVersion("@itwin/imodel-transformer"), + }; +} diff --git a/packages/performance-tests/test/quick/tests/integration/FixtureArtifact.test.ts b/packages/performance-tests/test/quick/tests/integration/FixtureArtifact.test.ts index 56248d21..f90b7a2e 100644 --- a/packages/performance-tests/test/quick/tests/integration/FixtureArtifact.test.ts +++ b/packages/performance-tests/test/quick/tests/integration/FixtureArtifact.test.ts @@ -359,6 +359,8 @@ describe("recipe data across the stage boundary", () => { ); return emitted; }, + validate: async (db, forDescriptor) => + balancedIncrementalRecipe.validate(db, forDescriptor), }); root = fs.mkdtempSync(path.join(os.tmpdir(), "quick-recipe-data-")); await startIsolatedHost(); @@ -442,6 +444,8 @@ describe("recipe data that cannot survive JSON", () => { ); return { elements: { deleteIds: new Set(["0x20", "0x21"]) } }; }, + validate: async (db, forDescriptor) => + balancedIncrementalRecipe.validate(db, forDescriptor), }); root = fs.mkdtempSync(path.join(os.tmpdir(), "quick-bad-recipe-")); await startIsolatedHost(); diff --git a/packages/performance-tests/test/quick/tests/unit/BenchmarkResolution.test.ts b/packages/performance-tests/test/quick/tests/unit/BenchmarkResolution.test.ts index 959acb31..6b368678 100644 --- a/packages/performance-tests/test/quick/tests/unit/BenchmarkResolution.test.ts +++ b/packages/performance-tests/test/quick/tests/unit/BenchmarkResolution.test.ts @@ -16,7 +16,9 @@ import { BenchmarkScenarioDefinition } from "../../src/framework/BenchmarkScenar import { balancedIncrementalDescriptor, balancedIncrementalSourceOnlyDescriptor, + updateHeavyScanDescriptor, } from "../../src/catalogs/FixtureCatalog.js"; +import { changesetScanningScenario } from "../../src/scenarios/changesetScanning.js"; import { incrementalSynchronizationScenario } from "../../src/scenarios/incrementalSynchronization.js"; describe("benchmark resolution", () => { @@ -28,6 +30,14 @@ describe("benchmark resolution", () => { ); }); + it("resolves the changeset-scanning fixture and capabilities", () => { + const resolved = resolveBenchmarkRun(changesetScanningScenario.id); + expect(resolved.descriptor).to.equal(updateHeavyScanDescriptor); + expect(() => + assertScenarioSupportsFixture(resolved.scenario, resolved.descriptor) + ).to.not.throw(); + }); + it("lets an explicit fixture id override the default", () => { const scenario: BenchmarkScenarioDefinition = { ...incrementalSynchronizationScenario, diff --git a/packages/performance-tests/test/quick/tests/unit/ScenarioCatalog.test.ts b/packages/performance-tests/test/quick/tests/unit/ScenarioCatalog.test.ts index 1686012a..991b72e5 100644 --- a/packages/performance-tests/test/quick/tests/unit/ScenarioCatalog.test.ts +++ b/packages/performance-tests/test/quick/tests/unit/ScenarioCatalog.test.ts @@ -4,6 +4,10 @@ *--------------------------------------------------------------------------------------------*/ import { describe, expect, it } from "vitest"; +import { + defaultScenarioBudgetMilliseconds, + scenarioBudgetMilliseconds, +} from "../../src/framework/BenchmarkScenario.js"; import { defaultQuickPerformanceScenarioId, getScenarioDefinition, @@ -19,9 +23,33 @@ describe("quick performance scenario catalog", () => { ); }); + it("defaults the budget when a scenario does not declare one", () => { + expect(scenarioBudgetMilliseconds(getScenarioDefinition())).to.equal( + defaultScenarioBudgetMilliseconds + ); + }); + + it("honours a declared budget", () => { + expect( + scenarioBudgetMilliseconds({ + ...getScenarioDefinition(), + budgetMilliseconds: 1000, + }) + ).to.equal(1000); + }); + + it("rejects a non-positive budget", () => { + expect(() => + scenarioBudgetMilliseconds({ + ...getScenarioDefinition(), + budgetMilliseconds: 0, + }) + ).to.throw(/invalid budget/); + }); + it("rejects unknown scenarios", () => { expect(() => getScenarioDefinition("not-a-scenario")).to.throw( - 'Unknown quick performance scenario "not-a-scenario". Available scenarios: incremental-synchronization' + 'Unknown quick performance scenario "not-a-scenario". Available scenarios: incremental-synchronization, changeset-scanning' ); }); });