Add focused changeset scanning benchmark

.github/workflows/quick-performance.yml

@@ -10,6 +10,7 @@ on:
         type: choice
         options:
           - incremental-synchronization
+          - changeset-scanning
       fixture:
         description: Fixture id (blank uses the scenario's default)
         required: false
@@ -78,6 +79,7 @@ jobs:
             "### Quick performance reliability",
             `Scenario: **${summary.scenarioId}**`,
             `Status: **${summary.varianceStatus}**`,
+            `Transformer: ${summary.versions?.transformer} / core-backend: ${summary.versions?.coreBackend} / node: ${summary.versions?.node}`,
             `CV: ${summary.wallMilliseconds.coefficientOfVariation}`,
             `Normalized MAD: ${summary.wallMilliseconds.normalizedMad}`,
             `Unstable samples: ${summary.unstableSamples.join(", ")}`,

packages/performance-tests/test/quick/QuickPerformance.test.ts

@@ -13,33 +13,46 @@ import { BenchmarkRunner } from "./src/framework/BenchmarkRunner.js";
 import { scenarioBudgetMilliseconds } from "./src/framework/BenchmarkScenario.js";
 import { quickPath } from "./src/support/paths.js";
 
+/**
+ * Headroom above the scenario budget so the budget assertion reports the overrun before Vitest
+ * kills the test.
+ */
+const budgetHeadroomMilliseconds = 60 * 1000;
+
 describe("quick performance", () => {
   const { descriptor, scenario } = resolveBenchmarkRunFromEnvironment();
   const measuredSamples = resolveMeasuredSamplesFromEnvironment();
   const budgetMilliseconds = scenarioBudgetMilliseconds(scenario);
 
-  it(`${scenario.id} completes within budget on ${descriptor.id}`, async () => {
-    const outputDir =
-      process.env.QUICK_PERF_OUTPUT ??
-      quickPath(".quick-output", descriptor.id);
-    const started = process.hrtime.bigint();
-    const samples = await new BenchmarkRunner(
-      descriptor,
-      outputDir,
-      scenario
-    ).run(measuredSamples);
-    const elapsedMilliseconds =
-      Number(process.hrtime.bigint() - started) / 1_000_000;
-    const summary = BenchmarkReporter.write(
-      outputDir,
-      samples,
-      elapsedMilliseconds
-    );
+  it(
+    `${scenario.id} completes within budget on ${descriptor.id}`,
+    async () => {
+      const outputDir =
+        process.env.QUICK_PERF_OUTPUT ??
+        quickPath(".quick-output", descriptor.id);
+      const started = process.hrtime.bigint();
+      const samples = await new BenchmarkRunner(
+        descriptor,
+        outputDir,
+        scenario
+      ).run(measuredSamples);
+      const elapsedMilliseconds =
+        Number(process.hrtime.bigint() - started) / 1_000_000;
+      const summary = BenchmarkReporter.write(
+        outputDir,
+        samples,
+        elapsedMilliseconds
+      );
 
-    expect(summary.measuredSamples).to.equal(measuredSamples);
-    expect(
-      new Set(samples.map((sample) => sample.semanticDigest)).size
-    ).to.equal(1, "every sample must observe the same fixture");
-    expect(elapsedMilliseconds).to.be.lessThan(budgetMilliseconds);
-  });
+      expect(summary.measuredSamples).to.equal(measuredSamples);
+      expect(
+        new Set(samples.map((sample) => sample.semanticDigest)).size
+      ).to.equal(1, "every sample must observe the same fixture");
+      expect(new Set(samples.map((sample) => sample.scenarioId))).to.deep.equal(
+        new Set([scenario.id])
+      );
+      expect(elapsedMilliseconds).to.be.lessThan(budgetMilliseconds);
+    },
+    budgetMilliseconds + budgetHeadroomMilliseconds
+  );
 });

packages/performance-tests/test/quick/assets/schemas/QuickPerfScan.ecschema.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ECSchema schemaName="QuickPerfScan" alias="qps" version="1.0.0"
+    xmlns="http://www.bentley.com/schemas/Bentley.ECXML.3.2">
+  <ECSchemaReference name="BisCore" version="01.00.00" alias="bis"/>
+
+  <ECEntityClass typeName="ScanAspect" modifier="Sealed">
+    <BaseClass>bis:ElementMultiAspect</BaseClass>
+    <ECProperty propertyName="Payload" typeName="string"/>
+    <ECProperty propertyName="Sequence" typeName="int"/>
+  </ECEntityClass>
+</ECSchema>

packages/performance-tests/test/quick/src/catalogs/FixtureCatalog.ts

@@ -4,15 +4,14 @@
  *--------------------------------------------------------------------------------------------*/
 
 import * as fs from "node:fs";
-import { createRequire } from "node:module";
 import * as path from "node:path";
 import {
   canonicalSha256,
   FixtureDescriptor,
 } from "../fixtures/FixtureDescriptor.js";
 import { quickPath, quickRootDirectory } from "../support/paths.js";
+import { resolvedVersions } from "../support/versions.js";
 
-const localRequire = createRequire(import.meta.url);
 const scale = 25;
 const distribution = {
   base: {
@@ -42,18 +41,7 @@ const distribution = {
   },
 } as const;
 
-function packageVersion(packageName: string): string {
-  const packageJson = JSON.parse(
-    fs.readFileSync(localRequire.resolve(`${packageName}/package.json`), "utf8")
-  ) as { version: string };
-  return packageJson.version;
-}
-
-const generator = {
-  coreBackend: packageVersion("@itwin/core-backend"),
-  node: process.version,
-  transformer: packageVersion("@itwin/imodel-transformer"),
-};
+const generator = resolvedVersions();
 
 const recipeIdentity = (topology: string) => ({
   schema: "QuickPerf.01.00.00",
@@ -129,12 +117,108 @@ export const balancedIncrementalSourceOnlyDescriptor: FixtureDescriptor = {
   recipeHash: canonicalSha256(recipeIdentity("source-only")),
 };
 
+/**
+ * Update-heavy multi-changeset source, sized so that scanning it dominates timer noise.
+ *
+ * Region sizes are derived from `base.elements` by `scanRegionSizes`; see
+ * `fixtures/recipes/updateHeavyScan` for what each region proves.
+ *
+ * Calibrated at 3,520 elements x 20 changesets: the scan measures ~3.46 s with a coefficient of
+ * variation of 1.0% over 8 samples, against ~51 ms of per-sample copy, verification and teardown.
+ * Scan cost is linear in changed rows at roughly 49 ms per (1,000 elements x changeset), so the
+ * shape is a single `scanScale` knob.
+ */
+const scanScale = 16;
+const scanDistribution = {
+  base: {
+    // Region A (updated throughout) plus region B (updated, then deleted last).
+    aspects: 220 * scanScale,
+    elements: 220 * scanScale,
+    geometricElements: 0,
+    relationships: 40 * scanScale,
+  },
+  operations: {
+    elements: {
+      // Regions C and D, both inserted in the first changeset.
+      inserts: 30 * scanScale,
+      // Every seeded element is updated, plus region C after its insert.
+      updates: 240 * scanScale,
+      // Region B at the end, region D at the end.
+      deletes: 30 * scanScale,
+    },
+    aspects: {
+      inserts: 20 * scanScale,
+      updates: 220 * scanScale,
+      // Region B's owned aspects, cascade-deleted with their elements.
+      deletes: 20 * scanScale,
+    },
+    relationships: {
+      inserts: 10 * scanScale,
+      updates: 10 * scanScale,
+      deletes: 10 * scanScale,
+    },
+    geometryUpdates: 0,
+    sourceChangesets: 20,
+  },
+} as const;
+
+const scanRecipeIdentity = {
+  schema: "QuickPerfScan.01.00.00",
+  seed: 328,
+  topology: "source-only",
+  distribution: scanDistribution,
+  inputs: {
+    recipe: fs.readFileSync(
+      quickPath("src", "fixtures", "recipes", "updateHeavyScan.ts"),
+      "utf8"
+    ),
+    schema: fs.readFileSync(
+      quickPath("assets", "schemas", "QuickPerfScan.ecschema.xml"),
+      "utf8"
+    ),
+    lockfile: fs.readFileSync(
+      path.resolve(
+        quickRootDirectory,
+        "..",
+        "..",
+        "..",
+        "..",
+        "pnpm-lock.yaml"
+      ),
+      "utf8"
+    ),
+  },
+  versions: generator,
+};
+
+export const updateHeavyScanDescriptor: FixtureDescriptor = {
+  id: "update-heavy-scan",
+  version: 1,
+  label: "update-heavy scan",
+  scenarioClaims: [
+    "changeset scanning",
+    "changed-instance squashing",
+    "aspect lifecycle",
+    "relationship lifecycle",
+  ],
+  layout: {
+    kind: "reconstructed",
+    topology: "source-only",
+    recipe: "update-heavy-scan",
+    seed: 328,
+  },
+  distribution: scanDistribution,
+  generator,
+  recipeHash: canonicalSha256(scanRecipeIdentity),
+};
+
 const fixtures = new Map<string, FixtureDescriptor>([
   [balancedIncrementalDescriptor.id, balancedIncrementalDescriptor],
   [
     balancedIncrementalSourceOnlyDescriptor.id,
     balancedIncrementalSourceOnlyDescriptor,
   ],
+  [updateHeavyScanDescriptor.id, updateHeavyScanDescriptor],
 ]);
 
 export function registerFixtureDescriptor(descriptor: FixtureDescriptor): void {

packages/performance-tests/test/quick/src/catalogs/ScenarioCatalog.ts

@@ -4,13 +4,15 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { BenchmarkScenarioDefinition } from "../framework/BenchmarkScenario.js";
+import { changesetScanningScenario } from "../scenarios/changesetScanning.js";
 import { incrementalSynchronizationScenario } from "../scenarios/incrementalSynchronization.js";
 
 export const defaultQuickPerformanceScenarioId =
   incrementalSynchronizationScenario.id;
 
 const scenarios = new Map<string, BenchmarkScenarioDefinition>([
   [incrementalSynchronizationScenario.id, incrementalSynchronizationScenario],
+  [changesetScanningScenario.id, changesetScanningScenario],
 ]);
 
 export function registerScenarioDefinition(

packages/performance-tests/test/quick/src/fixtures/FixtureRecipe.ts

@@ -11,6 +11,13 @@ import {
   BalancedRecipeState,
   createBalancedSeed,
 } from "./recipes/balancedIncremental.js";
+import {
+  applyScanChangesets,
+  createScanSeed,
+  ScanRecipeState,
+  validateScanFixture,
+} from "./recipes/updateHeavyScan.js";
+import { assertFixtureDistribution } from "./validation/validateFixture.js";
 
 /**
  * A recipe produces the *change mix* for a fixture: it seeds the source iModel and then applies a
@@ -40,6 +47,12 @@ export interface FixtureRecipe<TState = unknown, TArtifactData = unknown> {
     descriptor: FixtureDescriptor,
     state: TState
   ): Promise<TArtifactData | void>;
+  /**
+   * Assert the built source iModel matches what the descriptor promises.
+   *
+   * Validation is recipe-owned because it queries the classes that recipe created.
+   */
+  validate(db: BriefcaseDb, descriptor: FixtureDescriptor): Promise<void>;
 }
 
 export const balancedIncrementalRecipe: FixtureRecipe<BalancedRecipeState> = {
@@ -48,10 +61,21 @@ export const balancedIncrementalRecipe: FixtureRecipe<BalancedRecipeState> = {
     createBalancedSeed(fileName, descriptor),
   applySourceChangesets: async (db, accessToken, descriptor, state) =>
     applyBalancedChangesets(db, accessToken, descriptor, state),
+  validate: async (db, descriptor) => assertFixtureDistribution(db, descriptor),
+};
+
+export const updateHeavyScanRecipe: FixtureRecipe<ScanRecipeState> = {
+  id: "update-heavy-scan",
+  createSeed: async (fileName, descriptor) =>
+    createScanSeed(fileName, descriptor),
+  applySourceChangesets: async (db, accessToken, descriptor, state) =>
+    applyScanChangesets(db, accessToken, descriptor, state),
+  validate: async (db, descriptor) => validateScanFixture(db, descriptor),
 };
 
 const recipes = new Map<string, FixtureRecipe<any, any>>([
   [balancedIncrementalRecipe.id, balancedIncrementalRecipe],
+  [updateHeavyScanRecipe.id, updateHeavyScanRecipe],
 ]);
 
 export function registerFixtureRecipe(recipe: FixtureRecipe<any, any>): void {

packages/performance-tests/test/quick/src/fixtures/providers/detachedBriefcaseProvider.ts

@@ -35,7 +35,6 @@ import {
   reconstructSourceHub,
   shutdownHubMock,
 } from "../LocalHubFixture.js";
-import { assertFixtureDistribution } from "../validation/validateFixture.js";
 
 /**
  * Tolerant teardown for a build that may have failed at any point. Collects errors rather than
@@ -118,7 +117,7 @@ export const detachedBriefcaseFixtureProvider: FixtureProvider = {
         descriptor,
         recipeState
       );
-      await assertFixtureDistribution(hub.sourceDb, descriptor);
+      await recipe.validate(hub.sourceDb, descriptor);
 
       fs.mkdirSync(scratchDir, { recursive: true });
       const downloaded = await BriefcaseManager.downloadChangesets({

packages/performance-tests/test/quick/src/fixtures/providers/liveHubProvider.ts

@@ -18,7 +18,6 @@ import {
   ReconstructedHub,
   reconstructHub,
 } from "../LocalHubFixture.js";
-import { assertFixtureDistribution } from "../validation/validateFixture.js";
 
 /**
  * The `source-and-empty-target` topology.
@@ -73,7 +72,7 @@ export const liveHubFixtureProvider: FixtureProvider = {
         descriptor,
         recipeState
       );
-      await assertFixtureDistribution(hub.sourceDb, descriptor);
+      await recipe.validate(hub.sourceDb, descriptor);
       return {
         topology: "source-and-empty-target",
         descriptor,