From 6dac7a00cecc52b4d263348194df5b44516aec28 Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Date: Tue, 28 Jul 2026 21:13:07 -0400 Subject: [PATCH] perf: add isolated quick comparison workflow Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../quick-performance-comparison.yml | 449 +++++++++++ packages/performance-tests/README.md | 74 +- packages/performance-tests/package.json | 2 + .../test/quick/FixtureCatalog.ts | 9 +- .../test/quick/comparison/ArmModule.ts | 92 +++ .../quick/comparison/ComparisonFixture.ts | 105 +++ .../test/quick/comparison/ComparisonReport.ts | 23 + .../comparison/ComparisonRunner.quick-unit.ts | 399 ++++++++++ .../test/quick/comparison/ComparisonRunner.ts | 728 ++++++++++++++++++ .../test/quick/comparison/EnvironmentClass.ts | 16 +- .../test/quick/comparison/armProcessCli.ts | 43 ++ .../quick/comparison/comparison.quick-unit.ts | 1 + .../test/quick/comparison/comparisonCli.ts | 289 +++++++ .../test/quick/quickSource.ts | 16 + .../test/quick/recipes/balancedIncremental.ts | 4 +- .../test/quick/recipes/updateHeavyScan.ts | 4 +- .../test/quick/scenarios/changesetScanning.ts | 46 +- 17 files changed, 2278 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/quick-performance-comparison.yml create mode 100644 packages/performance-tests/test/quick/comparison/ComparisonFixture.ts create mode 100644 packages/performance-tests/test/quick/comparison/ComparisonRunner.quick-unit.ts create mode 100644 packages/performance-tests/test/quick/comparison/ComparisonRunner.ts create mode 100644 packages/performance-tests/test/quick/comparison/armProcessCli.ts create mode 100644 packages/performance-tests/test/quick/comparison/comparisonCli.ts create mode 100644 packages/performance-tests/test/quick/quickSource.ts diff --git a/.github/workflows/quick-performance-comparison.yml b/.github/workflows/quick-performance-comparison.yml new file mode 100644 index 00000000..db2ee159 --- /dev/null +++ b/.github/workflows/quick-performance-comparison.yml @@ -0,0 +1,449 @@ +name: Quick performance comparison + +on: + workflow_dispatch: + inputs: + mode: + description: Build A/A calibration or compare baseline with candidate + required: true + default: calibrate-a-a + type: choice + options: + - calibrate-a-a + - compare-a-b + scenario: + description: Comparison scenario + required: true + default: changeset-scanning + type: choice + options: + - changeset-scanning + calibration_ref: + description: Ref to build for both labeled A/A arms + required: true + default: main + type: string + baseline_ref: + description: Baseline ref for A/B + required: true + default: main + type: string + candidate_ref: + description: Candidate ref for A/B (blank uses the selected workflow ref) + required: false + type: string + pair_order: + description: Declared A/B job order + required: true + default: AB + type: choice + options: + - AB + - BA + calibration_run_id: + description: Prior successful calibration workflow run ID (A/B mode) + required: false + type: string + calibration_artifact: + description: Calibration artifact name from the prior run + required: true + default: QuickPerformanceCalibration + type: string + calibration_file: + description: Relative calibration JSON path inside the artifact + required: true + default: calibration.json + type: string + +permissions: + actions: read + contents: read + +env: + NODE_OPTIONS: --disable-warning=MODULE_TYPELESS_PACKAGE_JSON + +jobs: + calibrate-pair: + if: inputs.mode == 'calibrate-a-a' + name: A/A pair ${{ matrix.index }} (${{ matrix.order }}) + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - index: 0 + order: AB + - index: 1 + order: BA + - index: 2 + order: AB + + steps: + - name: Validate dispatch inputs + shell: bash + env: + CALIBRATION_REF: ${{ inputs.calibration_ref }} + SCENARIO: ${{ inputs.scenario }} + run: | + node - <<'NODE' + const { spawnSync } = require("child_process"); + function ref(value, label) { + if (!/^[A-Za-z0-9._/-]+$/.test(value) || value.includes("..") || value.includes("@{") || value.startsWith("-")) + throw new Error(`${label} contains unsafe characters`); + if (/^[0-9a-fA-F]{40}$/.test(value)) + return; + const branch = spawnSync("git", ["check-ref-format", "--branch", value]); + const full = spawnSync("git", ["check-ref-format", value]); + if (branch.status !== 0 && full.status !== 0) + throw new Error(`${label} is not a valid Git ref`); + } + ref(process.env.CALIBRATION_REF, "calibration_ref"); + if (process.env.SCENARIO !== "changeset-scanning") + throw new Error("Only changeset-scanning is supported"); + NODE + + - name: Checkout harness + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + path: harness + persist-credentials: false + + - name: Checkout calibration build + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + path: calibration + ref: ${{ inputs.calibration_ref }} + persist-credentials: false + + - name: Install pnpm + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 + with: + version: 11.9.0 + + - name: Use Node.js 24 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version: 24.18.0 + cache: pnpm + cache-dependency-path: | + harness/pnpm-lock.yaml + calibration/pnpm-lock.yaml + + - name: Build harness and calibration arm + shell: bash + run: | + pnpm --dir harness install --frozen-lockfile + pnpm --dir harness/packages/transformer build:cjs + pnpm --dir harness/packages/performance-tests build + pnpm --dir calibration install --frozen-lockfile + pnpm --dir calibration/packages/transformer build:cjs + + - name: Resolve checked-out calibration SHA + id: calibration + shell: bash + run: echo "sha=$(git -C calibration rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Validate comparison harness + shell: bash + run: pnpm --dir harness/packages/performance-tests test:comparison + + - name: Prepare deterministic fixture + shell: bash + run: | + node harness/packages/performance-tests/lib/quick/comparison/comparisonCli.js \ + prepare-fixture \ + --scenario changeset-scanning \ + --output "$RUNNER_TEMP/quick-comparison-fixture" + + - name: Run isolated A/A pair + id: pair + continue-on-error: true + shell: bash + env: + CALIBRATION_REF: ${{ inputs.calibration_ref }} + CALIBRATION_SHA: ${{ steps.calibration.outputs.sha }} + PAIR_INDEX: ${{ matrix.index }} + PAIR_ORDER: ${{ matrix.order }} + run: | + node harness/packages/performance-tests/lib/quick/comparison/comparisonCli.js \ + run-pair \ + --job-id "$GITHUB_RUN_ID-$PAIR_INDEX" \ + --pair "$PAIR_INDEX" \ + --order "$PAIR_ORDER" \ + --scenario changeset-scanning \ + --fixture "$RUNNER_TEMP/quick-comparison-fixture" \ + --arm-a-id calibration-a \ + --arm-a-label calibration \ + --arm-a-package "$GITHUB_WORKSPACE/calibration/packages/transformer" \ + --arm-a-ref "$CALIBRATION_REF" \ + --arm-a-sha "$CALIBRATION_SHA" \ + --arm-b-id calibration-b \ + --arm-b-label calibration \ + --arm-b-package "$GITHUB_WORKSPACE/calibration/packages/transformer" \ + --arm-b-ref "$CALIBRATION_REF" \ + --arm-b-sha "$CALIBRATION_SHA" \ + --output "$RUNNER_TEMP/quick-comparison-pair" + + - name: Add pair summary + if: always() + shell: bash + run: | + if [[ -f "$RUNNER_TEMP/quick-comparison-pair/pair-summary.md" ]]; then + cat "$RUNNER_TEMP/quick-comparison-pair/pair-summary.md" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Publish A/A observation + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: QuickPerformanceCalibrationObservation-${{ matrix.index }} + if-no-files-found: error + path: ${{ runner.temp }}/quick-comparison-pair + + - name: Fail invalid pair + if: steps.pair.outcome == 'failure' + shell: bash + run: exit 1 + + aggregate-calibration: + if: always() && inputs.mode == 'calibrate-a-a' + name: Aggregate A/A calibration + needs: calibrate-pair + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout harness + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + path: harness + persist-credentials: false + + - name: Install pnpm + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 + with: + version: 11.9.0 + + - name: Use Node.js 24 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version: 24.18.0 + cache: pnpm + cache-dependency-path: harness/pnpm-lock.yaml + + - name: Build aggregation harness + shell: bash + run: | + pnpm --dir harness install --frozen-lockfile + pnpm --dir harness/packages/transformer build:cjs + pnpm --dir harness/packages/performance-tests build + + - name: Download A/A observations + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + pattern: QuickPerformanceCalibrationObservation-* + path: ${{ runner.temp }}/quick-calibration-observations + + - name: Aggregate matching independent jobs + shell: bash + run: | + node harness/packages/performance-tests/lib/quick/comparison/comparisonCli.js \ + aggregate-calibration \ + --input "$RUNNER_TEMP/quick-calibration-observations" \ + --output "$RUNNER_TEMP/quick-calibration" + + - name: Add calibration summary + shell: bash + run: cat "$RUNNER_TEMP/quick-calibration/calibration-summary.md" >> "$GITHUB_STEP_SUMMARY" + + - name: Publish calibration + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: QuickPerformanceCalibration + if-no-files-found: error + path: ${{ runner.temp }}/quick-calibration + + compare: + if: inputs.mode == 'compare-a-b' + name: A/B comparison (${{ inputs.pair_order }}) + runs-on: ubuntu-latest + timeout-minutes: 35 + + steps: + - name: Validate dispatch inputs + shell: bash + env: + BASELINE_REF: ${{ inputs.baseline_ref }} + CANDIDATE_REF: ${{ inputs.candidate_ref || github.ref }} + CALIBRATION_RUN_ID: ${{ inputs.calibration_run_id }} + CALIBRATION_ARTIFACT: ${{ inputs.calibration_artifact }} + CALIBRATION_FILE: ${{ inputs.calibration_file }} + SCENARIO: ${{ inputs.scenario }} + run: | + node - <<'NODE' + const { spawnSync } = require("child_process"); + function ref(value, label) { + if (!/^[A-Za-z0-9._/-]+$/.test(value) || value.includes("..") || value.includes("@{") || value.startsWith("-")) + throw new Error(`${label} contains unsafe characters`); + if (/^[0-9a-fA-F]{40}$/.test(value)) + return; + const branch = spawnSync("git", ["check-ref-format", "--branch", value]); + const full = spawnSync("git", ["check-ref-format", value]); + if (branch.status !== 0 && full.status !== 0) + throw new Error(`${label} is not a valid Git ref`); + } + ref(process.env.BASELINE_REF, "baseline_ref"); + ref(process.env.CANDIDATE_REF, "candidate_ref"); + if (!/^[1-9][0-9]*$/.test(process.env.CALIBRATION_RUN_ID)) + throw new Error("calibration_run_id must contain digits only"); + if (!/^[A-Za-z0-9._-]+$/.test(process.env.CALIBRATION_ARTIFACT)) + throw new Error("calibration_artifact contains unsafe characters"); + const file = process.env.CALIBRATION_FILE; + if (!/^[A-Za-z0-9._/-]+$/.test(file) || file.startsWith("/") || file.includes("..")) + throw new Error("calibration_file must be a safe relative path"); + if (process.env.SCENARIO !== "changeset-scanning") + throw new Error("Only changeset-scanning is supported"); + NODE + + - name: Checkout harness + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + path: harness + persist-credentials: false + + - name: Checkout baseline + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + path: baseline + ref: ${{ inputs.baseline_ref }} + persist-credentials: false + + - name: Checkout candidate + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + path: candidate + ref: ${{ inputs.candidate_ref || github.ref }} + persist-credentials: false + + - name: Install pnpm + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 + with: + version: 11.9.0 + + - name: Use Node.js 24 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version: 24.18.0 + cache: pnpm + cache-dependency-path: | + harness/pnpm-lock.yaml + baseline/pnpm-lock.yaml + candidate/pnpm-lock.yaml + + - name: Build harness, baseline, and candidate + shell: bash + run: | + pnpm --dir harness install --frozen-lockfile + pnpm --dir harness/packages/transformer build:cjs + pnpm --dir harness/packages/performance-tests build + pnpm --dir baseline install --frozen-lockfile + pnpm --dir baseline/packages/transformer build:cjs + pnpm --dir candidate install --frozen-lockfile + pnpm --dir candidate/packages/transformer build:cjs + + - name: Resolve checked-out SHAs + id: refs + shell: bash + run: | + echo "baseline_sha=$(git -C baseline rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "candidate_sha=$(git -C candidate rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Validate comparison harness + shell: bash + run: pnpm --dir harness/packages/performance-tests test:comparison + + - name: Download prior calibration + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: ${{ inputs.calibration_artifact }} + path: ${{ runner.temp }}/quick-calibration + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ inputs.calibration_run_id }} + + - name: Prepare deterministic fixture + shell: bash + run: | + node harness/packages/performance-tests/lib/quick/comparison/comparisonCli.js \ + prepare-fixture \ + --scenario changeset-scanning \ + --output "$RUNNER_TEMP/quick-comparison-fixture" + + - name: Run isolated A/B pair + id: pair + continue-on-error: true + shell: bash + env: + BASELINE_REF: ${{ inputs.baseline_ref }} + BASELINE_SHA: ${{ steps.refs.outputs.baseline_sha }} + CANDIDATE_REF: ${{ inputs.candidate_ref || github.ref }} + CANDIDATE_SHA: ${{ steps.refs.outputs.candidate_sha }} + PAIR_INDEX: ${{ inputs.pair_order == 'AB' && '0' || '1' }} + PAIR_ORDER: ${{ inputs.pair_order }} + run: | + node harness/packages/performance-tests/lib/quick/comparison/comparisonCli.js \ + run-pair \ + --job-id "$GITHUB_RUN_ID" \ + --pair "$PAIR_INDEX" \ + --order "$PAIR_ORDER" \ + --scenario changeset-scanning \ + --fixture "$RUNNER_TEMP/quick-comparison-fixture" \ + --arm-a-id baseline \ + --arm-a-label baseline \ + --arm-a-package "$GITHUB_WORKSPACE/baseline/packages/transformer" \ + --arm-a-ref "$BASELINE_REF" \ + --arm-a-sha "$BASELINE_SHA" \ + --arm-b-id candidate \ + --arm-b-label candidate \ + --arm-b-package "$GITHUB_WORKSPACE/candidate/packages/transformer" \ + --arm-b-ref "$CANDIDATE_REF" \ + --arm-b-sha "$CANDIDATE_SHA" \ + --output "$RUNNER_TEMP/quick-comparison-pair" + + - name: Compare with matching calibration + if: steps.pair.outcome == 'success' + shell: bash + env: + CALIBRATION_FILE: ${{ inputs.calibration_file }} + run: | + node harness/packages/performance-tests/lib/quick/comparison/comparisonCli.js \ + compare \ + --observation "$RUNNER_TEMP/quick-comparison-pair/pair-observation.json" \ + --calibration "$RUNNER_TEMP/quick-calibration/$CALIBRATION_FILE" \ + --output "$RUNNER_TEMP/quick-comparison-report" + + - name: Add comparison summary + if: always() + shell: bash + run: | + if [[ -f "$RUNNER_TEMP/quick-comparison-report/comparison.md" ]]; then + cat "$RUNNER_TEMP/quick-comparison-report/comparison.md" >> "$GITHUB_STEP_SUMMARY" + elif [[ -f "$RUNNER_TEMP/quick-comparison-pair/pair-summary.md" ]]; then + cat "$RUNNER_TEMP/quick-comparison-pair/pair-summary.md" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Publish comparison report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: QuickPerformanceComparison + if-no-files-found: error + path: | + ${{ runner.temp }}/quick-comparison-pair + ${{ runner.temp }}/quick-comparison-report + + - name: Fail invalid pair + if: steps.pair.outcome == 'failure' + shell: bash + run: exit 1 diff --git a/packages/performance-tests/README.md b/packages/performance-tests/README.md index d08efd4a..7717e538 100644 --- a/packages/performance-tests/README.md +++ b/packages/performance-tests/README.md @@ -67,23 +67,78 @@ the repository's default branch. The `--ref` selects which committed branch revision runs after that requirement is met; there is intentionally no custom branch input and no automatic pull-request or push trigger. +### Quick A/A calibration and A/B comparison + +`quick-performance-comparison.yml` is a separate manual-only workflow for +`changeset-scanning`. It does not use the weekly Hub-backed regression suite and +never creates a merge-blocking performance gate. Each independent job produces +exactly one `PairObservation`: the declared `AB` or `BA` order runs one isolated +process per arm, each process performs one excluded warm-up plus three measured +scenario executions, and each arm's three measurements are median-collapsed +before computing one log-ratio. This is eight scenario executions per job; the +within-process samples are never treated as three independent pairs. + +Calibration mode builds one selected `calibration_ref` for both labeled arms and +runs three independent jobs in `AB`, `BA`, `AB` order. The aggregation job +rejects fixture, recipe, workload, environment, execution-policy, build, or +fingerprint mismatches. Three matching observations from three jobs establish +the initial pool. An absolute A/A band at or below 5% is target quality, between +5% and 10% is marginal, and 10% or greater is unresolvable. All three remain +informational; the practical effect and equivalence threshold is 10%. + +```sh +gh workflow run quick-performance-comparison.yml --ref main \ + -f mode=calibrate-a-a \ + -f scenario=changeset-scanning \ + -f calibration_ref=main +``` + +Comparison mode checks out and builds baseline and candidate separately. The +top-layer harness loads `ChangedInstanceIds` from each explicit built package in +its own child process, so the baseline ref does not need to contain the +comparison harness. Supply the prior calibration workflow run ID and artifact; +leave `candidate_ref` blank to use the workflow ref selected by `--ref`. + +```sh +gh workflow run quick-performance-comparison.yml --ref main \ + -f mode=compare-a-b \ + -f scenario=changeset-scanning \ + -f baseline_ref=main \ + -f candidate_ref=my-candidate \ + -f pair_order=AB \ + -f calibration_run_id=123456789 \ + -f calibration_artifact=QuickPerformanceCalibration \ + -f calibration_file=calibration.json +``` + +The workflow publishes raw arm samples, the pair observation (including discard +reasons), the calibration pool and band, summary JSON, and Markdown used for the +job summary. A/B rejects a missing calibration or any calibration fingerprint +or hosted-runner class mismatch instead of borrowing a nearby result. + +GitHub can dispatch this workflow only after +`quick-performance-comparison.yml` reaches the repository's default branch. +Before then, local smoke runs can exercise process orchestration but cannot +create hosted calibration evidence. The CLI's `prepare-fixture --smoke true` +uses an explicitly labeled reduced fixture; its different identity prevents it +from being accepted as calibration for the full fixture. + `pnpm quick:build-fixture` writes the canonical recipe manifest. `pnpm quick:verify-fixture` performs two fresh reconstructions (warm-up plus one measured sample), checks their semantic digests, and writes a diagnostic report. Here are tests we need but don't have: -- *Identity Transform* +- _Identity Transform_ transform the entire contents of the iModel to an empty iModel seed -- *JSON Geometry Editing Transform* +- _JSON Geometry Editing Transform_ transform the iModel, editing geometry as we go using the json format -- *Binary Geometry Editing Transform* +- _Binary Geometry Editing Transform_ transform the iModel, editing geometry as we go using elementGeometryBuilderParams -- *Optimistically Locking Remote Target* -- *Pessimistically Locking Remote Target* -- *Processing Changes* -- *More Branching Stuff* - +- _Optimistically Locking Remote Target_ +- _Pessimistically Locking Remote Target_ +- _Processing Changes_ +- _More Branching Stuff_ ## Usage @@ -97,13 +152,14 @@ Here are tests we need but don't have: 3. Create `.env` file using `template.env` template. -5. Run: +4. Run: ```sh pnpm test ``` + 6. Review results like: ```sh diff --git a/packages/performance-tests/package.json b/packages/performance-tests/package.json index 8739181a..08d526ca 100644 --- a/packages/performance-tests/package.json +++ b/packages/performance-tests/package.json @@ -11,8 +11,10 @@ "format": "prettier \"./test/**/*.ts\" --write", "test-mocha": "mocha --delay --timeout 300000 \"./lib/**/TransformerRegression.test.js\"", "test:quick": "mocha --require ts-node/register test/quick/QuickPerformance.quick.ts", + "test:comparison": "mocha --timeout 300000 --require ts-node/register test/quick/comparison/comparison.quick-unit.ts test/quick/comparison/ComparisonRunner.quick-unit.ts", "quick:build-fixture": "ts-node test/quick/cli.ts build-fixture", "quick:verify-fixture": "ts-node test/quick/cli.ts verify-fixture", + "quick:compare": "node lib/quick/comparison/comparisonCli.js", "process-reports": "node scripts/process-reports" }, "repository": {}, diff --git a/packages/performance-tests/test/quick/FixtureCatalog.ts b/packages/performance-tests/test/quick/FixtureCatalog.ts index 26d16e28..2dbe0046 100644 --- a/packages/performance-tests/test/quick/FixtureCatalog.ts +++ b/packages/performance-tests/test/quick/FixtureCatalog.ts @@ -7,6 +7,7 @@ import * as fs from "fs"; import * as path from "path"; import { DatasetDescriptor } from "./DatasetDescriptor"; import { canonicalSha256 } from "./FixtureManifest"; +import { resolveQuickSourceFile } from "./quickSource"; import { resolvedVersions } from "./versions"; const scale = 25; @@ -47,11 +48,11 @@ const recipeIdentity = (topology: string) => ({ distribution, inputs: { recipe: fs.readFileSync( - path.join(__dirname, "recipes/balancedIncremental.ts"), + resolveQuickSourceFile("recipes/balancedIncremental.ts"), "utf8" ), schema: fs.readFileSync( - path.join(__dirname, "schemas/QuickPerf.ecschema.xml"), + resolveQuickSourceFile("schemas/QuickPerf.ecschema.xml"), "utf8" ), lockfile: fs.readFileSync( @@ -166,11 +167,11 @@ const scanRecipeIdentity = { distribution: scanDistribution, inputs: { recipe: fs.readFileSync( - path.join(__dirname, "recipes/updateHeavyScan.ts"), + resolveQuickSourceFile("recipes/updateHeavyScan.ts"), "utf8" ), schema: fs.readFileSync( - path.join(__dirname, "schemas/QuickPerfScan.ecschema.xml"), + resolveQuickSourceFile("schemas/QuickPerfScan.ecschema.xml"), "utf8" ), lockfile: fs.readFileSync( diff --git a/packages/performance-tests/test/quick/comparison/ArmModule.ts b/packages/performance-tests/test/quick/comparison/ArmModule.ts index 7a4b2079..9baabe29 100644 --- a/packages/performance-tests/test/quick/comparison/ArmModule.ts +++ b/packages/performance-tests/test/quick/comparison/ArmModule.ts @@ -4,7 +4,10 @@ *--------------------------------------------------------------------------------------------*/ import * as fs from "fs"; +import * as crypto from "crypto"; +import { createRequire } from "module"; import * as path from "path"; +import type { ChangedInstanceIdsDependency } from "../scenarios/changesetScanning"; export type ArmOperation = "identity" | "fork-init" | "change-processing"; @@ -36,6 +39,33 @@ interface PackageManifest { readonly peerDependencies?: Readonly>; } +function hashRuntimeFiles( + packageRoot: string, + roots: readonly string[] +): string { + const files: string[] = []; + const visit = (current: string): void => { + const stat = fs.statSync(current); + if (stat.isFile()) { + files.push(current); + return; + } + for (const entry of fs.readdirSync(current, { withFileTypes: true })) { + if (entry.name === "node_modules") continue; + visit(path.join(current, entry.name)); + } + }; + roots.forEach(visit); + const hash = crypto.createHash("sha256"); + for (const file of [...new Set(files)].sort()) { + hash.update(path.relative(packageRoot, file).split(path.sep).join("/")); + hash.update("\0"); + hash.update(fs.readFileSync(file)); + hash.update("\0"); + } + return hash.digest("hex"); +} + function readManifest(packageRoot: string): PackageManifest { const manifestPath = path.join(packageRoot, "package.json"); if (!fs.existsSync(manifestPath)) @@ -79,11 +109,73 @@ export function resolveArmSpec(spec: ArmSpec): ResolvedArmSpec { export interface ArmRuntimeIdentity { readonly armId: string; readonly transformerVersion: string; + readonly transformerPackageHash: string; readonly coreBackendVersion: string; /** SHA-256 identity of the resolved core-backend package produced by the child process. */ readonly coreBackendPackageHash: string; } +export interface LoadedArmModule { + readonly changedInstanceIds: ChangedInstanceIdsDependency; + readonly runtime: ArmRuntimeIdentity; +} + +/** + * Make the child harness and selected transformer share the arm checkout's core-backend instance. + * + * This must run before importing any module that imports core-backend. + */ +export function aliasHarnessCoreBackendToArm(arm: ResolvedArmSpec): void { + const armRequire = createRequire(arm.modulePath); + const armEntry = armRequire.resolve("@itwin/core-backend"); + const harnessEntry = require.resolve("@itwin/core-backend"); + armRequire("@itwin/core-backend"); + const armModule = require.cache[armEntry]; + if (!armModule) + throw new Error( + `Arm "${arm.spec.id}" core-backend module could not be initialized` + ); + require.cache[harnessEntry] = armModule; +} + +/** + * Load the selected transformer only inside an arm child. The comparison parent must use + * {@link resolveArmSpec} instead so native dependencies from two checkouts cannot share a process. + */ +export function loadArmModule(arm: ResolvedArmSpec): LoadedArmModule { + const armRequire = createRequire(arm.modulePath); + const exports = armRequire(arm.modulePath) as { + readonly ChangedInstanceIds?: ChangedInstanceIdsDependency; + }; + if (!exports.ChangedInstanceIds?.initialize) + throw new Error( + `Arm "${arm.spec.id}" does not export ChangedInstanceIds.initialize` + ); + const coreManifestPath = armRequire.resolve( + "@itwin/core-backend/package.json" + ); + const coreManifestBytes = fs.readFileSync(coreManifestPath); + const coreManifest = JSON.parse(coreManifestBytes.toString("utf8")) as { + readonly version?: string; + }; + const corePackageRoot = path.dirname(coreManifestPath); + return { + changedInstanceIds: exports.ChangedInstanceIds, + runtime: { + armId: arm.spec.id, + transformerVersion: arm.transformerVersion, + transformerPackageHash: hashRuntimeFiles(arm.packageRoot, [ + path.join(arm.packageRoot, "package.json"), + path.dirname(arm.modulePath), + ]), + coreBackendVersion: coreManifest.version ?? "unknown", + coreBackendPackageHash: hashRuntimeFiles(corePackageRoot, [ + corePackageRoot, + ]), + }, + }; +} + /** * Validate child-process identities without loading either native module in this process. */ diff --git a/packages/performance-tests/test/quick/comparison/ComparisonFixture.ts b/packages/performance-tests/test/quick/comparison/ComparisonFixture.ts new file mode 100644 index 00000000..d1dc5c6f --- /dev/null +++ b/packages/performance-tests/test/quick/comparison/ComparisonFixture.ts @@ -0,0 +1,105 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "fs"; +import { IModelHost } from "@itwin/core-backend"; +import * as path from "path"; +import { assertSafeBenchmarkOutputPath } from "../BenchmarkRunner"; +import { DatasetDescriptor } from "../DatasetDescriptor"; +import { updateHeavyScanDescriptor } from "../FixtureCatalog"; +import { canonicalSha256 } from "../FixtureManifest"; +import { getFixtureProvider } from "../FixtureProvider"; +import { + ComparisonFingerprint, + fingerprintForArtifact, + hashFixtureArtifact, +} from "./ComparisonRunner"; + +const comparisonFixtureMarker = ".imodel-transformer-comparison-fixture"; + +function smokeDescriptor(): DatasetDescriptor { + const divide = (value: number): number => Math.max(1, Math.round(value / 16)); + const distribution = updateHeavyScanDescriptor.distribution; + const reduced = { + base: { + aspects: divide(distribution.base.aspects), + elements: divide(distribution.base.elements), + geometricElements: distribution.base.geometricElements, + relationships: divide(distribution.base.relationships), + }, + operations: { + elements: { + inserts: divide(distribution.operations.elements.inserts), + updates: divide(distribution.operations.elements.updates), + deletes: divide(distribution.operations.elements.deletes), + }, + aspects: { + inserts: divide(distribution.operations.aspects.inserts), + updates: divide(distribution.operations.aspects.updates), + deletes: divide(distribution.operations.aspects.deletes), + }, + relationships: { + inserts: divide(distribution.operations.relationships.inserts), + updates: divide(distribution.operations.relationships.updates), + deletes: divide(distribution.operations.relationships.deletes), + }, + geometryUpdates: distribution.operations.geometryUpdates, + sourceChangesets: distribution.operations.sourceChangesets, + }, + }; + return { + ...updateHeavyScanDescriptor, + id: `${updateHeavyScanDescriptor.id}-smoke`, + label: `${updateHeavyScanDescriptor.label} (smoke only)`, + distribution: reduced, + recipeHash: canonicalSha256({ + purpose: "local-smoke-not-calibration", + sourceRecipeHash: updateHeavyScanDescriptor.recipeHash, + distribution: reduced, + }), + }; +} + +export async function prepareComparisonFixture( + outputDirectory: string, + smoke = false +): Promise<{ + readonly directory: string; + readonly artifactHash: string; + readonly fingerprint: ComparisonFingerprint; +}> { + const descriptor = smoke ? smokeDescriptor() : updateHeavyScanDescriptor; + if (descriptor.layout.topology !== "source-only") + throw new Error("Comparison fixture must produce a detached artifact"); + const provider = getFixtureProvider(descriptor); + assertSafeBenchmarkOutputPath(outputDirectory); + fs.mkdirSync(outputDirectory, { recursive: true }); + const marker = path.join(outputDirectory, comparisonFixtureMarker); + if (fs.readdirSync(outputDirectory).length > 0 && !fs.existsSync(marker)) + throw new Error( + `Refusing to replace non-empty unowned fixture directory: ${outputDirectory}` + ); + fs.writeFileSync(marker, "Owned by quick performance comparison.\n"); + const profileName = `quick-compare-fixture-${process.pid}`; + await IModelHost.startup({ profileName }); + const profileDir = IModelHost.profileDir; + try { + try { + await provider.build(descriptor, outputDirectory); + } finally { + fs.mkdirSync(outputDirectory, { recursive: true }); + fs.writeFileSync(marker, "Owned by quick performance comparison.\n"); + } + } finally { + await IModelHost.shutdown(); + if (profileDir.includes(profileName)) + fs.rmSync(profileDir, { recursive: true, force: true }); + } + return { + directory: outputDirectory, + artifactHash: hashFixtureArtifact(outputDirectory), + fingerprint: fingerprintForArtifact(outputDirectory), + }; +} diff --git a/packages/performance-tests/test/quick/comparison/ComparisonReport.ts b/packages/performance-tests/test/quick/comparison/ComparisonReport.ts index 4498e6cc..a303a8f5 100644 --- a/packages/performance-tests/test/quick/comparison/ComparisonReport.ts +++ b/packages/performance-tests/test/quick/comparison/ComparisonReport.ts @@ -34,8 +34,12 @@ export type ComparisonMode = "paired"; export interface ArmDescription { readonly id: string; readonly label?: string; + readonly ref?: string; + readonly sha?: string; readonly transformerVersion: string; + readonly transformerPackageHash?: string; readonly coreBackendVersion: string; + readonly coreBackendPackageHash?: string; } export interface DetectabilitySummary { @@ -207,6 +211,25 @@ export function renderComparisonReport(report: ComparisonReport): string { `| Environment | ${report.environment.id} |`, `| Arm A | ${report.armA.id} (${report.armA.transformerVersion}) |`, `| Arm B | ${report.armB.id} (${report.armB.transformerVersion}) |`, + `| Arm A transformer hash | ${ + report.armA.transformerPackageHash ?? "unavailable" + } |`, + `| Arm B transformer hash | ${ + report.armB.transformerPackageHash ?? "unavailable" + } |`, + `| Arm A core-backend | ${report.armA.coreBackendVersion} (${ + report.armA.coreBackendPackageHash ?? "hash unavailable" + }) |`, + `| Arm B core-backend | ${report.armB.coreBackendVersion} (${ + report.armB.coreBackendPackageHash ?? "hash unavailable" + }) |`, + `| Arm A ref / SHA | ${report.armA.ref ?? "n/a"} / ${ + report.armA.sha ?? "n/a" + } |`, + `| Arm B ref / SHA | ${report.armB.ref ?? "n/a"} / ${ + report.armB.sha ?? "n/a" + } |`, + `| Fixture recipe hash | ${report.recipeHash} |`, `| Median change | ${formatPercent(report.aggregate.percentChange)} |`, `| Geometric-mean change | ${formatPercent( report.aggregate.geometricMeanPercentChange diff --git a/packages/performance-tests/test/quick/comparison/ComparisonRunner.quick-unit.ts b/packages/performance-tests/test/quick/comparison/ComparisonRunner.quick-unit.ts new file mode 100644 index 00000000..ef048985 --- /dev/null +++ b/packages/performance-tests/test/quick/comparison/ComparisonRunner.quick-unit.ts @@ -0,0 +1,399 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { expect } from "chai"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { loadArmModule, resolveArmSpec } from "./ArmModule"; +import { + aggregateCalibration, + ArmRunRequest, + ArmRunResult, + assertComparisonFingerprintMatches, + comparisonExecutionsPerPair, + comparisonMeasuredSamples, + comparisonWarmups, + fingerprintForArtifact, + hashFixtureArtifact, + PairRunArtifact, + runPair, +} from "./ComparisonRunner"; +import { + artifactBriefcaseFileName, + artifactChangesetDirectoryName, + artifactChangesetPropsFileName, + artifactManifestFileName, + fixtureArtifactVersion, +} from "../FixtureArtifact"; +import { updateHeavyScanDescriptor } from "../FixtureCatalog"; +import { classifyCalibrationQuality } from "./NoiseBand"; + +describe("quick comparison runner", () => { + let root: string; + let fixtureDirectory: string; + let armPackage: string; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "quick-comparison-unit-")); + fixtureDirectory = path.join(root, "fixture"); + fs.mkdirSync(path.join(fixtureDirectory, artifactChangesetDirectoryName), { + recursive: true, + }); + fs.writeFileSync( + path.join(fixtureDirectory, artifactBriefcaseFileName), + "" + ); + fs.writeFileSync( + path.join(fixtureDirectory, artifactChangesetPropsFileName), + "[]\n" + ); + fs.writeFileSync( + path.join(fixtureDirectory, artifactManifestFileName), + `${JSON.stringify( + { + artifactVersion: fixtureArtifactVersion, + descriptor: updateHeavyScanDescriptor, + briefcase: { + fileName: artifactBriefcaseFileName, + briefcaseId: 1, + changeset: { id: "", index: 0 }, + byteLength: 0, + }, + changesets: { + directory: artifactChangesetDirectoryName, + propsFile: artifactChangesetPropsFileName, + count: 0, + baseChangesetIndex: 0, + }, + buildMilliseconds: 1, + builtAt: new Date(0).toISOString(), + }, + undefined, + 2 + )}\n` + ); + armPackage = createArmPackage("explicit-arm", 17); + }); + + afterEach(() => { + fs.rmSync(root, { recursive: true, force: true }); + }); + + function createArmPackage(name: string, sentinel: number): string { + const directory = path.join(root, name); + fs.mkdirSync(directory); + fs.writeFileSync( + path.join(directory, "package.json"), + `${JSON.stringify({ + name, + version: `1.0.${sentinel}`, + main: "index.js", + peerDependencies: { "@itwin/core-backend": "^5.10.0" }, + })}\n` + ); + fs.writeFileSync( + path.join(directory, "index.js"), + `exports.ChangedInstanceIds = { initialize: async () => ${sentinel} };\n` + ); + const scope = path.join(directory, "node_modules", "@itwin"); + fs.mkdirSync(scope, { recursive: true }); + fs.symlinkSync( + path.dirname(require.resolve("@itwin/core-backend/package.json")), + path.join(scope, "core-backend"), + "junction" + ); + return directory; + } + + function armResult( + request: ArmRunRequest, + measured: readonly number[] + ): ArmRunResult { + return { + arm: request.arm, + source: request.source, + runtime: { + armId: request.arm.id, + transformerVersion: "1.0.0", + transformerPackageHash: "same-transformer", + coreBackendVersion: "5.10.3", + coreBackendPackageHash: "same-core", + }, + fingerprint: request.fingerprint, + fixtureArtifactHash: request.fixtureArtifactHash, + samples: [ + { + sample: 0, + measured: false, + wallMilliseconds: 999, + semanticDigest: "same", + reconstructionMilliseconds: 1, + verificationMilliseconds: 1, + teardownMilliseconds: 1, + }, + ...measured.map((wallMilliseconds, index) => ({ + sample: index + 1, + measured: true, + wallMilliseconds, + semanticDigest: "same", + reconstructionMilliseconds: 1, + verificationMilliseconds: 1, + teardownMilliseconds: 1, + })), + ], + generatedAt: new Date(0).toISOString(), + }; + } + + async function validPair( + pair: number, + order: "AB" | "BA", + armATimes: readonly number[] = [10, 12, 11], + armBTimes: readonly number[] = [20, 22, 21] + ): Promise { + return runPair({ + jobId: `job-${pair}`, + pair, + order, + scenarioId: "changeset-scanning", + fixtureDirectory, + armA: { + id: "A", + packageRoot: armPackage, + operation: "change-processing", + }, + armASource: { ref: "calibration", sha: "a".repeat(40) }, + armB: { + id: "B", + packageRoot: armPackage, + operation: "change-processing", + }, + armBSource: { ref: "calibration", sha: "a".repeat(40) }, + outputDirectory: path.join(root, `pair-${pair}`), + launcher: async (request) => + armResult(request, request.arm.id === "A" ? armATimes : armBTimes), + }); + } + + it("executes one warm-up plus three measurements per arm and collapses one pair", async () => { + const requests: ArmRunRequest[] = []; + const pair = await runPair({ + jobId: "job", + pair: 0, + order: "AB", + scenarioId: "changeset-scanning", + fixtureDirectory, + armA: { + id: "A", + packageRoot: armPackage, + operation: "change-processing", + }, + armASource: { ref: "base", sha: "a".repeat(40) }, + armB: { + id: "B", + packageRoot: armPackage, + operation: "change-processing", + }, + armBSource: { ref: "candidate", sha: "b".repeat(40) }, + outputDirectory: path.join(root, "pair"), + launcher: async (request) => { + requests.push(request); + return armResult( + request, + request.arm.id === "A" ? [10, 100, 11] : [20, 21, 200] + ); + }, + }); + expect(requests.map((request) => request.arm.id)).to.deep.equal(["A", "B"]); + expect( + requests.length * (comparisonWarmups + comparisonMeasuredSamples) + ).to.equal(comparisonExecutionsPerPair); + expect(pair.observation?.armASamples).to.deep.equal([10, 100, 11]); + expect(pair.collapsed?.armA).to.equal(11); + expect(pair.collapsed?.armB).to.equal(21); + }); + + it("executes BA in declared order without changing arm identity", async () => { + const launched: string[] = []; + const pair = await runPair({ + jobId: "job", + pair: 1, + order: "BA", + scenarioId: "changeset-scanning", + fixtureDirectory, + armA: { + id: "A", + packageRoot: armPackage, + operation: "change-processing", + }, + armASource: { ref: "base", sha: "a".repeat(40) }, + armB: { + id: "B", + packageRoot: armPackage, + operation: "change-processing", + }, + armBSource: { ref: "candidate", sha: "b".repeat(40) }, + outputDirectory: path.join(root, "pair"), + launcher: async (request) => { + launched.push(request.arm.id); + return armResult( + request, + request.arm.id === "A" ? [9, 10, 11] : [19, 20, 21] + ); + }, + }); + expect(launched).to.deep.equal(["B", "A"]); + expect(pair.observation?.order).to.equal("BA"); + expect(pair.collapsed?.armA).to.equal(10); + expect(pair.collapsed?.armB).to.equal(20); + }); + + it("discards the whole pair on child failure or malformed output", async () => { + let launches = 0; + const failed = await runPair({ + jobId: "failed", + pair: 0, + order: "AB", + scenarioId: "changeset-scanning", + fixtureDirectory, + armA: { + id: "A", + packageRoot: armPackage, + operation: "change-processing", + }, + armASource: { ref: "base", sha: "a".repeat(40) }, + armB: { + id: "B", + packageRoot: armPackage, + operation: "change-processing", + }, + armBSource: { ref: "candidate", sha: "b".repeat(40) }, + outputDirectory: path.join(root, "failed"), + launcher: async () => { + launches++; + throw new Error("child crashed"); + }, + }); + expect(launches).to.equal(1); + expect(failed.observation).to.be.undefined; + expect(failed.discardedReason).to.contain("child crashed"); + + const malformed = await runPair({ + jobId: "malformed", + pair: 0, + order: "AB", + scenarioId: "changeset-scanning", + fixtureDirectory, + armA: { + id: "A", + packageRoot: armPackage, + operation: "change-processing", + }, + armASource: { ref: "base", sha: "a".repeat(40) }, + armB: { + id: "B", + packageRoot: armPackage, + operation: "change-processing", + }, + armBSource: { ref: "candidate", sha: "b".repeat(40) }, + outputDirectory: path.join(root, "malformed"), + launcher: async () => ({ samples: [] }), + }); + expect(malformed.observation).to.be.undefined; + expect(malformed.discardedReason).to.contain("one warm-up and three"); + }); + + it("rejects fixture and comparison fingerprint mismatches", async () => { + const discarded = await runPair({ + jobId: "mismatch", + pair: 0, + order: "AB", + scenarioId: "changeset-scanning", + fixtureDirectory, + armA: { + id: "A", + packageRoot: armPackage, + operation: "change-processing", + }, + armASource: { ref: "base", sha: "a".repeat(40) }, + armB: { + id: "B", + packageRoot: armPackage, + operation: "change-processing", + }, + armBSource: { ref: "candidate", sha: "b".repeat(40) }, + outputDirectory: path.join(root, "mismatch"), + launcher: async (request) => ({ + ...armResult(request, [1, 2, 3]), + fixtureArtifactHash: "wrong", + }), + }); + expect(discarded.discardedReason).to.contain("malformed identity"); + expect(() => + assertComparisonFingerprintMatches( + { ...fingerprintForArtifact(fixtureDirectory), fixtureId: "wrong" }, + fingerprintForArtifact(fixtureDirectory) + ) + ).to.throw("Comparison fingerprint does not match"); + expect(hashFixtureArtifact(fixtureDirectory)).to.have.length(64); + }); + + it("aggregates three independent A/A jobs and enforces job-level order", async () => { + const artifacts = await Promise.all([ + validPair(0, "AB", [100, 101, 102], [101, 102, 103]), + validPair(1, "BA", [100, 101, 102], [99, 100, 101]), + validPair(2, "AB", [100, 101, 102], [100, 101, 102]), + ]); + const calibration = aggregateCalibration(artifacts); + expect(calibration.pool.independentJobs).to.equal(3); + expect(calibration.pool.observations).to.have.length(3); + expect(calibration.band.status).to.equal("established"); + expect(calibration.orders).to.deep.equal(["AB", "BA", "AB"]); + expect(() => + aggregateCalibration([ + artifacts[0], + { ...artifacts[1], order: "AB" }, + artifacts[2], + ]) + ).to.throw("requires BA"); + }); + + it("uses the layer-3 5% and 10% calibration quality boundaries", () => { + expect(classifyCalibrationQuality("established", 5)).to.equal("target"); + expect(classifyCalibrationQuality("established", 5.01)).to.equal( + "marginal" + ); + expect(classifyCalibrationQuality("established", 10)).to.equal( + "unresolvable" + ); + }); + + it("loads ChangedInstanceIds from the explicit arm package", async () => { + const baseline = createArmPackage("baseline-arm", 42); + const candidate = createArmPackage("candidate-arm", 84); + const loadedBaseline = loadArmModule( + resolveArmSpec({ + id: "baseline", + packageRoot: baseline, + operation: "change-processing", + }) + ); + const loadedCandidate = loadArmModule( + resolveArmSpec({ + id: "candidate", + packageRoot: candidate, + operation: "change-processing", + }) + ); + expect( + await loadedBaseline.changedInstanceIds.initialize(undefined as never) + ).to.equal(42); + expect( + await loadedCandidate.changedInstanceIds.initialize(undefined as never) + ).to.equal(84); + expect(loadedBaseline.runtime.transformerVersion).to.equal("1.0.42"); + }); +}); diff --git a/packages/performance-tests/test/quick/comparison/ComparisonRunner.ts b/packages/performance-tests/test/quick/comparison/ComparisonRunner.ts new file mode 100644 index 00000000..5957c78f --- /dev/null +++ b/packages/performance-tests/test/quick/comparison/ComparisonRunner.ts @@ -0,0 +1,728 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import { spawn } from "child_process"; +import * as crypto from "crypto"; +import * as fs from "fs"; +import { BriefcaseDb, IModelHost } from "@itwin/core-backend"; +import * as path from "path"; +import { + artifactBriefcasePath, + artifactManifestFileName, + FixtureArtifact, + readChangesetFileProps, + readFixtureArtifact, + readFixtureRecipeData, +} from "../FixtureArtifact"; +import type { PreparedDetachedDataset } from "../FixtureMaterializer"; +import { createChangesetScanningScenario } from "../scenarios/changesetScanning"; +import { + ArmRuntimeIdentity, + ArmSpec, + assertArmRuntimeComparable, + assertArmSpecsComparable, + loadArmModule, + resolveArmSpec, +} from "./ArmModule"; +import { classifyEnvironment, EnvironmentClass } from "./EnvironmentClass"; +import { + ExecutionFingerprint, + executionFingerprintKey, +} from "./ExecutionFingerprint"; +import { + CollapsedPair, + collapsePair, + logRatioToPercent, + PairObservation, + PairOrder, +} from "./logRatio"; +import { deriveNoiseBand, NoiseBand, NoiseBandPool } from "./NoiseBand"; + +export const comparisonScenarioId = "changeset-scanning"; +export const comparisonWarmups = 1; +export const comparisonMeasuredSamples = 3; +export const comparisonExecutionsPerPair = 8; +export const comparisonExecution: ExecutionFingerprint = { + warmupSamplesPerArm: comparisonWarmups, + measuredSamplesPerArm: comparisonMeasuredSamples, + processPolicy: { + kind: "one-process-per-arm", + restartBetweenPairs: true, + }, + pairPolicy: { kind: "paired", pairsPerJob: 1 }, + orderPolicy: { kind: "alternating", first: "AB" }, +}; + +export interface ComparisonFingerprint { + readonly scenarioId: string; + readonly fixtureId: string; + readonly recipeHash: string; + readonly workloadHash: string; + readonly execution: ExecutionFingerprint; +} + +export interface ArmSource { + readonly ref: string; + readonly sha: string; +} + +export interface ArmRunRequest { + readonly arm: ArmSpec; + readonly source: ArmSource; + readonly scenarioId: string; + readonly fixtureDirectory: string; + readonly fixtureArtifactHash: string; + readonly fingerprint: ComparisonFingerprint; + readonly outputDirectory: string; +} + +export interface RawArmSample { + readonly sample: number; + readonly measured: boolean; + readonly wallMilliseconds: number; + readonly semanticDigest: string; + readonly reconstructionMilliseconds: number; + readonly verificationMilliseconds: number; + readonly teardownMilliseconds: number; +} + +export interface ArmRunResult { + readonly arm: ArmSpec; + readonly source: ArmSource; + readonly runtime: ArmRuntimeIdentity; + readonly fingerprint: ComparisonFingerprint; + readonly fixtureArtifactHash: string; + readonly samples: readonly RawArmSample[]; + readonly generatedAt: string; +} + +export interface PairRunArtifact { + readonly jobId: string; + readonly pair: number; + readonly order: PairOrder; + readonly fingerprint: ComparisonFingerprint; + readonly environment: EnvironmentClass; + readonly fixtureArtifactHash: string; + readonly armA?: ArmRunResult; + readonly armB?: ArmRunResult; + readonly observation?: PairObservation; + readonly collapsed?: CollapsedPair; + readonly discardedReason?: string; + readonly generatedAt: string; +} + +export interface CalibrationArtifact { + readonly fingerprint: ComparisonFingerprint; + readonly environment: EnvironmentClass; + readonly pool: NoiseBandPool; + readonly band: NoiseBand; + readonly refs: readonly ArmSource[]; + readonly orders: readonly PairOrder[]; + readonly generatedAt: string; +} + +export type ArmLauncher = ( + request: ArmRunRequest, + timeoutMilliseconds: number +) => Promise; + +function canonicalJson(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) + return `[${value.map((entry) => canonicalJson(entry)).join(",")}]`; + return `{${Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`) + .join(",")}}`; +} + +function sha256(value: string | Buffer): string { + return crypto.createHash("sha256").update(value).digest("hex"); +} + +function normalizeError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} + +export function comparisonFingerprintKey( + fingerprint: ComparisonFingerprint +): string { + return canonicalJson({ + scenarioId: fingerprint.scenarioId, + fixtureId: fingerprint.fixtureId, + recipeHash: fingerprint.recipeHash, + workloadHash: fingerprint.workloadHash, + execution: executionFingerprintKey(fingerprint.execution), + }); +} + +export function assertComparisonFingerprintMatches( + actual: ComparisonFingerprint, + expected: ComparisonFingerprint +): void { + const actualKey = comparisonFingerprintKey(actual); + const expectedKey = comparisonFingerprintKey(expected); + if (actualKey !== expectedKey) + throw new Error( + `Comparison fingerprint does not match: ${actualKey} != ${expectedKey}` + ); +} + +export function fingerprintForArtifact( + fixtureDirectory: string +): ComparisonFingerprint { + const { descriptor } = readFixtureArtifact(fixtureDirectory).manifest; + return { + scenarioId: comparisonScenarioId, + fixtureId: descriptor.id, + recipeHash: descriptor.recipeHash, + workloadHash: sha256(canonicalJson(descriptor.distribution)), + execution: comparisonExecution, + }; +} + +export function hashFixtureArtifact(directory: string): string { + const files: string[] = []; + const visit = (current: string): void => { + for (const entry of fs.readdirSync(current, { withFileTypes: true })) { + const absolute = path.join(current, entry.name); + if (entry.isDirectory()) visit(absolute); + else if (entry.isFile()) files.push(absolute); + } + }; + visit(directory); + const hash = crypto.createHash("sha256"); + for (const file of files.sort()) { + hash.update(path.relative(directory, file).split(path.sep).join("/")); + hash.update("\0"); + hash.update(fs.readFileSync(file)); + hash.update("\0"); + } + return hash.digest("hex"); +} + +async function withHost( + profileName: string, + operation: () => Promise +): Promise { + await IModelHost.startup({ profileName }); + const profileDir = IModelHost.profileDir; + try { + return await operation(); + } finally { + await IModelHost.shutdown(); + if (profileDir.includes(profileName)) + fs.rmSync(profileDir, { recursive: true, force: true }); + } +} + +async function materializeArtifact( + artifact: FixtureArtifact, + sampleDirectory: string +): Promise { + const start = process.hrtime.bigint(); + fs.rmSync(sampleDirectory, { recursive: true, force: true }); + fs.mkdirSync(path.dirname(sampleDirectory), { recursive: true }); + fs.cpSync(artifact.directory, sampleDirectory, { recursive: true }); + const sourceDb = await BriefcaseDb.open({ + fileName: artifactBriefcasePath(sampleDirectory), + readonly: true, + }); + return { + topology: "source-only", + descriptor: artifact.manifest.descriptor, + directory: sampleDirectory, + sourceDb, + csFileProps: readChangesetFileProps(sampleDirectory), + manifest: artifact.manifest, + recipe: readFixtureRecipeData(sampleDirectory, artifact.manifest), + reconstructionMilliseconds: + Number(process.hrtime.bigint() - start) / 1_000_000, + }; +} + +function validateArmResult( + value: unknown, + request: ArmRunRequest +): ArmRunResult { + if (value === null || typeof value !== "object") + throw new Error(`Arm "${request.arm.id}" returned malformed output`); + const result = value as Partial; + if ( + !Array.isArray(result.samples) || + result.samples.length !== comparisonWarmups + comparisonMeasuredSamples || + result.samples.filter((sample) => sample.measured).length !== + comparisonMeasuredSamples || + result.samples[0]?.measured !== false + ) + throw new Error( + `Arm "${request.arm.id}" did not return exactly one warm-up and three measured samples` + ); + for (const [index, sample] of result.samples.entries()) { + if ( + sample.sample !== index || + typeof sample.wallMilliseconds !== "number" || + !Number.isFinite(sample.wallMilliseconds) || + sample.wallMilliseconds <= 0 || + typeof sample.semanticDigest !== "string" + ) + throw new Error( + `Arm "${request.arm.id}" returned malformed sample ${index}` + ); + } + if ( + !result.runtime || + result.arm?.id !== request.arm.id || + result.runtime.armId !== request.arm.id || + result.source?.ref !== request.source.ref || + result.source.sha !== request.source.sha || + result.fixtureArtifactHash !== request.fixtureArtifactHash || + !result.fingerprint + ) + throw new Error(`Arm "${request.arm.id}" returned malformed identity data`); + assertComparisonFingerprintMatches(result.fingerprint, request.fingerprint); + return result as ArmRunResult; +} + +export async function runArm(request: ArmRunRequest): Promise { + if (request.scenarioId !== comparisonScenarioId) + throw new Error( + `Unsupported comparison scenario "${request.scenarioId}"; only "${comparisonScenarioId}" is implemented` + ); + const artifact = readFixtureArtifact(request.fixtureDirectory); + const artifactHash = hashFixtureArtifact(request.fixtureDirectory); + if (artifactHash !== request.fixtureArtifactHash) + throw new Error( + `Fixture artifact hash changed: ${artifactHash} != ${request.fixtureArtifactHash}` + ); + const actualFingerprint = fingerprintForArtifact(request.fixtureDirectory); + assertComparisonFingerprintMatches(actualFingerprint, request.fingerprint); + const resolvedArm = resolveArmSpec(request.arm); + const loaded = loadArmModule(resolvedArm); + const samples: RawArmSample[] = []; + await withHost( + `quick-compare-${request.arm.id.replace(/[^A-Za-z0-9_-]/g, "-")}-${ + process.pid + }`, + async () => { + for ( + let sample = 0; + sample < comparisonWarmups + comparisonMeasuredSamples; + sample++ + ) { + const sampleDirectory = path.join( + request.outputDirectory, + `sample-${sample}` + ); + const dataset = await materializeArtifact(artifact, sampleDirectory); + const scenario = createChangesetScanningScenario( + loaded.changedInstanceIds + ).factory(dataset); + let operationError: unknown; + let completed: Omit | undefined; + try { + const start = process.hrtime.bigint(); + await scenario.measure(); + const wallMilliseconds = + Number(process.hrtime.bigint() - start) / 1_000_000; + const verificationStart = process.hrtime.bigint(); + const semanticDigest = await scenario.finish(); + const verificationMilliseconds = + Number(process.hrtime.bigint() - verificationStart) / 1_000_000; + completed = { + sample, + measured: sample >= comparisonWarmups, + wallMilliseconds, + semanticDigest, + reconstructionMilliseconds: dataset.reconstructionMilliseconds, + verificationMilliseconds, + }; + } catch (error) { + operationError = error; + } + const teardownStart = process.hrtime.bigint(); + let cleanupError: unknown; + try { + if (operationError) scenario.abort(); + dataset.sourceDb.close(); + fs.rmSync(sampleDirectory, { recursive: true, force: true }); + } catch (error) { + cleanupError = error; + } + const teardownMilliseconds = + Number(process.hrtime.bigint() - teardownStart) / 1_000_000; + if (operationError && cleanupError) + throw new AggregateError( + [operationError, cleanupError], + `Arm "${request.arm.id}" sample ${sample} and cleanup both failed` + ); + if (operationError) throw normalizeError(operationError); + if (cleanupError) throw normalizeError(cleanupError); + if (!completed) + throw new Error( + `Arm "${request.arm.id}" sample ${sample} produced no result` + ); + samples.push({ ...completed, teardownMilliseconds }); + } + } + ); + const digests = new Set(samples.map((sample) => sample.semanticDigest)); + if (digests.size !== 1) + throw new Error(`Arm "${request.arm.id}" produced inconsistent semantics`); + return { + arm: request.arm, + source: request.source, + runtime: loaded.runtime, + fingerprint: actualFingerprint, + fixtureArtifactHash: artifactHash, + samples, + generatedAt: new Date().toISOString(), + }; +} + +export async function spawnArmProcess( + armProcessPath: string, + request: ArmRunRequest, + timeoutMilliseconds: number +): Promise { + fs.mkdirSync(request.outputDirectory, { recursive: true }); + const requestFile = path.join(request.outputDirectory, "request.json"); + const resultFile = path.join(request.outputDirectory, "arm-result.json"); + fs.writeFileSync(requestFile, `${JSON.stringify(request, undefined, 2)}\n`); + return new Promise((resolve, reject) => { + const child = spawn( + process.execPath, + [armProcessPath, "--request", requestFile, "--output", resultFile], + { stdio: ["ignore", "inherit", "inherit"], shell: false } + ); + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + child.kill(); + }, timeoutMilliseconds); + child.once("error", (error) => { + clearTimeout(timer); + reject(error); + }); + child.once("exit", (code, signal) => { + clearTimeout(timer); + if (timedOut) + reject( + new Error( + `Arm "${request.arm.id}" timed out after ${timeoutMilliseconds}ms` + ) + ); + else if (code !== 0) + reject( + new Error( + `Arm "${request.arm.id}" exited with code ${String( + code + )} and signal ${String(signal)}` + ) + ); + else if (!fs.existsSync(resultFile)) + reject(new Error(`Arm "${request.arm.id}" produced no result file`)); + else { + try { + resolve(JSON.parse(fs.readFileSync(resultFile, "utf8"))); + } catch (error) { + reject( + new Error(`Arm "${request.arm.id}" produced malformed JSON`, { + cause: error, + }) + ); + } + } + }); + }); +} + +export async function runPair(options: { + readonly jobId: string; + readonly pair: number; + readonly order: PairOrder; + readonly scenarioId: string; + readonly fixtureDirectory: string; + readonly armA: ArmSpec; + readonly armASource: ArmSource; + readonly armB: ArmSpec; + readonly armBSource: ArmSource; + readonly outputDirectory: string; + readonly armProcessPath?: string; + readonly timeoutMilliseconds?: number; + readonly launcher?: ArmLauncher; +}): Promise { + const generatedAt = new Date().toISOString(); + const environment = classifyEnvironment(); + const fixtureArtifactHash = hashFixtureArtifact(options.fixtureDirectory); + const fingerprint = fingerprintForArtifact(options.fixtureDirectory); + if (options.scenarioId !== comparisonScenarioId) + throw new Error( + `Unsupported comparison scenario "${options.scenarioId}"; only "${comparisonScenarioId}" is implemented` + ); + if (options.order !== "AB" && options.order !== "BA") + throw new Error( + `Pair order must be AB or BA, received "${String(options.order)}"` + ); + if (options.jobId.trim().length === 0) + throw new Error("Pair job id cannot be empty"); + const scheduledOrder: PairOrder = options.pair % 2 === 0 ? "AB" : "BA"; + if (options.order !== scheduledOrder) + throw new Error( + `Pair ${options.pair} must use ${scheduledOrder}, received ${options.order}` + ); + const resolvedA = resolveArmSpec(options.armA); + const resolvedB = resolveArmSpec(options.armB); + assertArmSpecsComparable(resolvedA, resolvedB); + const requests: Record<"A" | "B", ArmRunRequest> = { + A: { + arm: options.armA, + source: options.armASource, + scenarioId: options.scenarioId, + fixtureDirectory: options.fixtureDirectory, + fixtureArtifactHash, + fingerprint, + outputDirectory: path.join(options.outputDirectory, "arm-a"), + }, + B: { + arm: options.armB, + source: options.armBSource, + scenarioId: options.scenarioId, + fixtureDirectory: options.fixtureDirectory, + fixtureArtifactHash, + fingerprint, + outputDirectory: path.join(options.outputDirectory, "arm-b"), + }, + }; + const launcher = + options.launcher ?? + (async (request, timeout) => + spawnArmProcess( + options.armProcessPath ?? path.join(__dirname, "armProcessCli.js"), + request, + timeout + )); + const results: Partial> = {}; + try { + for (const label of options.order.split("") as ("A" | "B")[]) { + const request = requests[label]; + results[label] = validateArmResult( + await launcher(request, options.timeoutMilliseconds ?? 10 * 60 * 1000), + request + ); + } + const armA = results.A; + const armB = results.B; + if (!armA || !armB) throw new Error("Pair did not execute both arms"); + assertArmRuntimeComparable(armA.runtime, armB.runtime); + if (armA.fixtureArtifactHash !== armB.fixtureArtifactHash) + throw new Error("Arms consumed different fixture artifact bytes"); + if ( + new Set([ + ...armA.samples.map((sample) => sample.semanticDigest), + ...armB.samples.map((sample) => sample.semanticDigest), + ]).size !== 1 + ) + throw new Error("Arms produced different semantic results"); + const armASamples = armA.samples + .filter((sample) => sample.measured) + .map((sample) => sample.wallMilliseconds); + const armBSamples = armB.samples + .filter((sample) => sample.measured) + .map((sample) => sample.wallMilliseconds); + const observation: PairObservation = { + pair: options.pair, + order: options.order, + armASamples, + armBSamples, + }; + return { + jobId: options.jobId, + pair: options.pair, + order: options.order, + fingerprint, + environment, + fixtureArtifactHash, + armA, + armB, + observation, + collapsed: collapsePair(observation), + generatedAt, + }; + } catch (error) { + return { + jobId: options.jobId, + pair: options.pair, + order: options.order, + fingerprint, + environment, + fixtureArtifactHash, + armA: results.A, + armB: results.B, + discardedReason: error instanceof Error ? error.message : String(error), + generatedAt, + }; + } +} + +export function aggregateCalibration( + artifacts: readonly PairRunArtifact[] +): CalibrationArtifact { + if (artifacts.length < 3) + throw new Error( + "Calibration requires at least three independent job artifacts" + ); + const first = artifacts[0]; + const jobs = new Set(); + const observations: number[] = []; + const refs: ArmSource[] = []; + for (const artifact of artifacts) { + if (jobs.has(artifact.jobId)) + throw new Error(`Duplicate calibration job id: ${artifact.jobId}`); + jobs.add(artifact.jobId); + assertComparisonFingerprintMatches(artifact.fingerprint, first.fingerprint); + if (artifact.environment.id !== first.environment.id) + throw new Error( + `Calibration environment mismatch: ${artifact.environment.id} != ${first.environment.id}` + ); + if (artifact.discardedReason || !artifact.collapsed) + throw new Error( + `Calibration pair ${artifact.jobId} was discarded: ${ + artifact.discardedReason ?? "missing collapsed observation" + }` + ); + const expectedOrder: PairOrder = artifact.pair % 2 === 0 ? "AB" : "BA"; + if (artifact.order !== expectedOrder) + throw new Error( + `Calibration job ${artifact.jobId} used ${artifact.order}; pair ${artifact.pair} requires ${expectedOrder}` + ); + if ( + !artifact.armA || + !artifact.armB || + artifact.armA.source.sha !== artifact.armB.source.sha || + artifact.armA.runtime.transformerVersion !== + artifact.armB.runtime.transformerVersion || + artifact.armA.runtime.transformerPackageHash !== + artifact.armB.runtime.transformerPackageHash + ) + throw new Error( + `Calibration pair ${artifact.jobId} did not use the same build for both labeled arms` + ); + observations.push(artifact.collapsed.logRatio); + refs.push(artifact.armA.source); + } + if (refs.some((source) => source.sha !== refs[0].sha)) + throw new Error( + "Calibration jobs did not use one identical calibration build" + ); + const pool: NoiseBandPool = { + scenarioId: first.fingerprint.scenarioId, + fixtureId: first.fingerprint.fixtureId, + recipeHash: first.fingerprint.recipeHash, + environmentClass: first.environment.id, + execution: first.fingerprint.execution, + kind: "paired", + observations, + independentJobs: jobs.size, + updatedAt: new Date().toISOString(), + }; + return { + fingerprint: first.fingerprint, + environment: first.environment, + pool, + band: deriveNoiseBand(pool, 1), + refs, + orders: artifacts.map((artifact) => artifact.order), + generatedAt: new Date().toISOString(), + }; +} + +export function renderCalibration(calibration: CalibrationArtifact): string { + return [ + "# Quick performance A/A calibration", + "", + `**Status:** ${calibration.band.status}`, + "", + `**Quality:** ${calibration.band.quality} (informational)`, + "", + "| Property | Value |", + "|---|---|", + `| Scenario | ${calibration.fingerprint.scenarioId} |`, + `| Fixture | ${calibration.fingerprint.fixtureId} |`, + `| Recipe hash | ${calibration.fingerprint.recipeHash} |`, + `| Fingerprint key | \`${sha256( + comparisonFingerprintKey(calibration.fingerprint) + )}\` |`, + `| Independent jobs | ${calibration.pool.independentJobs} |`, + `| Calibration ref / SHA | ${calibration.refs[0]?.ref ?? "n/a"} / ${ + calibration.refs[0]?.sha ?? "n/a" + } |`, + `| Pair orders | ${calibration.orders.join(", ")} |`, + `| Pair log-ratios | ${calibration.pool.observations + .map((value) => value.toFixed(6)) + .join(", ")} |`, + `| A/A band | ${calibration.band.bandPercent.toFixed(2)}% |`, + "| Target | <=5% actionable; 5-10% marginal; >=10% unresolvable |", + "", + "This calibration is informational and does not create a merge-blocking gate.", + "", + ].join("\n"); +} + +export function renderPairSummary(artifact: PairRunArtifact): string { + const lines = [ + "# Quick performance pair", + "", + `**Status:** ${artifact.discardedReason ? "DISCARDED" : "VALID"}`, + "", + "| Property | Value |", + "|---|---|", + `| Job | ${artifact.jobId} |`, + `| Scenario | ${artifact.fingerprint.scenarioId} |`, + `| Fixture | ${artifact.fingerprint.fixtureId} |`, + `| Fixture artifact hash | ${artifact.fixtureArtifactHash} |`, + `| Recipe hash | ${artifact.fingerprint.recipeHash} |`, + `| Fingerprint key | \`${sha256( + comparisonFingerprintKey(artifact.fingerprint) + )}\` |`, + `| Order | ${artifact.order} |`, + `| Arm A ref / SHA | ${artifact.armA?.source.ref ?? "n/a"} / ${ + artifact.armA?.source.sha ?? "n/a" + } |`, + `| Arm B ref / SHA | ${artifact.armB?.source.ref ?? "n/a"} / ${ + artifact.armB?.source.sha ?? "n/a" + } |`, + `| Arm A median | ${artifact.collapsed?.armA.toFixed(3) ?? "n/a"} ms |`, + `| Arm B median | ${artifact.collapsed?.armB.toFixed(3) ?? "n/a"} ms |`, + `| Delta | ${ + artifact.collapsed + ? `${logRatioToPercent(artifact.collapsed.logRatio).toFixed(2)}%` + : "n/a" + } |`, + ]; + if (artifact.discardedReason) + lines.push("", `Discarded pair: ${artifact.discardedReason}`); + lines.push( + "", + "This result is informational and does not create a merge-blocking gate.", + "" + ); + return lines.join("\n"); +} + +export function readPairArtifact(fileName: string): PairRunArtifact { + return JSON.parse(fs.readFileSync(fileName, "utf8")) as PairRunArtifact; +} + +export function readCalibrationArtifact(fileName: string): CalibrationArtifact { + return JSON.parse(fs.readFileSync(fileName, "utf8")) as CalibrationArtifact; +} + +export function writeJson(fileName: string, value: unknown): void { + fs.mkdirSync(path.dirname(fileName), { recursive: true }); + fs.writeFileSync(fileName, `${JSON.stringify(value, undefined, 2)}\n`); +} + +export { artifactManifestFileName }; diff --git a/packages/performance-tests/test/quick/comparison/EnvironmentClass.ts b/packages/performance-tests/test/quick/comparison/EnvironmentClass.ts index 781ca5b9..04bc2f60 100644 --- a/packages/performance-tests/test/quick/comparison/EnvironmentClass.ts +++ b/packages/performance-tests/test/quick/comparison/EnvironmentClass.ts @@ -23,6 +23,8 @@ export interface EnvironmentDescriptor { readonly memoryGibBucket: number; readonly nodeMajor: number; readonly runner: string; + /** Immutable hosted image identity when the runner exposes one. */ + readonly runnerImage?: string; } export interface EnvironmentClass { @@ -45,8 +47,19 @@ export function describeEnvironment(): EnvironmentDescriptor { memoryGibBucket: memoryBucketGib(os.totalmem()), nodeMajor: Number(process.versions.node.split(".")[0]), runner: process.env.GITHUB_ACTIONS - ? (process.env.RUNNER_NAME ?? process.env.RUNNER_OS ?? "github-hosted") + ? process.env.RUNNER_ENVIRONMENT === "github-hosted" + ? `github-hosted:${process.env.RUNNER_OS ?? process.platform}:${ + process.env.RUNNER_ARCH ?? process.arch + }` + : `self-hosted:${ + process.env.RUNNER_NAME ?? process.env.RUNNER_OS ?? "unknown" + }` : "local", + runnerImage: process.env.GITHUB_ACTIONS + ? `${process.env.ImageOS ?? process.env.RUNNER_OS ?? "unknown"}:${ + process.env.ImageVersion ?? "unknown" + }` + : undefined, }; } @@ -67,6 +80,7 @@ export function classifyEnvironment( descriptor.memoryGibBucket, descriptor.nodeMajor, descriptor.runner, + descriptor.runnerImage ?? "none", ].join("|"); return { id: crypto diff --git a/packages/performance-tests/test/quick/comparison/armProcessCli.ts b/packages/performance-tests/test/quick/comparison/armProcessCli.ts new file mode 100644 index 00000000..bf5b761a --- /dev/null +++ b/packages/performance-tests/test/quick/comparison/armProcessCli.ts @@ -0,0 +1,43 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "fs"; +import * as path from "path"; +import { + aliasHarnessCoreBackendToArm, + ArmSpec, + resolveArmSpec, +} from "./ArmModule"; + +interface ArmRequestFile { + readonly arm: ArmSpec; +} + +function argument(name: string): string { + const index = process.argv.indexOf(`--${name}`); + const value = index < 0 ? undefined : process.argv[index + 1]; + if (!value) throw new Error(`Missing required argument --${name}`); + return value; +} + +async function main(): Promise { + const requestFile = argument("request"); + const outputFile = argument("output"); + const request = JSON.parse( + fs.readFileSync(requestFile, "utf8") + ) as ArmRequestFile; + aliasHarnessCoreBackendToArm(resolveArmSpec(request.arm)); + const { runArm } = await import("./ComparisonRunner"); + const result = await runArm(request as Parameters[0]); + fs.mkdirSync(path.dirname(outputFile), { recursive: true }); + fs.writeFileSync(outputFile, `${JSON.stringify(result, undefined, 2)}\n`); +} + +void main().catch((error) => { + process.stderr.write( + `${error instanceof Error ? (error.stack ?? error.message) : String(error)}\n` + ); + process.exitCode = 1; +}); diff --git a/packages/performance-tests/test/quick/comparison/comparison.quick-unit.ts b/packages/performance-tests/test/quick/comparison/comparison.quick-unit.ts index c8fda127..27898654 100644 --- a/packages/performance-tests/test/quick/comparison/comparison.quick-unit.ts +++ b/packages/performance-tests/test/quick/comparison/comparison.quick-unit.ts @@ -627,6 +627,7 @@ describe("quick performance comparison statistics", () => { const identity: ArmRuntimeIdentity = { armId: "A", transformerVersion: "2.0.0", + transformerPackageHash: "transformer", coreBackendVersion: "5.10.3", coreBackendPackageHash: "abc", }; diff --git a/packages/performance-tests/test/quick/comparison/comparisonCli.ts b/packages/performance-tests/test/quick/comparison/comparisonCli.ts new file mode 100644 index 00000000..a7e3bccc --- /dev/null +++ b/packages/performance-tests/test/quick/comparison/comparisonCli.ts @@ -0,0 +1,289 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "fs"; +import * as crypto from "crypto"; +import { spawnSync } from "child_process"; +import * as path from "path"; +import { + buildComparisonReport, + writeComparisonReport, +} from "./ComparisonReport"; +import { + aggregateCalibration, + assertComparisonFingerprintMatches, + comparisonFingerprintKey, + comparisonScenarioId, + readCalibrationArtifact, + readPairArtifact, + renderCalibration, + renderPairSummary, + runPair, + writeJson, +} from "./ComparisonRunner"; +import { prepareComparisonFixture } from "./ComparisonFixture"; +import { ArmOperation, ArmSpec } from "./ArmModule"; + +function parseArguments(args: readonly string[]): Map { + const parsed = new Map(); + for (let index = 0; index < args.length; index += 2) { + const key = args[index]; + const value = args[index + 1]; + if (!key?.startsWith("--") || value === undefined) + throw new Error( + `Expected --name value, received "${key ?? ""}"` + ); + if (parsed.has(key)) throw new Error(`Duplicate argument: ${key}`); + parsed.set(key, value); + } + return parsed; +} + +function required(args: Map, name: string): string { + const value = args.get(`--${name}`); + if (!value) throw new Error(`Missing required argument --${name}`); + return value; +} + +function optional(args: Map, name: string): string | undefined { + return args.get(`--${name}`); +} + +function integer(args: Map, name: string): number { + const raw = required(args, name); + const value = Number(raw); + if (!Number.isInteger(value) || value < 0) + throw new Error(`--${name} must be a non-negative integer`); + return value; +} + +function assertScenario(args: Map): string { + const scenario = required(args, "scenario"); + if (scenario !== comparisonScenarioId) + throw new Error( + `Unsupported comparison scenario "${scenario}"; only "${comparisonScenarioId}" is implemented` + ); + return scenario; +} + +function armSpec( + args: Map, + prefix: "arm-a" | "arm-b" +): ArmSpec { + const operation: ArmOperation = "change-processing"; + return { + id: required(args, `${prefix}-id`), + label: optional(args, `${prefix}-label`), + packageRoot: required(args, `${prefix}-package`), + modulePath: optional(args, `${prefix}-module`), + operation, + }; +} + +function findFiles(directory: string, name: string): string[] { + const files: string[] = []; + const visit = (current: string): void => { + for (const entry of fs.readdirSync(current, { withFileTypes: true })) { + const absolute = path.join(current, entry.name); + if (entry.isDirectory()) visit(absolute); + else if (entry.isFile() && entry.name === name) files.push(absolute); + } + }; + visit(directory); + return files.sort(); +} + +async function prepareFixture(args: Map): Promise { + assertScenario(args); + const output = required(args, "output"); + const result = await prepareComparisonFixture( + output, + optional(args, "smoke") === "true" + ); + process.stdout.write(`${JSON.stringify(result)}\n`); +} + +function executeArm(args: Map): void { + const result = spawnSync( + process.execPath, + [ + path.join(__dirname, "armProcessCli.js"), + "--request", + required(args, "request"), + "--output", + required(args, "output"), + ], + { shell: false, stdio: "inherit" } + ); + if (result.error) throw result.error; + if (result.status !== 0) + throw new Error( + `Isolated arm process exited with code ${String(result.status)}` + ); +} + +async function executePair(args: Map): Promise { + const output = required(args, "output"); + const artifact = await runPair({ + jobId: required(args, "job-id"), + pair: integer(args, "pair"), + order: required(args, "order") as "AB" | "BA", + scenarioId: assertScenario(args), + fixtureDirectory: required(args, "fixture"), + armA: armSpec(args, "arm-a"), + armASource: { + ref: required(args, "arm-a-ref"), + sha: required(args, "arm-a-sha"), + }, + armB: armSpec(args, "arm-b"), + armBSource: { + ref: required(args, "arm-b-ref"), + sha: required(args, "arm-b-sha"), + }, + outputDirectory: output, + timeoutMilliseconds: optional(args, "timeout-ms") + ? integer(args, "timeout-ms") + : undefined, + }); + writeJson(path.join(output, "pair-observation.json"), artifact); + fs.writeFileSync( + path.join(output, "pair-summary.md"), + renderPairSummary(artifact) + ); + if (artifact.discardedReason) + throw new Error(`Pair discarded: ${artifact.discardedReason}`); +} + +function aggregate(args: Map): void { + const input = required(args, "input"); + const output = required(args, "output"); + const files = findFiles(input, "pair-observation.json"); + const calibration = aggregateCalibration(files.map(readPairArtifact)); + fs.mkdirSync(output, { recursive: true }); + writeJson(path.join(output, "calibration.json"), calibration); + writeJson(path.join(output, "noise-pool.json"), calibration.pool); + writeJson(path.join(output, "noise-band.json"), calibration.band); + fs.writeFileSync( + path.join(output, "calibration-summary.md"), + renderCalibration(calibration) + ); +} + +function compare(args: Map): void { + const pair = readPairArtifact(required(args, "observation")); + const calibration = readCalibrationArtifact(required(args, "calibration")); + const output = required(args, "output"); + if (pair.discardedReason || !pair.collapsed || !pair.armA || !pair.armB) + throw new Error( + `Cannot compare a discarded pair: ${ + pair.discardedReason ?? "missing arm or collapsed result" + }` + ); + assertComparisonFingerprintMatches(pair.fingerprint, calibration.fingerprint); + if (pair.environment.id !== calibration.environment.id) + throw new Error( + `Comparison environment ${pair.environment.id} does not match calibration ${calibration.environment.id}` + ); + const baseReport = buildComparisonReport({ + scenarioId: pair.fingerprint.scenarioId, + fixtureId: pair.fingerprint.fixtureId, + recipeHash: pair.fingerprint.recipeHash, + mode: "paired", + environment: pair.environment, + execution: pair.fingerprint.execution, + armA: { + id: pair.armA.arm.id, + label: pair.armA.arm.label, + ref: pair.armA.source.ref, + sha: pair.armA.source.sha, + transformerVersion: pair.armA.runtime.transformerVersion, + transformerPackageHash: pair.armA.runtime.transformerPackageHash, + coreBackendVersion: pair.armA.runtime.coreBackendVersion, + coreBackendPackageHash: pair.armA.runtime.coreBackendPackageHash, + }, + armB: { + id: pair.armB.arm.id, + label: pair.armB.arm.label, + ref: pair.armB.source.ref, + sha: pair.armB.source.sha, + transformerVersion: pair.armB.runtime.transformerVersion, + transformerPackageHash: pair.armB.runtime.transformerPackageHash, + coreBackendVersion: pair.armB.runtime.coreBackendVersion, + coreBackendPackageHash: pair.armB.runtime.coreBackendPackageHash, + }, + pairs: [pair.collapsed], + independentJobs: 1, + pool: calibration.pool, + band: calibration.band, + }); + const report = { + ...baseReport, + verdict: { + ...baseReport.verdict, + evidence: "informational" as const, + reason: `${baseReport.verdict.reason} This workflow is informational-only.`, + }, + }; + writeComparisonReport(output, report); + fs.appendFileSync( + path.join(output, "comparison.md"), + [ + "", + "## Runner identity", + "", + `- Fingerprint key: \`${crypto + .createHash("sha256") + .update(comparisonFingerprintKey(pair.fingerprint)) + .digest("hex")}\``, + `- Fixture artifact hash: \`${pair.fixtureArtifactHash}\``, + `- Pair order: ${pair.order}`, + `- Arm A median: ${pair.collapsed.armA.toFixed(3)} ms`, + `- Arm B median: ${pair.collapsed.armB.toFixed(3)} ms`, + "", + "This workflow is informational-only and never creates a merge-blocking performance gate.", + "", + ].join("\n") + ); + writeJson(path.join(output, "summary.json"), { + report, + pair, + calibration: { + fingerprint: calibration.fingerprint, + environment: calibration.environment, + band: calibration.band, + }, + }); +} + +async function main(): Promise { + const command = process.argv[2]; + const args = parseArguments(process.argv.slice(3)); + switch (command) { + case "prepare-fixture": + await prepareFixture(args); + return; + case "run-arm": + executeArm(args); + return; + case "run-pair": + await executePair(args); + return; + case "aggregate-calibration": + aggregate(args); + return; + case "compare": + compare(args); + return; + default: + throw new Error(`Unknown comparison command: ${command ?? ""}`); + } +} + +void main().catch((error) => { + process.stderr.write( + `${error instanceof Error ? (error.stack ?? error.message) : String(error)}\n` + ); + process.exitCode = 1; +}); diff --git a/packages/performance-tests/test/quick/quickSource.ts b/packages/performance-tests/test/quick/quickSource.ts new file mode 100644 index 00000000..c91fb701 --- /dev/null +++ b/packages/performance-tests/test/quick/quickSource.ts @@ -0,0 +1,16 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "fs"; +import * as path from "path"; + +/** Resolve a versioned quick-harness source asset from ts-node or compiled `lib` output. */ +export function resolveQuickSourceFile(relativePath: string): string { + const adjacent = path.join(__dirname, relativePath); + if (fs.existsSync(adjacent)) return adjacent; + const compiledSource = path.join(__dirname, "../../test/quick", relativePath); + if (fs.existsSync(compiledSource)) return compiledSource; + throw new Error(`Quick performance source input is missing: ${relativePath}`); +} diff --git a/packages/performance-tests/test/quick/recipes/balancedIncremental.ts b/packages/performance-tests/test/quick/recipes/balancedIncremental.ts index 87fc6ec6..e04ffcbb 100644 --- a/packages/performance-tests/test/quick/recipes/balancedIncremental.ts +++ b/packages/performance-tests/test/quick/recipes/balancedIncremental.ts @@ -3,7 +3,6 @@ * See LICENSE.md in the project root for license terms and full copyright notice. *--------------------------------------------------------------------------------------------*/ -import * as path from "path"; import { Id64String } from "@itwin/core-bentley"; import { Code, @@ -30,6 +29,7 @@ import { YawPitchRollAngles, } from "@itwin/core-geometry"; import { DatasetDescriptor } from "../DatasetDescriptor"; +import { resolveQuickSourceFile } from "../quickSource"; const uniqueAspectClass = "QuickPerf:BalancedUniqueAspect"; const multiAspectClass = "QuickPerf:BalancedMultiAspect"; @@ -127,7 +127,7 @@ export async function createBalancedSeed( }); try { await db.importSchemas([ - path.join(__dirname, "../schemas/QuickPerf.ecschema.xml"), + resolveQuickSourceFile("schemas/QuickPerf.ecschema.xml"), ]); const { categoryIds, modelIds } = withEditTxn( diff --git a/packages/performance-tests/test/quick/recipes/updateHeavyScan.ts b/packages/performance-tests/test/quick/recipes/updateHeavyScan.ts index 253280c3..e90e3a21 100644 --- a/packages/performance-tests/test/quick/recipes/updateHeavyScan.ts +++ b/packages/performance-tests/test/quick/recipes/updateHeavyScan.ts @@ -3,7 +3,6 @@ * See LICENSE.md in the project root for license terms and full copyright notice. *--------------------------------------------------------------------------------------------*/ -import * as path from "path"; import { AccessToken, Id64String } from "@itwin/core-bentley"; import { Code, @@ -25,6 +24,7 @@ import { withEditTxn, } from "@itwin/core-backend"; import { DatasetDescriptor } from "../DatasetDescriptor"; +import { resolveQuickSourceFile } from "../quickSource"; import { ScanLedger, ScanLedgerEntry } from "../validation/scanOracle"; import { queryCount } from "../validation/validateFixture"; @@ -160,7 +160,7 @@ export async function createScanSeed( }); try { await db.importSchemas([ - path.join(__dirname, "../schemas/QuickPerfScan.ecschema.xml"), + resolveQuickSourceFile("schemas/QuickPerfScan.ecschema.xml"), ]); const { categoryId, modelId } = withEditTxn( diff --git a/packages/performance-tests/test/quick/scenarios/changesetScanning.ts b/packages/performance-tests/test/quick/scenarios/changesetScanning.ts index 74b06bd4..61be0686 100644 --- a/packages/performance-tests/test/quick/scenarios/changesetScanning.ts +++ b/packages/performance-tests/test/quick/scenarios/changesetScanning.ts @@ -3,7 +3,9 @@ * See LICENSE.md in the project root for license terms and full copyright notice. *--------------------------------------------------------------------------------------------*/ -import { ChangedInstanceIds } from "@itwin/imodel-transformer"; +import type { BriefcaseDb } from "@itwin/core-backend"; +import type { ChangesetFileProps } from "@itwin/core-common"; +import { createRequire } from "module"; import { BenchmarkScenarioDefinition } from "../BenchmarkScenario"; import { PreparedDataset, @@ -19,6 +21,23 @@ import { squashLedger, } from "../validation/scanOracle"; +const workspaceRequire = createRequire(__filename); + +export interface ChangedInstanceIdsDependency { + initialize(args: { + readonly csFileProps: ChangesetFileProps[]; + readonly iModel: BriefcaseDb; + }): Promise; +} + +function workspaceChangedInstanceIds(): ChangedInstanceIdsDependency { + return ( + workspaceRequire("@itwin/imodel-transformer") as { + readonly ChangedInstanceIds: ChangedInstanceIdsDependency; + } + ).ChangedInstanceIds; +} + /** * Measures {@link ChangedInstanceIds.initialize} over a set of local changeset files. * @@ -38,11 +57,14 @@ class ChangesetScanningScenario { private _result?: ScanExpectation; private _aborted = false; - constructor(private readonly _dataset: PreparedDataset) {} + constructor( + private readonly _dataset: PreparedDataset, + private readonly _changedInstanceIds: ChangedInstanceIdsDependency + ) {} public async measure(): Promise { const dataset = requireDetachedDataset(this._dataset); - this._result = await ChangedInstanceIds.initialize({ + this._result = await this._changedInstanceIds.initialize({ csFileProps: dataset.csFileProps, iModel: dataset.sourceDb, }); @@ -74,6 +96,21 @@ class ChangesetScanningScenario { } } +export function createChangesetScanningScenario( + changedInstanceIds: ChangedInstanceIdsDependency = workspaceChangedInstanceIds() +): BenchmarkScenarioDefinition { + return { + id: "changeset-scanning", + defaultFixtureId: updateHeavyScanDescriptor.id, + capabilities: { + topology: "source-only", + requiredClaims: ["changeset scanning"], + }, + factory: (dataset) => + new ChangesetScanningScenario(dataset, changedInstanceIds), + }; +} + export const changesetScanningScenario: BenchmarkScenarioDefinition = { id: "changeset-scanning", defaultFixtureId: updateHeavyScanDescriptor.id, @@ -81,5 +118,6 @@ export const changesetScanningScenario: BenchmarkScenarioDefinition = { topology: "source-only", requiredClaims: ["changeset scanning"], }, - factory: (dataset) => new ChangesetScanningScenario(dataset), + factory: (dataset) => + new ChangesetScanningScenario(dataset, workspaceChangedInstanceIds()), };