diff --git a/common/api/core-backend.api.md b/common/api/core-backend.api.md index 5bfa5f628492..445ea18f667d 100644 --- a/common/api/core-backend.api.md +++ b/common/api/core-backend.api.md @@ -128,6 +128,7 @@ import { GeometryContainmentResponseProps } from '@itwin/core-common'; import { GeometryParams } from '@itwin/core-common'; import { GeometryPartProps } from '@itwin/core-common'; import { GeometryStreamProps } from '@itwin/core-common'; +import { GetSchemaViewArgs } from '@itwin/ecschema-metadata'; import { GuidString } from '@itwin/core-bentley'; import { Id64Arg } from '@itwin/core-bentley'; import { Id64Array } from '@itwin/core-bentley'; @@ -4104,7 +4105,7 @@ export abstract class IModelDb extends IModel { getMetaData(classFullName: string): EntityMetaData; getSchemaProps(name: string): ECSchemaProps; // @beta - getSchemaView(): Promise; + getSchemaView(args?: GetSchemaViewArgs): Promise; get holdsSchemaLock(): boolean; get iModelId(): GuidString; importSchemas(schemaFileNames: LocalFileName[], options?: SchemaImportOptions): Promise; diff --git a/common/api/core-frontend.api.md b/common/api/core-frontend.api.md index d0b79aaa6d2d..ba2fa4135acf 100644 --- a/common/api/core-frontend.api.md +++ b/common/api/core-frontend.api.md @@ -135,6 +135,7 @@ import { GeometryContainmentResponseProps } from '@itwin/core-common'; import { GeometryQuery } from '@itwin/core-geometry'; import { GeometryStreamProps } from '@itwin/core-common'; import { GeometrySummaryRequestProps } from '@itwin/core-common'; +import { GetSchemaViewArgs } from '@itwin/ecschema-metadata'; import { GlobeMode } from '@itwin/core-common'; import { Gradient } from '@itwin/core-common'; import { GraphicParams } from '@itwin/core-common'; @@ -5208,7 +5209,7 @@ export abstract class IModelConnection extends IModel { // @deprecated getMassPropertiesPerCandidate(requestProps: MassPropertiesPerCandidateRequestProps): Promise; // @beta - getSchemaView(): Promise; + getSchemaView(args?: GetSchemaViewArgs): Promise; getToolTipMessage(id: Id64String): Promise; readonly hilited: HiliteSet; // @internal diff --git a/common/api/ecschema-metadata.api.md b/common/api/ecschema-metadata.api.md index 90df3de34527..29a998638992 100644 --- a/common/api/ecschema-metadata.api.md +++ b/common/api/ecschema-metadata.api.md @@ -923,6 +923,13 @@ export class FormatSetFormatsProvider implements MutableFormatsProvider { // @internal (undocumented) export function getFormatProps(format: Format | OverrideFormat): FormatProps; +// @beta +export interface GetSchemaViewArgs { + // @internal + readonly forceReload?: boolean; + readonly schemas?: readonly string[]; +} + // @public @preview export interface HasMixins { // (undocumented) @@ -2492,6 +2499,56 @@ export interface SchemaLocaterOptions { readonly loadPartialSchemaOnly?: boolean; } +// @beta +export class SchemaManifest { + constructor(entries: readonly SchemaManifestEntry[]); + // (undocumented) + get entries(): readonly SchemaManifestEntry[]; + findByName(name: string): SchemaManifestEntry | undefined; + // @internal + static fromRows(schemaRows: readonly SchemaManifestSchemaRow[], referenceRows: readonly SchemaManifestReferenceRow[]): SchemaManifest; + getAvailableSchemaNames(): string[]; + getSchemaClosure(requestedNames: Iterable): string[]; + get schemaCount(): number; + // @internal + sortInDependencyOrder(schemaNames: Iterable): string[]; +} + +// @beta +export interface SchemaManifestEntry { + // (undocumented) + readonly minorVersion: number; + // (undocumented) + readonly name: string; + // (undocumented) + readonly readVersion: number; + readonly references: readonly SchemaManifestEntry[]; + // (undocumented) + readonly writeVersion: number; +} + +// @internal +export interface SchemaManifestReferenceRow { + // (undocumented) + readonly sourceECInstanceId: number; + // (undocumented) + readonly targetECInstanceId: number; +} + +// @internal +export interface SchemaManifestSchemaRow { + // (undocumented) + readonly ecInstanceId: number; + // (undocumented) + readonly name: string; + // (undocumented) + readonly versionMajor: number; + // (undocumented) + readonly versionMinor: number; + // (undocumented) + readonly versionWrite: number; +} + // @public @preview export enum SchemaMatchType { // (undocumented) @@ -2584,17 +2641,17 @@ export class SchemaView { // @internal (undocumented) readonly [_storage]: SchemaViewStorage; // @internal - constructor(data: SchemaViewData, schemaToken?: string); + constructor(data: SchemaViewData, schemaToken?: string, mergeContext?: SchemaViewMergeContext); // @internal (undocumented) buildDerivedClassMap(): ReadonlyMap; get classCount(): number; + // @internal + static createMergeable(schemaToken?: string): SchemaView; findClass(qualifiedName: string): SchemaView.Class | undefined; findEnumeration(qualifiedName: string): SchemaView.Enumeration | undefined; findKindOfQuantity(qualifiedName: string): SchemaView.KindOfQuantity | undefined; findPropertyCategory(qualifiedName: string): SchemaView.PropertyCategory | undefined; static fromBinary(blob: Uint8Array, schemaToken?: string): SchemaView; - // @internal - static fromBuilder(builder: SchemaViewBuilder, schemaToken?: string): SchemaView; getSchema(name: string): SchemaView.Schema | undefined; getSchemaByAlias(alias: string): SchemaView.Schema | undefined; getSchemas(): IterableIterator; @@ -2603,12 +2660,16 @@ export class SchemaView { get isOutdated(): boolean; // @internal markOutdated(): void; + // @internal + mergeFragment(blob: Uint8Array): void; // @internal (undocumented) resolveAllProperties(classIdx: number): readonly ResolvedPropertyRef[]; // @internal (undocumented) resolveClassIdx(qualifiedName: string): number; get schemaCount(): number; get schemaToken(): string; + // @internal + setSchemaToken(token: string): void; } // @beta (undocumented) @@ -2894,6 +2955,12 @@ export namespace SchemaView { } } +// @beta +export interface SchemaViewBlob { + readonly data: Uint8Array; + readonly schemaToken: string; +} + // @internal export class SchemaViewBuilder { addClass(data: ClassData): number; @@ -2907,13 +2974,20 @@ export class SchemaViewBuilder { addPropertyRef(ref: PropertyRef): void; addRelConstraint(data: RelConstraintData): number; addSchema(data: SchemaData): number; + assembleData(): SchemaViewData; build(schemaToken?: string): SchemaView; + get classCount(): number; get classMixinCount(): number; get constraintClassRefCount(): number; + get enumerationCount(): number; get enumeratorCount(): number; + extendLookupMaps(): void; getString(sid: number): string; internString(value: string | undefined): number; + get koqCount(): number; + get propCategoryCount(): number; get propertyRefCount(): number; + get schemaCount(): number; updateClass(classIdx: number, data: ClassData): void; updateSchemaRanges(schemaIdx: number, ranges: { classRangeStart: number; @@ -2969,9 +3043,46 @@ export interface SchemaViewData { readonly strings: readonly string[]; } +// @beta +export interface SchemaViewDataProvider { + fetchFragmentBlob(schemaNames: readonly string[]): Promise; + fetchFullBlob(): Promise; + fetchManifest(): Promise; + fetchSchemaToken(): Promise; +} + // @beta export const schemaViewFormatVersion = 1; +// @beta +export class SchemaViewManager { + constructor(dataProvider: SchemaViewDataProvider); + getSchemaView(args?: GetSchemaViewArgs): Promise; + invalidateIfChanged(): Promise; + reset(): void; +} + +// @internal +export class SchemaViewMergeContext { + // (undocumented) + readonly builder: SchemaViewBuilder; + // (undocumented) + readonly catRowIdToIdx: Map; + // (undocumented) + readonly classResolver: Map; + // (undocumented) + readonly classRowIdToIdx: Map; + // (undocumented) + readonly enumRowIdToIdx: Map; + // (undocumented) + readonly koqRowIdToIdx: Map; + mergeBlob(data: Uint8Array): void; + // (undocumented) + readonly schemaECIdToIdx: Map; + // (undocumented) + readonly schemaNames: string[]; +} + // @beta export enum SchemaViewPrimitiveType { // (undocumented) diff --git a/common/api/summary/ecschema-metadata.exports.csv b/common/api/summary/ecschema-metadata.exports.csv index da051c6de391..cc934656ae18 100644 --- a/common/api/summary/ecschema-metadata.exports.csv +++ b/common/api/summary/ecschema-metadata.exports.csv @@ -96,6 +96,7 @@ preview;class;Format beta;interface;FormatSet beta;class;FormatSetFormatsProvider internal;function;getFormatProps +beta;interface;GetSchemaViewArgs public;interface;HasMixins preview;interface;HasMixins beta;interface;ILocalizationProvider @@ -258,6 +259,10 @@ beta;class;SchemaLoader beta;class;SchemaLocalization beta;interface;SchemaLocalizationJson beta;interface;SchemaLocaterOptions +beta;class;SchemaManifest +beta;interface;SchemaManifestEntry +internal;interface;SchemaManifestReferenceRow +internal;interface;SchemaManifestSchemaRow public;enum;SchemaMatchType preview;enum;SchemaMatchType public;class;SchemaPartVisitorDelegate @@ -275,9 +280,13 @@ beta;namespace;SchemaView internal;function;createClass internal;function;createProperty internal;function;parseFormatString +beta;interface;SchemaViewBlob internal;class;SchemaViewBuilder internal;interface;SchemaViewData +beta;interface;SchemaViewDataProvider beta;const;schemaViewFormatVersion +beta;class;SchemaViewManager +internal;class;SchemaViewMergeContext beta;enum;SchemaViewPrimitiveType internal;class;SchemaWalker public;enum;StrengthDirection diff --git a/common/changes/@itwin/core-backend/rschili-schema-view-fragment_2026-07-23-15-54-03.json b/common/changes/@itwin/core-backend/rschili-schema-view-fragment_2026-07-23-15-54-03.json new file mode 100644 index 000000000000..07e8cd13f71d --- /dev/null +++ b/common/changes/@itwin/core-backend/rschili-schema-view-fragment_2026-07-23-15-54-03.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@itwin/core-backend", + "comment": "Add filtering support to SchemaView and improve performance.", + "type": "none" + } + ], + "packageName": "@itwin/core-backend" +} diff --git a/common/changes/@itwin/core-frontend/rschili-schema-view-fragment_2026-07-23-15-54-03.json b/common/changes/@itwin/core-frontend/rschili-schema-view-fragment_2026-07-23-15-54-03.json new file mode 100644 index 000000000000..ef12d474b7fd --- /dev/null +++ b/common/changes/@itwin/core-frontend/rschili-schema-view-fragment_2026-07-23-15-54-03.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@itwin/core-frontend", + "comment": "Add filtering support to SchemaView and improve performance.", + "type": "none" + } + ], + "packageName": "@itwin/core-frontend" +} diff --git a/common/changes/@itwin/ecschema-metadata/rschili-schema-view-fragment_2026-07-23-15-54-03.json b/common/changes/@itwin/ecschema-metadata/rschili-schema-view-fragment_2026-07-23-15-54-03.json new file mode 100644 index 000000000000..d230900a317a --- /dev/null +++ b/common/changes/@itwin/ecschema-metadata/rschili-schema-view-fragment_2026-07-23-15-54-03.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@itwin/ecschema-metadata", + "comment": "Add performance improvements for SchemaView. Filtering by schema, incremental loading, and a common lifetime manager class for invalidation.", + "type": "none" + } + ], + "packageName": "@itwin/ecschema-metadata" +} diff --git a/core/backend/src/IModelDb.ts b/core/backend/src/IModelDb.ts index 9725c00c1f31..ad6900716486 100644 --- a/core/backend/src/IModelDb.ts +++ b/core/backend/src/IModelDb.ts @@ -71,7 +71,7 @@ import { createNoOpLockControl } from "./internal/NoLocks"; import { IModelDbFonts } from "./IModelDbFonts"; import { createIModelDbFonts } from "./internal/IModelDbFontsImpl"; import { _activeTxn, _cache, _close, _hubAccess, _implicitTxn, _instanceKeyCache, _nativeDb, _releaseAllLocks, _resetIModelDb } from "./internal/Symbols"; -import { ECSpecVersion, ECVersion, SchemaContext, SchemaJsonLocater, SchemaView } from "@itwin/ecschema-metadata"; +import { ECSpecVersion, ECVersion, type GetSchemaViewArgs, SchemaContext, SchemaJsonLocater, SchemaManifest, type SchemaManifestReferenceRow, type SchemaManifestSchemaRow, SchemaView, type SchemaViewBlob, type SchemaViewDataProvider, SchemaViewManager } from "@itwin/ecschema-metadata"; import { SchemaMap } from "./Schema"; import { ElementLRUCache, InstanceKeyLRUCache } from "./internal/ElementLRUCache"; import { IModelIncrementalSchemaLocater } from "./IModelIncrementalSchemaLocater"; @@ -475,7 +475,9 @@ export abstract class IModelDb extends IModel { private _jsClassMap?: EntityJsClassMap; private _schemaMap?: SchemaMap; private _schemaContext?: SchemaContext; - private _schemasPromise?: Promise; + // Created lazily on the first getSchemaView call. Owns the SchemaView's lifetime and does all its + // data access through the SchemaViewDataProvider implemented below. + private _schemaViewManager?: SchemaViewManager; /** @deprecated in 5.0.0 - might be removed in next major version. Use [[fonts]]. */ protected _fontMap?: FontMap; // eslint-disable-line @typescript-eslint/no-deprecated private readonly _fonts: IModelDbFonts = createIModelDbFonts(this); @@ -1180,11 +1182,7 @@ export abstract class IModelDb extends IModel { this._jsClassMap = undefined; this._schemaMap = undefined; this._schemaContext = undefined; - if (this._schemasPromise) { - const old = this._schemasPromise; - this._schemasPromise = undefined; - old.then((view) => view.markOutdated()).catch(() => { }); - } + this._schemaViewManager?.reset(); this[_nativeDb].clearECDbCache(); } this.elements[_cache].clear(); @@ -1752,54 +1750,78 @@ export abstract class IModelDb extends IModel { } /** Get the schema view for this iModel. The view is built lazily on - * first call by fetching compact binary schema data via `PRAGMA schema_view` through - * the ConcurrentQuery thread pool. Subsequent calls return the cached view. Multiple - * concurrent callers share a single in-flight build. + * first call by fetching compact binary schema data through + * the ConcurrentQuery thread pool. * * The returned `SchemaView` is a lightweight, read-only, synchronous API for * navigating schema metadata - classes, properties, relationships, enumerations, etc. * It is the recommended default for runtime read-only metadata access and is significantly * faster and lower-memory than [[schemaContext]]. Use [[schemaContext]] for schema authoring, * custom-attribute deserialization, or anywhere you need the full ecschema-metadata object graph. + * + * Every call shares one accumulating view instance and concurrent calls are serialized, so a + * caller never observes a partially loaded view. The instance is discarded by [[clearCaches]], + * for example after a schema import; the next call builds a new one. See + * [GetSchemaViewArgs]($ecschema-metadata) for the arguments. * @beta */ - public async getSchemaView(): Promise { - if (this._schemasPromise) { - const ctx = await this._schemasPromise; - if (!ctx.isOutdated) - return ctx; - } - // Capture the in-flight promise locally so the rejection handler only clears - // `_schemasPromise` if it still points at this build. A concurrent invalidation + - // re-fetch could otherwise replace the field before our hydrate fails, and a naive - // `_schemasPromise = undefined` would clobber that newer reference. - const inflight = this._hydrateSchemas(); - this._schemasPromise = inflight; - inflight.catch(() => { - if (this._schemasPromise === inflight) - this._schemasPromise = undefined; - }); - return inflight; - } - - private async _hydrateSchemas(): Promise { - // PRAGMA returns exactly one row with format, formatVersion, data (binary), schemaToken. - // Important: only call reader.next() once - do NOT use `for await` on PRAGMA results. - // ConcurrentQuery wraps regular ECSQL in LIMIT/OFFSET for pagination but skips this for - // PRAGMAs. If the serialized result exceeds the memory threshold, the response is marked - // "Partial", and a `for await` loop would re-issue the same PRAGMA forever since PRAGMAs - // don't support OFFSET-based pagination. - // This implementation uses the non-pinned version of the pragma other than frontend - because backend - // is always strictly coupled with the native code. - const reader = this.createQueryReader("PRAGMA schema_view"); + public async getSchemaView(args?: GetSchemaViewArgs): Promise { + this._schemaViewManager ??= new SchemaViewManager(this._createSchemaViewDataProvider()); + return this._schemaViewManager.getSchemaView(args); + } + + /** The [SchemaViewDataProvider]($ecschema-metadata) backing this iModel's [[getSchemaView]]: the + * transport-specific half of schema-view loading. The backend always uses the latest blob version + * since it is strictly coupled with native code. + */ + private _createSchemaViewDataProvider(): SchemaViewDataProvider { + return { + fetchFullBlob: async () => this._fetchSchemaBlob("PRAGMA schema_view"), + // Names are ECNames, so a comma can never occur in one. Native re-validates each token as an + // ECName and fails the pragma on an unknown name. + fetchFragmentBlob: async (schemaNames) => this._fetchSchemaBlob(`PRAGMA schema_view_fragment('${schemaNames.join(",")}')`), + fetchManifest: async () => { + const schemaRows: SchemaManifestSchemaRow[] = []; + const schemaSql = "SELECT ECInstanceId, Name, VersionMajor, VersionWrite, VersionMinor FROM meta.ECSchemaDef"; + for await (const row of this.createQueryReader(schemaSql)) { + // ECInstanceId arrives as a hex Id64String. `ec_` metadata rowids carry no briefcase + // prefix, so the local id is the full value. + schemaRows.push({ ecInstanceId: Id64.getLocalId(row[0]), name: row[1], versionMajor: row[2], versionWrite: row[3], versionMinor: row[4] }); + } + + const referenceRows: SchemaManifestReferenceRow[] = []; + const referenceSql = "SELECT SourceECInstanceId, TargetECInstanceId FROM meta.SchemaHasSchemaReferences"; + for await (const row of this.createQueryReader(referenceSql)) + referenceRows.push({ sourceECInstanceId: Id64.getLocalId(row[0]), targetECInstanceId: Id64.getLocalId(row[1]) }); + + return SchemaManifest.fromRows(schemaRows, referenceRows); + }, + fetchSchemaToken: async () => { + const reader = this.createQueryReader("PRAGMA checksum(schema_token)"); + const result = await reader.next(); + if (result.done) + throw new IModelError(DbResult.BE_SQLITE_ERROR, "PRAGMA checksum(schema_token) returned no rows"); + return result.value.sha3_256 as string; + }, + }; + } + + /** Fetch one schema-view blob (full or fragment). Both `PRAGMA schema_view` and + * `PRAGMA schema_view_fragment` return a single row with the same columns. */ + private async _fetchSchemaBlob(pragma: string): Promise { + // Only call reader.next() once - do NOT use `for await` on PRAGMA results. ConcurrentQuery wraps + // regular ECSQL in LIMIT/OFFSET for pagination but skips this for PRAGMAs; if the serialized result + // exceeds the memory threshold the response is marked "Partial", and a `for await` loop would + // re-issue the same PRAGMA forever since PRAGMAs don't support OFFSET-based pagination. + const reader = this.createQueryReader(pragma); const result = await reader.next(); if (result.done) - throw new IModelError(DbResult.BE_SQLITE_ERROR, "PRAGMA schema_view returned no rows"); + throw new IModelError(DbResult.BE_SQLITE_ERROR, `${pragma} returned no rows`); const data = result.value.data as Uint8Array | undefined; const token = result.value.schemaToken as string | undefined; if (data === undefined || data === null) - throw new IModelError(DbResult.BE_SQLITE_ERROR, "PRAGMA schema_view returned null data column"); - return SchemaView.fromBinary(data, token ?? ""); + throw new IModelError(DbResult.BE_SQLITE_ERROR, `${pragma} returned null data column`); + return { data, schemaToken: token ?? "" }; } /** Get the linkTableRelationships for this IModel */ diff --git a/core/backend/src/test/schema/SchemaViewFragmentLoading.test.ts b/core/backend/src/test/schema/SchemaViewFragmentLoading.test.ts new file mode 100644 index 000000000000..3b381abb4fbf --- /dev/null +++ b/core/backend/src/test/schema/SchemaViewFragmentLoading.test.ts @@ -0,0 +1,594 @@ +/*--------------------------------------------------------------------------------------------- +* Copyright (c) Bentley Systems, Incorporated. All rights reserved. +* See LICENSE.md in the project root for license terms and full copyright notice. +*--------------------------------------------------------------------------------------------*/ + +import { Guid, Logger, LogLevel, OpenMode } from "@itwin/core-bentley"; +import { GenericSchema, IModelHost, IModelJsFs, StandaloneDb } from "../../core-backend"; +import { SchemaView, schemaViewFormatVersion, SchemaViewPrimitiveType } from "@itwin/ecschema-metadata"; +import { expect } from "chai"; +import * as path from "path"; +import * as sinon from "sinon"; +import { KnownTestLocations } from "../KnownTestLocations"; +import { TestUtils } from "../TestUtils"; + +/** Categories of native log messages worth treating as a test failure. Native code routes through + * `Logger`, so a C++ `warningv`/`errorv` surfaces here once the category level is raised. */ +const monitoredNativeCategories = ["ECDb", "ECObjectsNative"]; + +/** A captured native log message. */ +interface CapturedNativeLog { + level: "Warning" | "Error"; + category: string; + message: string; +} + +/** Run `body` while capturing native warnings and errors from the monitored categories. + * + * The test harness initializes the logger with all categories off, so native warnings/errors are + * dropped before they ever reach JS. To observe them we must (1) raise each monitored category to + * Warning - which fires onLogLevelChanged, prompting the native side to re-read levels and start + * emitting (Warning and anything more severe, i.e. Error too) - and (2) install sinks to collect + * what arrives. Returns the captured logs so the caller can assert none were produced. + */ +async function captureNativeLogs(body: () => Promise): Promise { + const logs: CapturedNativeLog[] = []; + const previousLevels = new Map(monitoredNativeCategories.map((category) => [category, Logger.getLevel(category)])); + + const warningStub = sinon.stub(Logger, "logWarning").callsFake((category: string, message: string) => { + if (monitoredNativeCategories.includes(category)) + logs.push({ level: "Warning", category, message }); + }); + const errorStub = sinon.stub(Logger, "logError").callsFake((category: string, messageOrError: unknown) => { + if (monitoredNativeCategories.includes(category)) + logs.push({ level: "Error", category, message: typeof messageOrError === "string" ? messageOrError : String(messageOrError) }); + }); + + for (const category of monitoredNativeCategories) + Logger.setLevel(category, LogLevel.Warning); + + try { + await body(); + } finally { + warningStub.restore(); + errorStub.restore(); + for (const [category, previousLevel] of previousLevels) + Logger.setLevel(category, previousLevel ?? LogLevel.None); + } + return logs; +} + +/** SchemaB references BisCore and defines BElement deriving from a BisCore class. */ +const schemaB = ` + + + + bis:PhysicalElement + + + `; + +/** SchemaA references FragB and derives AElement from FragB:BElement. + * So loading FragA must transitively pull FragB (and, through it, BisCore). */ +const schemaA = ` + + + + fb:BElement + + + `; + +/** A third schema imported after the first load, to exercise cache invalidation. */ +const schemaC = ` + + + + bis:PhysicalElement + + + `; + +/** + * Tests the incremental ("husk") schema-view path: `getSchemaView({ schemas: [...] })` loads only + * the requested schemas plus their references via `PRAGMA schema_view_fragment`, accumulating + * across calls. These run against a real iModel - the blob is exercised end to end (native writer -> + * TS reader/merge), which is where the fragment mechanism actually has to work. The blob format + * itself is an internal C++-writer-to-TS-reader contract. + */ +describe("SchemaView fragment loading", () => { + // Building an empty iModel and importing schemas into it dominates this suite's runtime, so each + // seed below is created once in `before`. Each test then just needs a writable file of its own - + // a raw file copy plus `StandaloneDb.openFile` - rather than paying for createEmpty + schema import + // again, or for the checkpoint/vacuum overhead that `SnapshotDb.createFrom` adds on top of the copy. + let fragSeedDb: StandaloneDb; + let genericSeedDb: StandaloneDb; + + before(async () => { + if (!IModelHost.isValid) + await TestUtils.startBackend(); + + const fragSeedPath = path.join(KnownTestLocations.outputDir, `SchemaViewFragmentSeed-${Guid.createValue()}.bim`); + fragSeedDb = StandaloneDb.createEmpty(fragSeedPath, { rootSubject: { name: "SchemaViewFragmentLoadingSeed" } }); + await fragSeedDb.importSchemaStrings([schemaB, schemaA]); + fragSeedDb.performCheckpoint(); // flush + truncate the WAL so the raw file copy below is consistent + + GenericSchema.registerSchema(); + const genericSeedPath = path.join(KnownTestLocations.outputDir, `SchemaViewFragmentGenericSeed-${Guid.createValue()}.bim`); + genericSeedDb = StandaloneDb.createEmpty(genericSeedPath, { rootSubject: { name: "SchemaViewFragmentGenericSeed" } }); + await genericSeedDb.importSchemas([GenericSchema.schemaFilePath]); + genericSeedDb.performCheckpoint(); + }); + + after(() => { + fragSeedDb.close(); + genericSeedDb.close(); + }); + + /** Clone a writable iModel from a seed's file via a raw file copy - no checkpoint/vacuum. Only + * needed by the handful of tests that mutate the iModel (e.g. importing another schema); every + * other test just opens a second read-only connection directly onto the seed's file below. */ + function cloneSeed(seedDb: StandaloneDb, namePrefix: string): StandaloneDb { + const filePath = path.join(KnownTestLocations.outputDir, `${namePrefix}-${Guid.createValue()}.bim`); + IModelJsFs.copySync(seedDb.pathName, filePath); + return StandaloneDb.openFile(filePath, OpenMode.ReadWrite); + } + + /** Get an iModel backed by the FragA/FragB (and BisCore) seed. Read-only by default - just another + * connection onto the same file, no copy - since most tests only read a `SchemaView` off it. Pass + * `writable: true` for the rare test that needs to mutate the iModel (e.g. import another schema), + * which gets its own private copy so it can't affect the shared seed or other tests. */ + async function createIModelWithFragSchemas(options?: { writable?: boolean }): Promise { + if (options?.writable) + return cloneSeed(fragSeedDb, "SchemaViewFragment"); + return StandaloneDb.openFile(fragSeedDb.pathName, OpenMode.Readonly); + } + + /** Every schema name in the iModel, straight from ECDbMeta - the same source the schema manifest + * is built from - so tests never hardcode the schema inventory of an empty snapshot. */ + async function queryAllSchemaNames(iModel: StandaloneDb): Promise { + const schemaNames: string[] = []; + for await (const row of iModel.createQueryReader("SELECT Name FROM meta.ECSchemaDef")) + schemaNames.push(row[0] as string); + return schemaNames; + } + + /** The sorted names of every schema present in a view. */ + function getSortedViewSchemaNames(view: SchemaView): string[] { + return [...view.getSchemas()].map((schema) => schema.name).sort(); + } + + it("loads a requested schema and its reference closure, leaving unrequested schemas absent", async () => { + const iModel = await createIModelWithFragSchemas(); + try { + // Request only FragB. It references BisCore, so both must be present; FragA must not be. + const view = await iModel.getSchemaView({ schemas: ["FragB"] }); + + expect(view.getSchema("FragB"), "FragB was requested").to.not.be.undefined; + expect(view.getSchema("BisCore"), "BisCore is in FragB's closure").to.not.be.undefined; + expect(view.getSchema("FragA"), "FragA was not requested and nothing pulled it in").to.be.undefined; + + expect(view.findClass("FragB:BElement")).to.not.be.undefined; + expect(view.findClass("FragA:AElement")).to.be.undefined; + } finally { + iModel.close(); + } + }); + + it("pulls transitive dependencies when loading a dependent schema", async () => { + const iModel = await createIModelWithFragSchemas(); + try { + // FragA -> FragB -> BisCore. Requesting FragA alone must make the whole chain resolvable. + const view = await iModel.getSchemaView({ schemas: ["FragA"] }); + + expect(view.getSchema("FragA")).to.not.be.undefined; + expect(view.getSchema("FragB")).to.not.be.undefined; + expect(view.getSchema("BisCore")).to.not.be.undefined; + + const aElement = view.findClass("FragA:AElement"); + expect(aElement).to.not.be.undefined; + // Cross-schema base class resolves across the closure. + expect(aElement!.baseClass?.fullName).to.equal("FragB:BElement"); + // And transitively up into BisCore. + expect(aElement!.is("BisCore:PhysicalElement")).to.be.true; + } finally { + iModel.close(); + } + }); + + it("accumulates schemas across calls into one view", async () => { + const iModel = await createIModelWithFragSchemas(); + try { + const view1 = await iModel.getSchemaView({ schemas: ["FragB"] }); + expect(view1.getSchema("FragA")).to.be.undefined; + + // A second call for FragA merges into the same accumulating view. + const view2 = await iModel.getSchemaView({ schemas: ["FragA"] }); + expect(view2, "subset view is a single accumulating instance").to.equal(view1); + + // Both schemas are now resolvable on the one view. + expect(view2.findClass("FragA:AElement")).to.not.be.undefined; + expect(view2.findClass("FragB:BElement")).to.not.be.undefined; + } finally { + iModel.close(); + } + }); + + it("is idempotent when re-requesting an already-loaded schema", async () => { + const iModel = await createIModelWithFragSchemas(); + try { + const view1 = await iModel.getSchemaView({ schemas: ["FragA"] }); + const aElement1 = view1.findClass("FragA:AElement"); + expect(aElement1).to.not.be.undefined; + + // Re-requesting loads nothing new and returns the same instance with stable indices. + const view2 = await iModel.getSchemaView({ schemas: ["FragA", "FragB"] }); + expect(view2).to.equal(view1); + expect(view2.findClass("FragA:AElement")!.idx).to.equal(aElement1!.idx); + } finally { + iModel.close(); + } + }); + + it("ignores schema names the iModel does not contain", async () => { + const iModel = await createIModelWithFragSchemas(); + try { + const view = await iModel.getSchemaView({ schemas: ["DoesNotExist"] }); + expect(view.getSchema("DoesNotExist")).to.be.undefined; + // No real schemas requested; the view is a valid, empty husk. + expect(view.schemaCount).to.equal(0); + } finally { + iModel.close(); + } + }); + + it("serializes overlapping concurrent loads without double-merging", async () => { + const iModel = await createIModelWithFragSchemas(); + try { + // Fire overlapping requests whose closures share BisCore. The merge queue must coalesce so no + // schema is merged twice (which would surface as duplicate classes). + const [v1, v2] = await Promise.all([ + iModel.getSchemaView({ schemas: ["FragA"] }), + iModel.getSchemaView({ schemas: ["FragB"] }), + ]); + expect(v2).to.equal(v1); + + // Exactly one BElement and one AElement - no duplicates from a double merge. + let bElementCount = 0; + let aElementCount = 0; + for (const schema of v1.getSchemas()) { + for (const cls of schema.getClasses()) { + if (cls.fullName === "FragB:BElement") bElementCount++; + if (cls.fullName === "FragA:AElement") aElementCount++; + } + } + expect(bElementCount, "BElement merged exactly once").to.equal(1); + expect(aElementCount, "AElement merged exactly once").to.equal(1); + } finally { + iModel.close(); + } + }); + + it("invalidates the subset husk after a schema import", async () => { + const iModel = await createIModelWithFragSchemas({ writable: true }); + try { + const view1 = await iModel.getSchemaView({ schemas: ["FragB"] }); + expect(view1.getSchema("FragB")).to.not.be.undefined; + expect(view1.isOutdated).to.be.false; + + // Importing a schema calls clearCaches, which drops the husk and its loaded-set. + await iModel.importSchemaStrings([schemaC]); + expect(view1.isOutdated, "the old husk is marked outdated").to.be.true; + + // A fresh request rebuilds against the new schema state - a different instance that can now + // load the newly imported schema. + const view2 = await iModel.getSchemaView({ schemas: ["FragC"] }); + expect(view2, "a new husk is built after invalidation").to.not.equal(view1); + expect(view2.findClass("FragC:CElement")).to.not.be.undefined; + } finally { + iModel.close(); + } + }); + + it("the full view (no args) still contains everything the subset path can load", async () => { + const iModel = await createIModelWithFragSchemas(); + try { + const full = await iModel.getSchemaView(); + expect(full.findClass("FragA:AElement"), "full view has the domain class").to.not.be.undefined; + expect(full.findClass("FragB:BElement")).to.not.be.undefined; + } finally { + iModel.close(); + } + }); + + it("an unfiltered call after a filtered one merges the remaining schemas into the same view and collapses to full mode", async () => { + const iModel = await createIModelWithFragSchemas(); + try { + // Start incremental: only FragB (+ BisCore) is loaded. + const subsetView = await iModel.getSchemaView({ schemas: ["FragB"] }); + expect(subsetView.getSchema("FragA")).to.be.undefined; + + // The unfiltered call means "I need everything": it must merge the rest into the SAME instance. + const fullView = await iModel.getSchemaView(); + expect(fullView, "the husk accumulates; no new instance").to.equal(subsetView); + expect(fullView.findClass("FragA:AElement"), "previously missing schema is now merged").to.not.be.undefined; + expect(fullView.findClass("FragB:BElement"), "earlier schemas remain available").to.not.be.undefined; + + // The unfiltered merge must have collapsed the incremental bookkeeping into full mode: any + // further request - filtered or not - is a synchronous no-op that issues no queries. + const queryReaderSpy = sinon.spy(iModel, "createQueryReader"); + try { + const filteredAgain = await iModel.getSchemaView({ schemas: ["FragA"] }); + const unfilteredAgain = await iModel.getSchemaView(); + expect(filteredAgain).to.equal(fullView); + expect(unfilteredAgain).to.equal(fullView); + expect(queryReaderSpy.notCalled, "fully loaded view is served without any queries").to.be.true; + } finally { + queryReaderSpy.restore(); + } + } finally { + iModel.close(); + } + }); + + it("filling the view schema-by-schema via filters collapses to full mode and matches a full load", async () => { + const iModel = await createIModelWithFragSchemas(); + try { + // Request every schema the iModel contains, one filtered call at a time. The names come from + // ECDbMeta (the manifest's own source), so nothing about the snapshot's schema inventory is + // hardcoded - this also exercises requesting excluded schemas (e.g. CoreCustomAttributes) + // directly, which must be tolerated and contribute nothing. + const allSchemaNames = await queryAllSchemaNames(iModel); + expect(allSchemaNames.length, "snapshot contains more schemas than the two imported ones").to.be.greaterThan(2); + + let accumulatingView: SchemaView | undefined; + for (const schemaName of allSchemaNames) { + const view = await iModel.getSchemaView({ schemas: [schemaName] }); + if (accumulatingView === undefined) + accumulatingView = view; + else + expect(view, "every filtered call returns the one accumulating instance").to.equal(accumulatingView); + } + if (accumulatingView === undefined) + expect.fail("no schema view was obtained"); + + // Once every schema has been requested, the view must have collapsed to full mode: an + // unfiltered request is a synchronous no-op on the same instance, with no queries issued. + const queryReaderSpy = sinon.spy(iModel, "createQueryReader"); + try { + const fullRequest = await iModel.getSchemaView(); + expect(fullRequest).to.equal(accumulatingView); + expect(queryReaderSpy.notCalled, "collapsed view is served without any queries").to.be.true; + } finally { + queryReaderSpy.restore(); + } + + // The fragment-filled view must be equivalent to a from-scratch full load of the same iModel: + // same schemas present (both paths apply the same exclusion list in the native writer). + const rebuiltFullView = await iModel.getSchemaView({ forceReload: true }); + expect(rebuiltFullView).to.not.equal(accumulatingView); + expect(getSortedViewSchemaNames(accumulatingView), "fragment fill and full load agree on the schema set") + .to.deep.equal(getSortedViewSchemaNames(rebuiltFullView)); + } finally { + iModel.close(); + } + }); + + describe("forceReload", () => { + it("discards an accumulated husk and rebuilds from scratch", async () => { + const iModel = await createIModelWithFragSchemas(); + try { + // Accumulate FragA + FragB into one husk. + const view1 = await iModel.getSchemaView({ schemas: ["FragA"] }); + expect(view1.getSchema("FragB")).to.not.be.undefined; + expect(view1.isOutdated).to.be.false; + + // forceReload with a narrower filter: the old husk is dropped and marked outdated, and the + // rebuilt view contains only the newly requested closure - FragA is gone. + const view2 = await iModel.getSchemaView({ schemas: ["FragB"], forceReload: true }); + expect(view2).to.not.equal(view1); + expect(view1.isOutdated, "the discarded husk is marked outdated").to.be.true; + expect(view2.isOutdated).to.be.false; + expect(view2.getSchema("FragB")).to.not.be.undefined; + expect(view2.getSchema("FragA"), "rebuild starts from scratch; earlier schemas are not carried over").to.be.undefined; + } finally { + iModel.close(); + } + }); + + it("rebuilds a fully loaded view", async () => { + const iModel = await createIModelWithFragSchemas(); + try { + const view1 = await iModel.getSchemaView(); + const view2 = await iModel.getSchemaView({ forceReload: true }); + expect(view2).to.not.equal(view1); + expect(view1.isOutdated).to.be.true; + expect(view2.isOutdated).to.be.false; + // The rebuilt view is complete again. + expect(getSortedViewSchemaNames(view2)).to.deep.equal(getSortedViewSchemaNames(view1)); + } finally { + iModel.close(); + } + }); + + it("is serialized behind an in-flight load", async () => { + const iModel = await createIModelWithFragSchemas(); + try { + // Fire a load and a forceReload without awaiting in between. The reload must wait for the + // first load to finish, then discard its result - never tearing down state mid-load. + const [view1, view2] = await Promise.all([ + iModel.getSchemaView({ schemas: ["FragA"] }), + iModel.getSchemaView({ schemas: ["FragA"], forceReload: true }), + ]); + expect(view2).to.not.equal(view1); + expect(view1.isOutdated, "the first load's view was discarded by the reload").to.be.true; + expect(view2.isOutdated).to.be.false; + expect(view2.findClass("FragA:AElement"), "the rebuilt view is fully usable").to.not.be.undefined; + } finally { + iModel.close(); + } + }); + }); + + describe("PRAGMA schema_view_fragment version prefix", () => { + // The frontend must pin the blob format version (frontend and backend can be version-skewed), + // so it will issue the pragma with the `v;` prefix. These tests prove the prefix contract + // works end to end through ConcurrentQuery, not just in the native unit tests. + + /** Fetch one fragment blob directly via the pragma. Calls `next()` exactly once - PRAGMA + * results must not be iterated, since ConcurrentQuery cannot paginate them. */ + async function fetchFragmentBlob(iModel: StandaloneDb, argument: string): Promise<{ data: Uint8Array, formatVersion: number }> { + const reader = iModel.createQueryReader(`PRAGMA schema_view_fragment('${argument}')`); + const result = await reader.next(); + expect(result.done, "pragma returned a row").to.be.false; + const data = result.value.data as Uint8Array | undefined; + if (data === undefined || data === null) + expect.fail("pragma returned no data column"); + return { data, formatVersion: result.value.formatVersion as number }; + } + + it("accepts a v1; prefix and returns the same blob as the unprefixed form", async () => { + const iModel = await createIModelWithFragSchemas(); + try { + const nameList = (await queryAllSchemaNames(iModel)).join(","); + + const prefixed = await fetchFragmentBlob(iModel, `v${schemaViewFormatVersion};${nameList}`); + expect(prefixed.formatVersion, "reported format version matches the requested one").to.equal(schemaViewFormatVersion); + + // Omitting the prefix means "latest", which today is the same version - identical blob bytes. + const unprefixed = await fetchFragmentBlob(iModel, nameList); + expect(unprefixed.formatVersion).to.equal(schemaViewFormatVersion); + expect(Buffer.from(prefixed.data).equals(Buffer.from(unprefixed.data)), "explicit version and latest produce identical blobs").to.be.true; + + // The prefixed blob is a valid fragment end to end: it merges into a husk and resolves classes. + const husk = SchemaView.createMergeable(); + husk.mergeFragment(prefixed.data); + expect(husk.findClass("FragA:AElement")).to.not.be.undefined; + expect(husk.findClass("FragB:BElement")).to.not.be.undefined; + } finally { + iModel.close(); + } + }); + + it("rejects malformed and unsupported version prefixes", async () => { + const iModel = await createIModelWithFragSchemas(); + try { + const nameList = (await queryAllSchemaNames(iModel)).join(","); + const invalidArguments = [ + `v99;${nameList}`, // unsupported version + `v;${nameList}`, // malformed version token (no digits) + `vx;${nameList}`, // malformed version token (non-digit) + "v1;", // version present, empty name list + ]; + // Native rejects these at prepare time; ConcurrentQuery surfaces that as a rejected read. + for (const argument of invalidArguments) { + const reader = iModel.createQueryReader(`PRAGMA schema_view_fragment('${argument}')`); + let rejected = false; + try { + await reader.next(); + } catch { + rejected = true; + } + expect(rejected, `'${argument}' is rejected`).to.be.true; + } + } finally { + iModel.close(); + } + }); + }); + + it("loads a real domain schema's reference closure, dropping references that are excluded", async () => { + // Walk against real schemas. Generic references BisCore plus three schemas that are all on SchemaView's + // exclusion list. BisCore in turn references four schemas that are ALL excluded. + // So requesting Generic must yield a view containing exactly Generic + BisCore - the walk + // pulls BisCore, and every excluded reference contributes no rows. + const iModel = StandaloneDb.openFile(genericSeedDb.pathName, OpenMode.Readonly); + try { + // Capture native warnings and errors while the fragment blob is written + parsed. + // assert nothing was logged as a safeguard against regressions. + let schemaView: Awaited> | undefined; + const nativeLogs = await captureNativeLogs(async () => { + schemaView = await iModel.getSchemaView({ schemas: ["Generic"] }); + }); + expect( + nativeLogs, + `native warnings/errors during getSchemaView:\n${nativeLogs.map((l) => `${l.level} | ${l.category} | ${l.message}`).join("\n")}`, + ).to.be.empty; + expect(schemaView, "schema view was obtained").to.not.be.undefined; + if (!schemaView) + return; + + // The requested schema and its one non-excluded reference are present. + expect(schemaView.getSchema("Generic"), "Generic was requested").to.not.be.undefined; + expect(schemaView.getSchema("BisCore"), "BisCore is Generic's only non-excluded reference").to.not.be.undefined; + + // Every excluded reference in the closure contributes nothing to the view. + for (const excluded of ["CoreCustomAttributes", "BisCustomAttributes", "ECDbMap", "ECDbSchemaPolicies"]) + expect(schemaView.getSchema(excluded), `${excluded} is excluded from SchemaView`).to.be.undefined; + + // The view contains exactly the two non-excluded schemas - nothing leaked in from the closure. + expect(schemaView.schemaCount, "view holds exactly Generic + BisCore").to.equal(2); + + // A cross-schema base-class reference resolves across the loaded closure. + const physicalObject = schemaView.findClass("Generic:PhysicalObject"); + expect(physicalObject, "Generic:PhysicalObject is loaded").to.not.be.undefined; + expect(physicalObject!.is("BisCore:PhysicalElement"), "resolves up into BisCore").to.be.true; + + // PhysicalObject's immediate base is bis:PhysicalElement, resolved across the schema boundary. + const baseClass = physicalObject!.baseClass; + expect(baseClass, "PhysicalObject has a base class").to.not.be.undefined; + expect(baseClass!.fullName, "base class is bis:PhysicalElement").to.equal("BisCore:PhysicalElement"); + + // PhysicalObject defines no own properties, so walking its properties must surface only the + // ones inherited from the BisCore ancestor chain (PhysicalElement -> ... -> Element). This + // proves the cross-schema property inheritance walk is hydrated correctly. + expect(physicalObject!.getOwnProperties(), "PhysicalObject has no own properties").to.have.lengthOf(0); + + // Expected primitive type for a sampling of inherited primitive properties, one (or more) from + // each ancestor. + const expectedPrimitiveTypes = new Map([ + ["FederationGuid", SchemaViewPrimitiveType.Binary], // Element, binary + ["CodeValue", SchemaViewPrimitiveType.String], // Element, string + ["UserLabel", SchemaViewPrimitiveType.String], // Element, string + ["LastMod", SchemaViewPrimitiveType.DateTime], // Element, dateTime + ["InSpatialIndex", SchemaViewPrimitiveType.Boolean], // GeometricElement3d, boolean + ["Origin", SchemaViewPrimitiveType.Point3d], // GeometricElement3d, point3d + ["Yaw", SchemaViewPrimitiveType.Double], // GeometricElement3d, double + ["GeometryStream", SchemaViewPrimitiveType.Binary], // GeometricElement3d, binary + ]); + // Navigation properties contributed by the chain + const expectedNavProperties = ["Model", "CodeSpec", "Category", "PhysicalMaterial"]; + + // First walk: getProperties() returns the full inherited set. Verify each expected property is + // present and, for primitives, that its primitive type survived hydration intact. + const propertiesByName = new Map(physicalObject!.getProperties().map((p) => [p.name, p])); + for (const [name, primitiveType] of expectedPrimitiveTypes) { + const property = propertiesByName.get(name); + expect(property, `inherited property "${name}" is present`).to.not.be.undefined; + if (!property) continue; // narrows for the type checker; expect already failed if undefined + if (!property.isPrimitive()) + expect.fail(`inherited property "${name}" should be primitive`); + expect(property.primitiveType, `"${name}" primitive type`).to.equal(primitiveType); + } + for (const name of expectedNavProperties) { + const property = propertiesByName.get(name); + expect(property, `inherited navigation property "${name}" is present`).to.not.be.undefined; + if (!property) continue; + expect(property.isNavigation(), `"${name}" is a navigation property`).to.be.true; + } + + // Second walk: the single getProperty(name) accessor resolves the same inherited properties + // (case-insensitively) and reports the same primitive types. + for (const [name, primitiveType] of expectedPrimitiveTypes) { + const property = physicalObject!.getProperty(name); + expect(property, `getProperty("${name}") resolves`).to.not.be.undefined; + if (!property) continue; + if (!property.isPrimitive()) + expect.fail(`getProperty("${name}") should be primitive`); + expect(property.primitiveType, `getProperty("${name}") primitive type`).to.equal(primitiveType); + } + } finally { + iModel.close(); + } + }); +}); diff --git a/core/backend/src/test/schema/SchemaViewLifecycle.test.ts b/core/backend/src/test/schema/SchemaViewLifecycle.test.ts index ff477b7b241e..47d6b2ec6dc5 100644 --- a/core/backend/src/test/schema/SchemaViewLifecycle.test.ts +++ b/core/backend/src/test/schema/SchemaViewLifecycle.test.ts @@ -65,7 +65,7 @@ describe("SchemaView lifecycle", () => { expect(ctx1.isOutdated).to.be.false; // Import v1 schema - adds TestElement with PropA - // importSchemaStrings calls clearCaches which immediately invalidates _schemasPromise. + // importSchemaStrings calls clearCaches which immediately invalidates the cached schema view. await bc.importSchemaStrings([testSchemaV1]); // importSchemaStrings calls clearCaches which immediately invalidates the cached view @@ -132,7 +132,7 @@ describe("SchemaView lifecycle", () => { // bc2: pull changes await bc2.pullChanges(); - // After pulling a schema changeset, clearCaches is called which immediately nulls _schemasPromise. + // After pulling a schema changeset, clearCaches is called which immediately drops the cached schema view. const ctxAfterPull = await bc2.getSchemaView(); expect(ctxBefore.isOutdated).to.be.true; const testClass = ctxAfterPull.findClass("SchemaViewLifecycleTest:TestElement"); diff --git a/core/ecschema-metadata/src/Localization/SchemaLocalization.ts b/core/ecschema-metadata/src/Localization/SchemaLocalization.ts index c5602ed42243..976af77e9eda 100644 --- a/core/ecschema-metadata/src/Localization/SchemaLocalization.ts +++ b/core/ecschema-metadata/src/Localization/SchemaLocalization.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { SchemaKey } from "../SchemaKey"; -import { SchemaView } from "../SchemaView"; +import { SchemaView } from "../SchemaView/SchemaView"; import { Schema } from "../Metadata/Schema"; import { ECClass } from "../Metadata/Class"; import { Logger } from "@itwin/core-bentley"; diff --git a/core/ecschema-metadata/src/SchemaView/SchemaManifest.ts b/core/ecschema-metadata/src/SchemaView/SchemaManifest.ts new file mode 100644 index 000000000000..ca5ec949beb5 --- /dev/null +++ b/core/ecschema-metadata/src/SchemaView/SchemaManifest.ts @@ -0,0 +1,181 @@ +/*--------------------------------------------------------------------------------------------- +* Copyright (c) Bentley Systems, Incorporated. All rights reserved. +* See LICENSE.md in the project root for license terms and full copyright notice. +*--------------------------------------------------------------------------------------------*/ +/** @packageDocumentation + * @module Schema + */ + +/** One schema in a {@link SchemaManifest}: its name, version, and the entries it directly references. + * @beta + */ +export interface SchemaManifestEntry { + readonly name: string; + readonly readVersion: number; + readonly writeVersion: number; + readonly minorVersion: number; + /** The schemas this schema directly references. */ + readonly references: readonly SchemaManifestEntry[]; +} + +/** One row of `SELECT ECInstanceId, Name, VersionMajor, VersionWrite, VersionMinor FROM + * meta.ECSchemaDef`, as passed to {@link SchemaManifest.fromRows}. + * @note `ecInstanceId` is a plain number, matching SchemaView's convention for schema-related rows: + * `ec_` metadata rowids carry no briefcase prefix, so they are exactly representable. It is used + * only to wire reference edges and is not retained in the manifest. + * @internal + */ +export interface SchemaManifestSchemaRow { + readonly ecInstanceId: number; + readonly name: string; + readonly versionMajor: number; + readonly versionWrite: number; + readonly versionMinor: number; +} + +/** One row of `SELECT SourceECInstanceId, TargetECInstanceId FROM meta.SchemaHasSchemaReferences`, + * as passed to {@link SchemaManifest.fromRows}. + * @internal + */ +export interface SchemaManifestReferenceRow { + readonly sourceECInstanceId: number; + readonly targetECInstanceId: number; +} + +/** The reference graph of every schema in one iModel - names, versions and reference edges, without + * any schema data. A {@link (SchemaView:class)} husk loads it up front to answer which schemas exist + * and which dependency-ordered set it must load to satisfy a request. + * + * A `SchemaViewDataProvider` builds the manifest from ECDbMeta rows via {@link SchemaManifest.fromRows}. + * The entries are a flat array with no iModel or platform dependency; even the largest iModels hold + * on the order of a hundred schemas, so the closure and topological walks are plain recursion. + * @note The manifest does not track which schemas are already loaded. `SchemaViewManager` does that + * and filters the result of {@link SchemaManifest.getSchemaClosure} itself. + * @beta + */ +export class SchemaManifest { + private readonly _entries: readonly SchemaManifestEntry[]; + private readonly _byLowerName: ReadonlyMap; + + /** Wraps a set of entries whose references are already wired to one another. */ + public constructor(entries: readonly SchemaManifestEntry[]) { + this._entries = entries; + const byLowerName = new Map(); + for (const entry of entries) + byLowerName.set(entry.name.toLowerCase(), entry); + this._byLowerName = byLowerName; + } + + /** Build a manifest from raw ECDbMeta query rows, so a `SchemaViewDataProvider` only has to run + * the two queries and hand the rows over. Reference rows whose endpoints are unknown or + * self-referential are skipped; that cannot happen for a well-formed iModel. + * @internal + */ + public static fromRows(schemaRows: readonly SchemaManifestSchemaRow[], referenceRows: readonly SchemaManifestReferenceRow[]): SchemaManifest { + // Mutable during the wiring walk below; entries are read-only once handed to the manifest. + type MutableEntry = Omit & { references: SchemaManifestEntry[] }; + + const entries: MutableEntry[] = []; + const entryByECInstanceId = new Map(); + for (const row of schemaRows) { + const entry: MutableEntry = { + name: row.name, + readVersion: row.versionMajor, + writeVersion: row.versionWrite, + minorVersion: row.versionMinor, + references: [], + }; + entries.push(entry); + entryByECInstanceId.set(row.ecInstanceId, entry); + } + + for (const row of referenceRows) { + const source = entryByECInstanceId.get(row.sourceECInstanceId); + const target = entryByECInstanceId.get(row.targetECInstanceId); + if (source === undefined || target === undefined || source === target || source.references.includes(target)) + continue; + source.references.push(target); + } + + return new SchemaManifest(entries); + } + + /** The number of schemas in the iModel. */ + public get schemaCount(): number { return this._entries.length; } + + /** The names of every schema in the iModel, in manifest order. */ + public getAvailableSchemaNames(): string[] { + return this._entries.map((entry) => entry.name); + } + + public get entries(): readonly SchemaManifestEntry[] { + return this._entries; + } + + /** The entry for a schema by name (case-insensitive), or `undefined` if the iModel has no such schema. */ + public findByName(name: string): SchemaManifestEntry | undefined { + return this._byLowerName.get(name.toLowerCase()); + } + + /** The transitive reference closure of the requested schemas, as a flat, duplicate-free list of + * names: the full set that must be present to use them. The order is unspecified - run + * {@link SchemaManifest.sortInDependencyOrder} on the result when a load order is needed. + * @note Requested names the iModel does not contain are ignored; check + * {@link SchemaManifest.findByName} first to detect them. + */ + public getSchemaClosure(requestedNames: Iterable): string[] { + const result: string[] = []; + const visited = new Set(); + + const visit = (entry: SchemaManifestEntry): void => { + if (visited.has(entry)) + return; + visited.add(entry); + result.push(entry.name); + for (const reference of entry.references) + visit(reference); + }; + + for (const name of requestedNames) { + const entry = this._byLowerName.get(name.toLowerCase()); + if (entry !== undefined) + visit(entry); + } + return result; + } + + /** Orders the given schema names so each appears after every schema it references, directly or + * transitively. References through schemas not in `schemaNames` are still honored, so the order is + * correct even when an intermediate schema is left out. Names the iModel does not contain are + * ignored, and reference cycles - which EC forbids - are broken arbitrarily rather than looping. + * @internal + */ + public sortInDependencyOrder(schemaNames: Iterable): string[] { + const requested = new Set(); + for (const name of schemaNames) { + const entry = this._byLowerName.get(name.toLowerCase()); + if (entry !== undefined) + requested.add(entry); + } + + const result: string[] = []; + const visited = new Set(); + const visiting = new Set(); + + const visit = (entry: SchemaManifestEntry): void => { + if (visited.has(entry) || visiting.has(entry)) + return; + visiting.add(entry); + for (const reference of entry.references) + visit(reference); + visiting.delete(entry); + visited.add(entry); + if (requested.has(entry)) + result.push(entry.name); + }; + + for (const entry of requested) + visit(entry); + return result; + } +} diff --git a/core/ecschema-metadata/src/SchemaView.ts b/core/ecschema-metadata/src/SchemaView/SchemaView.ts similarity index 91% rename from core/ecschema-metadata/src/SchemaView.ts rename to core/ecschema-metadata/src/SchemaView/SchemaView.ts index 91603dfde015..0b1881277a9d 100644 --- a/core/ecschema-metadata/src/SchemaView.ts +++ b/core/ecschema-metadata/src/SchemaView/SchemaView.ts @@ -7,8 +7,8 @@ */ import { type ClassData, ClassModifier, ClassType, type EnumerationData, type EnumeratorData, type KoqData, type PropCategoryData, type PropertyDef, PropertyKind, type PropertyRef, type RelConstraintData, type SchemaData, SchemaViewPrimitiveType } from "./SchemaViewInterfaces"; -import { parseSchemaViewBlob } from "./SchemaViewBinaryReader"; -import { StrengthDirection, StrengthType } from "./ECObjects"; +import { parseSchemaViewBlob, SchemaViewMergeContext } from "./SchemaViewBinaryReader"; +import { StrengthDirection, StrengthType } from "../ECObjects"; // Module-local symbol used as the storage key on SchemaView instances. Mirrors the pattern in // core-backend/src/internal/Symbols.ts (e.g. `_nativeDb` on IModelDb): the data is reachable @@ -78,17 +78,23 @@ export class SchemaView { private _schemaToken: string; private _outdated = false; + /** When present, this view is a *husk*: it retains the live builder and cross-reference maps, so + * `mergeFragment` can append further fragment blobs. Undefined for one-shot views from `fromBinary`. */ + private readonly _mergeContext?: SchemaViewMergeContext; + /** @internal */ - constructor(data: SchemaViewData, schemaToken?: string) { + constructor(data: SchemaViewData, schemaToken?: string, mergeContext?: SchemaViewMergeContext) { this[_storage] = { ...data, transitiveBaseCache: new Map>(), derivedClassMap: undefined, }; this._schemaToken = schemaToken ?? ""; + this._mergeContext = mergeContext; } - /** SHA3-256 content hash of the ec_ schema tables at the time this view was built. + /** Cache-invalidation token identifying the schemas this view was built from: a hash of every + * schema's name and version (see `PRAGMA checksum(schema_token)`), not of their full contents. * Empty string if not set (e.g., when built from a builder without a token). * @beta */ @@ -106,6 +112,12 @@ export class SchemaView { */ public markOutdated(): void { this._outdated = true; } + /** Set the cache-invalidation token. Called by the host after loading schema data into a husk, + * since the token is not known until the first blob is fetched. + * @internal + */ + public setSchemaToken(token: string): void { this._schemaToken = token; } + /** Number of schemas in the view. */ public get schemaCount(): number { return this[_storage].schemas.length; } @@ -164,18 +176,39 @@ export class SchemaView { /** Parse a binary blob into a SchemaView. Synchronous. * @param blob - The binary blob from `PRAGMA schema_view`. - * @param schemaToken - Optional SHA3-256 content hash for cache invalidation. + * @param schemaToken - Optional cache-invalidation token (schema name+version hash; see `PRAGMA checksum(schema_token)`). * @beta */ public static fromBinary(blob: Uint8Array, schemaToken?: string): SchemaView { return parseSchemaViewBlob(blob, schemaToken); } - /** Build from a pre-populated builder (used by the binary parser). + /** Create an empty, *mergeable* view (a "husk"). It contains no schemas until `mergeFragment` is + * called. Each merged fragment is appended into the same instance, so flyweights and cached + * cross-references obtained earlier stay valid - indices are append-only and never reordered. + * @note Fragments must be merged in dependency order: a fragment may only reference schemas from + * fragments already merged. + * @internal + */ + public static createMergeable(schemaToken?: string): SchemaView { + const ctx = new SchemaViewMergeContext(); + return new SchemaView(ctx.builder.assembleData(), schemaToken, ctx); + } + + /** Merge one fragment blob into this view. Only valid on a view created via `createMergeable`. + * Synchronous. Any flyweight or cached index obtained before the call remains valid. + * @note If the merge throws (e.g. on a malformed blob), the view may be left partially extended + * and must be discarded by the host. * @internal */ - public static fromBuilder(builder: SchemaViewBuilder, schemaToken?: string): SchemaView { - return builder.build(schemaToken); + public mergeFragment(blob: Uint8Array): void { + if (this._mergeContext === undefined) + throw new Error("SchemaView is not mergeable: create it via SchemaView.createMergeable to merge fragments."); + this._mergeContext.mergeBlob(blob); + // A new fragment can add subclasses to an already-present base class, so the derived-class map + // must be rebuilt. transitiveBaseCache stays valid: a class's ancestors are always merged + // before it (dependency order), so an append never adds an ancestor to an already-cached class. + this[_storage].derivedClassMap = undefined; } // --- Internal helpers used by view objects --- @@ -1193,6 +1226,17 @@ export class SchemaViewBuilder { // For PropertyDef dedup private readonly _propDefMap = new Map(); // signature string -> defIdx + // Lookup maps - owned by the builder so a husk can share and extend them across fragment merges. + // `build()` and each merge call `extendLookupMaps()`; the view receives these same Map objects + // via `assembleData()`, so in-place growth is visible. + private readonly _schemaByName = new Map(); + private readonly _schemaByAlias = new Map(); + private readonly _classByName = new Map>(); + private readonly _enumByName = new Map>(); + private readonly _koqByName = new Map>(); + private readonly _catByName = new Map>(); + private _lookupMapsBuiltUpto = 0; // number of schemas whose lookup entries are already built + /** Intern a string, returning its SID. Empty/undefined strings return 0. * Interning is case-sensitive - "MyLabel" and "MYLABEL" get distinct SIDs. * The `lowerStrings` array provides case-insensitive lookup without mutating display values. @@ -1294,6 +1338,24 @@ export class SchemaViewBuilder { /** The current count of class mixins (used to set mixinStartIdx). */ public get classMixinCount(): number { return this._classMixins.length; } + // The counts below are the base indices fragment merging uses to translate a fragment's local + // indices into global ones. + + /** The current count of schemas. @internal */ + public get schemaCount(): number { return this._schemas.length; } + + /** The current count of classes. @internal */ + public get classCount(): number { return this._classes.length; } + + /** The current count of enumerations. @internal */ + public get enumerationCount(): number { return this._enumerations.length; } + + /** The current count of KindOfQuantities. @internal */ + public get koqCount(): number { return this._koqs.length; } + + /** The current count of property categories. @internal */ + public get propCategoryCount(): number { return this._propCategories.length; } + /** Get a string by SID. @internal */ public getString(sid: number): string { return this._strings[sid]; } @@ -1308,46 +1370,54 @@ export class SchemaViewBuilder { /** Freeze all data and produce an immutable SchemaView. */ public build(schemaToken?: string): SchemaView { - const schemaByName = new Map(); - const schemaByAlias = new Map(); - const classByName = new Map>(); - const enumByName = new Map>(); - const koqByName = new Map>(); - const catByName = new Map>(); - - // Build schema lookup maps - for (let i = 0; i < this._schemas.length; i++) { + this.extendLookupMaps(); + return new SchemaView(this.assembleData(), schemaToken); + } + + /** Build lookup-map entries for any schemas added since the last call. Idempotent and + * append-only, so it is safe to call after each fragment merge. A schema's item ranges must + * already be finalized (via `updateSchemaRanges`) before it is processed. + * @internal */ + public extendLookupMaps(): void { + for (let i = this._lookupMapsBuiltUpto; i < this._schemas.length; i++) { const s = this._schemas[i]; - schemaByName.set(this._lowerStrings[s.nameStringIdx], i); + this._schemaByName.set(this._lowerStrings[s.nameStringIdx], i); if (s.aliasStringIdx !== 0) - schemaByAlias.set(this._lowerStrings[s.aliasStringIdx], i); + this._schemaByAlias.set(this._lowerStrings[s.aliasStringIdx], i); // Build class-by-name map for this schema const classMap = new Map(); for (let c = s.classRangeStart; c < s.classRangeStart + s.classCount; c++) classMap.set(this._lowerStrings[this._classes[c].nameStringIdx], c); - classByName.set(i, classMap); + this._classByName.set(i, classMap); // Build enum-by-name map for this schema const eMap = new Map(); for (let e = s.enumRangeStart; e < s.enumRangeStart + s.enumCount; e++) eMap.set(this._lowerStrings[this._enumerations[e].nameStringIdx], e); - enumByName.set(i, eMap); + this._enumByName.set(i, eMap); // Build koq-by-name map for this schema const kMap = new Map(); for (let k = s.koqRangeStart; k < s.koqRangeStart + s.koqCount; k++) kMap.set(this._lowerStrings[this._koqs[k].nameStringIdx], k); - koqByName.set(i, kMap); + this._koqByName.set(i, kMap); // Build category-by-name map for this schema const cMap = new Map(); for (let p = s.catRangeStart; p < s.catRangeStart + s.catCount; p++) cMap.set(this._lowerStrings[this._propCategories[p].nameStringIdx], p); - catByName.set(i, cMap); + this._catByName.set(i, cMap); } + this._lookupMapsBuiltUpto = this._schemas.length; + } - return new SchemaView({ + /** Assemble a {@link SchemaViewData} bag that references this builder's live arrays and lookup + * maps. Continued building - fragment merges that append to the arrays and extend the maps in + * place - is visible through the shared references, so a husk holding this data sees merged + * schemas without rebuilding. @internal */ + public assembleData(): SchemaViewData { + return { strings: this._strings, lowerStrings: this._lowerStrings, schemas: this._schemas, @@ -1361,13 +1431,13 @@ export class SchemaViewBuilder { enumerators: this._enumerators, koqs: this._koqs, propCategories: this._propCategories, - schemaByName, - schemaByAlias, - classByName, - enumByName, - koqByName, - catByName, - }, schemaToken); + schemaByName: this._schemaByName, + schemaByAlias: this._schemaByAlias, + classByName: this._classByName, + enumByName: this._enumByName, + koqByName: this._koqByName, + catByName: this._catByName, + }; } /** Produce a dedup signature for a PropertyDef. Label and priority are excluded because diff --git a/core/ecschema-metadata/src/SchemaViewBinaryReader.ts b/core/ecschema-metadata/src/SchemaView/SchemaViewBinaryReader.ts similarity index 82% rename from core/ecschema-metadata/src/SchemaViewBinaryReader.ts rename to core/ecschema-metadata/src/SchemaView/SchemaViewBinaryReader.ts index be35fa44b55e..ba3c70914c2b 100644 --- a/core/ecschema-metadata/src/SchemaViewBinaryReader.ts +++ b/core/ecschema-metadata/src/SchemaView/SchemaViewBinaryReader.ts @@ -8,7 +8,7 @@ import { BentleyError, Logger } from "@itwin/core-bentley"; import { SchemaView, SchemaViewBuilder } from "./SchemaView"; -import { StrengthDirection, StrengthType } from "./ECObjects"; +import { StrengthDirection, StrengthType } from "../ECObjects"; import { ClassData, ClassModifier, ClassType, PropertyDef, PropertyKind, schemaViewFormatVersion, SchemaViewPrimitiveType } from "./SchemaViewInterfaces"; /** Binary record tags for the SchemaView blob format. @@ -166,7 +166,37 @@ function expectTag(reader: BinaryReader, expected: Tag): void { throw new Error(`Expected tag 0x${expected.toString(16)} but found 0x${tag.toString(16)} at offset ${reader.pos - 1}`); } -/** Parse a schema view blob (binary format) into a `SchemaView`. +/** Reusable parse/merge context for SchemaView blobs. Holds the live `SchemaViewBuilder` plus the + * cross-reference maps that must persist across fragment merges. Each `mergeBlob` appends one blob's + * schemas into the shared builder and resolves cross-references - including cross-fragment ones - + * against the accumulated maps. A whole-iModel blob is the degenerate case of a single fragment. + * @note Fragments must be merged in dependency order, so a fragment only references schemas already + * merged by an earlier one (or excluded schemas, which resolve to "not present"). + * @internal + */ +export class SchemaViewMergeContext { + public readonly builder = new SchemaViewBuilder(); + + // Cross-reference maps. Persist across fragments: a row id / name from a later fragment resolves + // against a class, enum, koq, or category merged by an earlier fragment. + public readonly schemaECIdToIdx = new Map(); + public readonly enumRowIdToIdx = new Map(); + public readonly koqRowIdToIdx = new Map(); + public readonly catRowIdToIdx = new Map(); + public readonly classRowIdToIdx = new Map(); + // "schemaname:classname" (both lowercased) -> global class index. Resolves base/mixin/constraint + // refs, which the blob encodes as schema+class name pairs, across fragment boundaries. + public readonly classResolver = new Map(); + // global schemaIdx -> schema name, for building resolver keys and dangling-ref diagnostics. + public readonly schemaNames: string[] = []; + + /** Parse one fragment blob and append its schemas into the shared builder. Synchronous. */ + public mergeBlob(data: Uint8Array): void { + mergeFragmentBlob(this, data); + } +} + +/** Parse a schema view blob (binary format) into a fresh `SchemaView`. * * Layout: Header, PropertyDefTable, SchemaTable, EnumTable, KoQTable, PropCatTable, ClassTable, StringTable. * Each table is count-prefixed. Schema items carry their schema's ecInstanceId for ownership resolution. @@ -177,6 +207,17 @@ function expectTag(reader: BinaryReader, expected: Tag): void { * @internal */ export function parseSchemaViewBlob(data: Uint8Array, schemaToken?: string): SchemaView { + const ctx = new SchemaViewMergeContext(); + ctx.mergeBlob(data); + return ctx.builder.build(schemaToken); +} + +/** Parse one fragment blob and append it into the context's builder, resolving cross-references + * against the context's accumulated global maps. Per-fragment scratch state stays local; everything + * that must outlive the fragment lives on `ctx`. + */ +function mergeFragmentBlob(ctx: SchemaViewMergeContext, data: Uint8Array): void { + const builder = ctx.builder; const reader = new BinaryReader(data); // Header: magic(4) + version(1) + stringTableOffset(4) @@ -189,19 +230,12 @@ export function parseSchemaViewBlob(data: Uint8Array, schemaToken?: string): Sch const stOffset = reader.readU32(); reader.parseStringTable(stOffset); - const builder = new SchemaViewBuilder(); - - // Cross-reference maps (ecInstanceId -> builder array index) - const schemaEcIdToIdx = new Map(); - const enumRowIdToIdx = new Map(); - const koqRowIdToIdx = new Map(); - const catRowIdToIdx = new Map(); - const classRowIdToIdx = new Map(); + // Global index of the first schema this fragment adds. Local schema index = global - base. + const schemaBaseIdx = builder.schemaCount; - // Per-schema metadata for name-based class resolution - const schemaInfos: Array<{ name: string; schemaIdx: number; classNameToIdx: Map }> = []; + const { schemaECIdToIdx: schemaEcIdToIdx, enumRowIdToIdx, koqRowIdToIdx, catRowIdToIdx, classRowIdToIdx, classResolver } = ctx; - // Per-schema item range tracking (indexed by schemaIdx) + // Per-schema item range tracking, indexed by LOCAL schema index (0-based within this fragment). const schemaEnumStarts: number[] = []; const schemaEnumCounts: number[] = []; const schemaKoqStarts: number[] = []; @@ -211,15 +245,15 @@ export function parseSchemaViewBlob(data: Uint8Array, schemaToken?: string): Sch const schemaClassStarts: number[] = []; const schemaClassCounts: number[] = []; - // Deferred class data for cross-reference resolution + // Deferred class data for cross-reference resolution (local to this fragment). const pendingClasses: PendingClass[] = []; // ---- PropertyDefTable ---- expectTag(reader, Tag.PropertyDefTable); const defCount = reader.readU32(); - // Each PreParsedDef consumes at least ~30 bytes (mix of u8/u16/u32 fields + string refs). - // Use a conservative lower bound of 8 bytes to catch wildly oversized counts on malformed blobs. - reader.validateCount(defCount, 8, "PropertyDefTable"); + // Each PreParsedDef is a fixed 46 bytes (10x u32 + 1x u16 + 4x u8). The bound is deliberately + // about half that, so a valid blob never trips the check. + reader.validateCount(defCount, 23, "PropertyDefTable"); const preParsedDefs: PreParsedDef[] = new Array(defCount); for (let i = 0; i < defCount; i++) { preParsedDefs[i] = { @@ -244,7 +278,8 @@ export function parseSchemaViewBlob(data: Uint8Array, schemaToken?: string): Sch // ---- SchemaTable ---- expectTag(reader, Tag.SchemaTable); const schemaCount = reader.readU32(); - reader.validateCount(schemaCount, 8, "SchemaTable"); + // Fixed 27 bytes per record (4x SRef + 3x u16 + u32 + u8). + reader.validateCount(schemaCount, 13, "SchemaTable"); for (let i = 0; i < schemaCount; i++) { const name = reader.readSRef(); const vRead = reader.readU16(); @@ -272,34 +307,40 @@ export function parseSchemaViewBlob(data: Uint8Array, schemaToken?: string): Sch isHidden, }); schemaEcIdToIdx.set(ecInstanceId, schemaIdx); - schemaInfos.push({ name, schemaIdx, classNameToIdx: new Map() }); + ctx.schemaNames[schemaIdx] = name; schemaEnumStarts.push(0); schemaEnumCounts.push(0); schemaKoqStarts.push(0); schemaKoqCounts.push(0); schemaCatStarts.push(0); schemaCatCounts.push(0); schemaClassStarts.push(0); schemaClassCounts.push(0); } - /** Track an item's schema ownership and update range counters. */ + /** Track an item's schema ownership and update its owning schema's range counters. `globalIdx` + * is the item's global index in the builder; `starts`/`counts` are indexed by LOCAL schema index. + * An item must be owned by a schema in its own fragment, so a foreign owner is a malformed blob. */ function trackItem(schemaEcId: number, globalIdx: number, starts: number[], counts: number[]): number { const schemaIdx = schemaEcIdToIdx.get(schemaEcId); if (schemaIdx === undefined) throw new Error(`SchemaView blob: unknown schema ecInstanceId ${schemaEcId}`); - if (counts[schemaIdx] === 0) - starts[schemaIdx] = globalIdx; - counts[schemaIdx]++; + const localIdx = schemaIdx - schemaBaseIdx; + if (localIdx < 0 || localIdx >= schemaCount) + throw new Error(`SchemaView fragment: item references schema ecInstanceId ${schemaEcId} not present in this fragment`); + if (counts[localIdx] === 0) + starts[localIdx] = globalIdx; + counts[localIdx]++; return schemaIdx; } // ---- EnumTable ---- expectTag(reader, Tag.EnumTable); const enumTotalCount = reader.readU32(); - reader.validateCount(enumTotalCount, 8, "EnumTable"); + // Fixed 27 bytes per record (5x SRef + u16 + u8 + u32); enumerators live inside a JSON SRef. + reader.validateCount(enumTotalCount, 13, "EnumTable"); for (let i = 0; i < enumTotalCount; i++) { const schemaEcId = reader.readU32(); - const schemaIdx = trackItem(schemaEcId, i, schemaEnumStarts, schemaEnumCounts); + const schemaIdx = trackItem(schemaEcId, builder.enumerationCount, schemaEnumStarts, schemaEnumCounts); const eName = reader.readSRef(); - const ePrimType = reader.readU8(); + const ePrimType = reader.readU16(); const eIsStrict = reader.readU8() !== 0; const eLabel = reader.readSRef(); const eDesc = reader.readSRef(); @@ -345,10 +386,11 @@ export function parseSchemaViewBlob(data: Uint8Array, schemaToken?: string): Sch // ---- KoQTable ---- expectTag(reader, Tag.KoQTable); const koqTotalCount = reader.readU32(); - reader.validateCount(koqTotalCount, 8, "KoQTable"); + // Fixed 36 bytes per record (6x SRef + f64 + u32). + reader.validateCount(koqTotalCount, 18, "KoQTable"); for (let i = 0; i < koqTotalCount; i++) { const schemaEcId = reader.readU32(); - const schemaIdx = trackItem(schemaEcId, i, schemaKoqStarts, schemaKoqCounts); + const schemaIdx = trackItem(schemaEcId, builder.koqCount, schemaKoqStarts, schemaKoqCounts); const kName = reader.readSRef(); const kLabel = reader.readSRef(); @@ -374,10 +416,11 @@ export function parseSchemaViewBlob(data: Uint8Array, schemaToken?: string): Sch // ---- PropCatTable ---- expectTag(reader, Tag.PropCatTable); const catTotalCount = reader.readU32(); - reader.validateCount(catTotalCount, 8, "PropCatTable"); + // Fixed 24 bytes per record (4x SRef + i32 + u32). + reader.validateCount(catTotalCount, 12, "PropCatTable"); for (let i = 0; i < catTotalCount; i++) { const schemaEcId = reader.readU32(); - const schemaIdx = trackItem(schemaEcId, i, schemaCatStarts, schemaCatCounts); + const schemaIdx = trackItem(schemaEcId, builder.propCategoryCount, schemaCatStarts, schemaCatCounts); const pcName = reader.readSRef(); const pcLabel = reader.readSRef(); @@ -399,11 +442,13 @@ export function parseSchemaViewBlob(data: Uint8Array, schemaToken?: string): Sch // ---- ClassTable ---- expectTag(reader, Tag.ClassTable); const classTotalCount = reader.readU32(); - reader.validateCount(classTotalCount, 8, "ClassTable"); + // Minimum 27 bytes per record (3x SRef + 3x u8 + u32 + 2x u16 count prefixes); records grow with + // relationship strength/direction and the base-class, prop-ref, and constraint sub-lists. + reader.validateCount(classTotalCount, 13, "ClassTable"); for (let i = 0; i < classTotalCount; i++) { const schemaEcId = reader.readU32(); - const schemaIdx = trackItem(schemaEcId, i, schemaClassStarts, schemaClassCounts); - const schemaInfo = schemaInfos[schemaIdx]; + const schemaIdx = trackItem(schemaEcId, builder.classCount, schemaClassStarts, schemaClassCounts); + const schemaName = ctx.schemaNames[schemaIdx]; const cName = reader.readSRef(); const cType = reader.readU8(); @@ -500,7 +545,7 @@ export function parseSchemaViewBlob(data: Uint8Array, schemaToken?: string): Sch targetConstraintIdx: -1, isHidden: cIsHidden, }); - schemaInfo.classNameToIdx.set(cName.toLowerCase(), classIdx); + classResolver.set(`${schemaName.toLowerCase()}:${cName.toLowerCase()}`, classIdx); classRowIdToIdx.set(cEcInstanceId, classIdx); pendingClasses.push({ @@ -517,14 +562,15 @@ export function parseSchemaViewBlob(data: Uint8Array, schemaToken?: string): Sch baseClasses, propRefs, constraints, - schemaName: schemaInfo.name, + schemaName, isHidden: cIsHidden, }); } // ---- Finalize per-schema item ranges ---- + // Local index i maps to global schema index schemaBaseIdx + i; range starts are already global. for (let i = 0; i < schemaCount; i++) { - builder.updateSchemaRanges(i, { + builder.updateSchemaRanges(schemaBaseIdx + i, { classRangeStart: schemaClassStarts[i] ?? 0, classCount: schemaClassCounts[i] ?? 0, enumRangeStart: schemaEnumStarts[i] ?? 0, @@ -536,13 +582,6 @@ export function parseSchemaViewBlob(data: Uint8Array, schemaToken?: string): Sch }); } - // Build a global name resolver: "SchemaName:ClassName" -> classIdx - const classResolver = new Map(); - for (const s of schemaInfos) { - for (const [lowerName, idx] of s.classNameToIdx) - classResolver.set(`${s.name.toLowerCase()}:${lowerName}`, idx); - } - // Resolve pre-parsed defs to PropertyDef objects. Maps preParsedDef index -> builder defIdx. const danglingRefs: string[] = []; const resolvedDefMap = new Map(); @@ -715,6 +754,8 @@ export function parseSchemaViewBlob(data: Uint8Array, schemaToken?: string): Sch Logger.logWarning("ecschema-metadata.SchemaView", `${danglingRefs.length} unresolved cross-reference(s) in schema view blob (likely from excluded schemas):\n ${lines.join("\n ")}`); } - return builder.build(schemaToken); + // Bring the builder's lookup maps up to date for the schemas this fragment added. Their item + // ranges were finalized above, so the per-schema name maps build correctly. + builder.extendLookupMaps(); } diff --git a/core/ecschema-metadata/src/SchemaViewInterfaces.ts b/core/ecschema-metadata/src/SchemaView/SchemaViewInterfaces.ts similarity index 99% rename from core/ecschema-metadata/src/SchemaViewInterfaces.ts rename to core/ecschema-metadata/src/SchemaView/SchemaViewInterfaces.ts index 5aa55a8d9d46..19d8dea695bb 100644 --- a/core/ecschema-metadata/src/SchemaViewInterfaces.ts +++ b/core/ecschema-metadata/src/SchemaView/SchemaViewInterfaces.ts @@ -65,7 +65,7 @@ export enum SchemaViewPrimitiveType { IGeometry = 0xa01, } -import { StrengthDirection, StrengthType } from "./ECObjects"; +import { StrengthDirection, StrengthType } from "../ECObjects"; /** Internal storage for a schema. Schemas own contiguous ranges of classes, enums, KoQs, and categories. * @internal diff --git a/core/ecschema-metadata/src/SchemaView/SchemaViewManager.ts b/core/ecschema-metadata/src/SchemaView/SchemaViewManager.ts new file mode 100644 index 000000000000..0d535e69589c --- /dev/null +++ b/core/ecschema-metadata/src/SchemaView/SchemaViewManager.ts @@ -0,0 +1,271 @@ +/*--------------------------------------------------------------------------------------------- +* Copyright (c) Bentley Systems, Incorporated. All rights reserved. +* See LICENSE.md in the project root for license terms and full copyright notice. +*--------------------------------------------------------------------------------------------*/ +/** @packageDocumentation + * @module Schema + */ + +import { SchemaManifest } from "./SchemaManifest"; +import { SchemaView } from "./SchemaView"; + +/** One schema-view blob with its cache-invalidation token, as fetched by a + * {@link SchemaViewDataProvider}. Full and fragment blobs share this shape. + * @beta + */ +export interface SchemaViewBlob { + /** The binary schema metadata (the `data` column of `PRAGMA schema_view` / `schema_view_fragment`). */ + readonly data: Uint8Array; + /** Schema-identity hash of the iModel's whole schema set (the `schemaToken` column). The token is + * the same for a full blob or any fragment. Empty string when unavailable. */ + readonly schemaToken: string; +} + +/** The data source a {@link SchemaViewManager} loads schema-view data from, implemented by the hosts + * that own the query APIs: `IModelDb` on the backend and `IModelConnection` on the frontend. The + * manager deals only in schema names, blobs and the manifest; everything transport-specific - pragma + * strings, format-version pinning, mapping names to `ec_Schema` ids - belongs to the provider. + * @beta + */ +export interface SchemaViewDataProvider { + /** Fetch the blob containing every schema in the iModel (`PRAGMA schema_view`). */ + fetchFullBlob(): Promise; + + /** Fetch one blob containing exactly the given schemas (`PRAGMA schema_view_fragment`). The + * requested set is always dependency-closed - the manager computes the reference closure from the + * manifest before calling. */ + fetchFragmentBlob(schemaNames: readonly string[]): Promise; + + /** Fetch the reference graph of every schema in the iModel, built from ECDbMeta + * (`meta.ECSchemaDef` + `meta.SchemaHasSchemaReferences`; see {@link SchemaManifest.fromRows}). */ + fetchManifest(): Promise; + + /** Fetch the current schema-identity token (`PRAGMA checksum(schema_token)`), used by + * {@link SchemaViewManager.invalidateIfChanged} to detect schema changes. */ + fetchSchemaToken(): Promise; +} + +/** Options for `getSchemaView` (see `IModelDb.getSchemaView` / `IModelConnection.getSchemaView`, + * which delegate to {@link SchemaViewManager.getSchemaView}). + * @beta + */ +export interface GetSchemaViewArgs { + /** When provided, return a view loaded with at least these schemas plus their references, instead + * of every schema in the iModel. + * + * The view accumulates: one instance is reused across calls, so a later request - with different + * schemas or with no filter at all - merges the still-missing schemas into the same view and + * everything requested earlier stays available. It resets when the iModel's schemas change. + * + * Names the iModel does not contain are ignored. Omitting this option loads all schemas, identical + * to calling `getSchemaView()` with no arguments. + */ + readonly schemas?: readonly string[]; + + /** When `true`, discard whatever is currently loaded and rebuild the view from scratch before + * returning it. The previously returned view instance (if any) is marked outdated. Like every other + * request this is serialized behind any in-flight load, so it never leaves the view in an invalid + * intermediate state. + * @internal + */ + readonly forceReload?: boolean; +} + +/** Owns the lifetime of one iModel's {@link (SchemaView:class)}: lazy loading, incremental (filtered) + * hydration, serialization of concurrent requests, and invalidation. Hosts (`IModelDb`, + * `IModelConnection`) hold one instance and delegate to it; all data access goes through the + * host-implemented {@link SchemaViewDataProvider}. + * @beta + */ +export class SchemaViewManager { + private readonly _dataProvider: SchemaViewDataProvider; + + // Single accumulating schema view, held as a promise so every getSchemaView call chains onto it + // and loads never overlap. An `undefined` field, or a promise resolving to `undefined` (the + // continuation queued by `reset`), both mean nothing is loaded. + private _viewPromise?: Promise; + + // Loaded lazily the first time an incremental (filtered) load is needed. `undefined` means either + // the view is fully loaded, or nothing was loaded yet (`_viewPromise` is undefined too). + private _manifest?: SchemaManifest; + + // Lower-cased names already merged into the SchemaView. Only used in incremental mode, to decide + // what a later filtered request still needs. + private readonly _loadedSchemaNames = new Set(); + + public constructor(dataProvider: SchemaViewDataProvider) { + this._dataProvider = dataProvider; + } + + /** Get the schema view, loading whatever the request needs that is not present yet. See + * {@link GetSchemaViewArgs} for filtering and reload semantics; hosts document the full + * user-facing contract on their `getSchemaView` methods. + */ + public async getSchemaView(args?: GetSchemaViewArgs): Promise { + // Chain onto the previous request so loads run one at a time and only a single in-flight load + // ever mutates the shared state. `_loadSchemaView` swallows the previous load's failure; each + // caller observes its own outcome through the promise returned here. + const previous = this._viewPromise; + const next = this._loadSchemaView(previous, args?.schemas, args?.forceReload === true); + this._viewPromise = next; + return next; + } + + /** Throw away the current schema view. Called by the host when it *knows* schemas may have + * changed (e.g. `IModelDb.clearCaches` after a schema import). The teardown chains behind any + * in-flight load, marks the discarded view outdated and clears the incremental bookkeeping. The + * next getSchemaView chains after this and starts over. + */ + public reset(): void { + if (this._viewPromise) { + this._viewPromise = this._viewPromise.then( + (view) => { view?.markOutdated(); this._resetIncrementalState(); return undefined; }, + () => { this._resetIncrementalState(); return undefined; }, + ); + } + } + + /** Check whether the iModel's schemas have changed since the current view was built, and discard + * the view only if they have. For hosts that *cannot* determine whether an operation actually + * modified schemas - e.g. `BriefcaseConnection.pullChanges` on the frontend, whose IPC response + * carries only the new changeset id, not the applied changesets' types. Discarding after every + * such operation would reload unnecessarily in the common case where schemas are unchanged, so + * this compares the cheap schema-identity token instead. + * @note If the token cannot be fetched, the view is discarded rather than risking stale metadata. + */ + public async invalidateIfChanged(): Promise { + const previous = this._viewPromise; + if (previous === undefined) + return; + // Queue the check on the same chain as loading, so a getSchemaView arriving while the token is + // in flight cannot swap in a new promise for the same stale view and hide the schema change. + const next = this._invalidateIfChanged(previous); + this._viewPromise = next; + await next; + } + + /** Serialized body of {@link SchemaViewManager.invalidateIfChanged}. Resolves to the view to keep, + * or to `undefined` when it was discarded and the next getSchemaView has to start over. + */ + private async _invalidateIfChanged(previous: Promise): Promise { + let existing: SchemaView | undefined; + try { + existing = await previous; + } catch { + // The load failed and already cleared the incremental state; nothing to invalidate. + return undefined; + } + // Nothing loaded (an earlier reset) - nothing to invalidate. + if (existing === undefined) + return undefined; + // A husk with no token and no cached manifest never fetched anything. With a manifest an + // incremental request did run (loading only the manifest, e.g. all its names were missing), and + // that manifest can go stale - fall through so the token check drops it; a live token can never + // equal "". + if (existing.schemaToken === "" && this._manifest === undefined) + return existing; + + try { + if (await this._dataProvider.fetchSchemaToken() === existing.schemaToken) + return existing; + } catch { + // Cannot verify the cached view: drop it rather than risk stale metadata. + } + existing.markOutdated(); + this._resetIncrementalState(); + return undefined; + } + + /** Serialized body of {@link SchemaViewManager.getSchemaView}. Waits for the prior load, optionally discards + * everything (`forceReload`), then ensures the requested schemas (or all schemas, when no filter + * is given) are present in the single accumulating view. On failure it resets and rejects, so the + * next call retries from scratch. + */ + private async _loadSchemaView(previous: Promise | undefined, schemas: readonly string[] | undefined, forceReload: boolean): Promise { + // A prior failure or reset leaves no usable view, so we simply start over. + let currentView: SchemaView | undefined; + if (previous !== undefined) { + try { + currentView = await previous; + } catch { + currentView = undefined; + } + } + + if (forceReload) { + currentView?.markOutdated(); + currentView = undefined; + this._resetIncrementalState(); + } + + try { + return await this._ensureSchemasLoaded(currentView, schemas); + } catch (err) { + // `_viewPromise` deliberately keeps pointing at this (now rejected) promise: the next call + // chains onto it, catches the rejection above and rebuilds, which also avoids stomping any + // newer queued load. A failed merge may have left the current view partially extended, so + // mark it outdated for callers still holding it. + currentView?.markOutdated(); + this._resetIncrementalState(); + throw err; + } + } + + /** Clear the incremental schema-view bookkeeping. */ + private _resetIncrementalState(): void { + this._manifest = undefined; + this._loadedSchemaNames.clear(); + } + + /** Ensures the requested schemas (or all schemas, when no filter is given) are present in + * `currentView`, or in a freshly created view when nothing is loaded yet, and returns it. + * + * The strategy is fixed by the *first* load: + * - No filter -> fetch every schema as one full blob (one round trip, best cross-schema dedup). + * `_manifest` stays `undefined` and all later calls short-circuit. + * - Filter -> fetch only the requested schemas and their references as a fragment blob, and keep + * the manifest and loaded-name set to extend the *same* view on later calls. Once every schema + * has been loaded this way, `_manifest` is cleared to collapse back to full mode. + */ + private async _ensureSchemasLoaded(currentView: SchemaView | undefined, schemas: readonly string[] | undefined): Promise { + const isFirstLoad = currentView === undefined; + // No manifest means everything is loaded; in incremental mode a filtered request is satisfied as + // soon as every requested name is present. + if (!isFirstLoad && + (this._manifest === undefined || + (schemas !== undefined && schemas.every((name) => this._loadedSchemaNames.has(name.toLowerCase()))))) + return currentView; + + if (isFirstLoad && schemas === undefined) { + const blob = await this._dataProvider.fetchFullBlob(); + const schemaView = SchemaView.fromBinary(blob.data, blob.schemaToken); + this._resetIncrementalState(); + return schemaView; + } + + let manifest = this._manifest; + if (manifest === undefined) + manifest = this._manifest = await this._dataProvider.fetchManifest(); + + // No filter in incremental mode means "load whatever is left". + const requested = schemas ?? manifest.getAvailableSchemaNames(); + // The manifest returns the whole closure of the request; dropping the already-loaded schemas is + // our concern, not the manifest's. Fragment load order does not matter (see the writer). + const namesToLoad = manifest.getSchemaClosure(requested).filter((name) => !this._loadedSchemaNames.has(name.toLowerCase())); + const husk = currentView ?? SchemaView.createMergeable(); + if (namesToLoad.length > 0) { + const blob = await this._dataProvider.fetchFragmentBlob(namesToLoad); + husk.mergeFragment(blob.data); + husk.setSchemaToken(blob.schemaToken); + // Record every closure entry as loaded, including *excluded* schemas (e.g. CoreCustomAttributes) + // that the writer emits no rows for and so never appear in the view. Tracking names rather than + // view contents is what lets a later request prune them instead of re-fetching. + for (const name of namesToLoad) + this._loadedSchemaNames.add(name.toLowerCase()); + } + // Once everything is loaded, drop the incremental state so future requests hit the fast path above. + if (schemas === undefined || (this._loadedSchemaNames.size >= manifest.schemaCount && manifest.entries.every((entry) => this._loadedSchemaNames.has(entry.name.toLowerCase())))) + this._resetIncrementalState(); + return husk; + } +} diff --git a/core/ecschema-metadata/src/ecschema-metadata.ts b/core/ecschema-metadata/src/ecschema-metadata.ts index a80971bdae2c..ac61ab9be711 100644 --- a/core/ecschema-metadata/src/ecschema-metadata.ts +++ b/core/ecschema-metadata/src/ecschema-metadata.ts @@ -50,9 +50,11 @@ export * from "./IncrementalLoading/ECSqlSchemaLocater"; export * from "./IncrementalLoading/IncrementalSchemaLocater"; export { CustomAttribute, CustomAttributeContainerProps } from "./Metadata/CustomAttribute"; export { SchemaGraph } from "./utils/SchemaGraph"; -export * from "./SchemaViewBinaryReader"; -export * from "./SchemaView"; -export * from "./SchemaViewInterfaces"; +export * from "./SchemaView/SchemaViewBinaryReader"; +export * from "./SchemaView/SchemaView"; +export * from "./SchemaView/SchemaViewInterfaces"; +export * from "./SchemaView/SchemaManifest"; +export * from "./SchemaView/SchemaViewManager"; export * from "./Localization/LocalizationTypes"; export * from "./Localization/LocalizationProvider"; export * from "./Localization/SchemaLocalization"; diff --git a/core/ecschema-metadata/src/test/SchemaManifest.test.ts b/core/ecschema-metadata/src/test/SchemaManifest.test.ts new file mode 100644 index 000000000000..dc120bf22e04 --- /dev/null +++ b/core/ecschema-metadata/src/test/SchemaManifest.test.ts @@ -0,0 +1,272 @@ +/*--------------------------------------------------------------------------------------------- +* Copyright (c) Bentley Systems, Incorporated. All rights reserved. +* See LICENSE.md in the project root for license terms and full copyright notice. +*--------------------------------------------------------------------------------------------*/ + +import { describe, expect, it } from "vitest"; +import { SchemaManifest, SchemaManifestEntry, SchemaManifestReferenceRow, SchemaManifestSchemaRow } from "../SchemaView/SchemaManifest"; + +/** Spec for one schema in a test manifest: its name, minor version, and the names it references. */ +interface EntrySpec { + name: string; + minorVersion?: number; + refs?: string[]; +} + +/** Builds a {@link SchemaManifest} from name-based specs, wiring each entry's references to the + * entry objects named in `refs`. Unknown reference names are ignored. */ +function makeManifest(specs: EntrySpec[]): SchemaManifest { + const byName = new Map(); + const entries = specs.map((spec) => { + const entry = { + name: spec.name, + readVersion: 1, + writeVersion: 0, + minorVersion: spec.minorVersion ?? 0, + references: [] as SchemaManifestEntry[], + }; + byName.set(spec.name.toLowerCase(), entry); + return entry; + }); + for (const spec of specs) { + const entry = byName.get(spec.name.toLowerCase())!; + for (const ref of spec.refs ?? []) { + const target = byName.get(ref.toLowerCase()); + if (target !== undefined) + entry.references.push(target); + } + } + return new SchemaManifest(entries); +} + +/** A small reference graph mirroring a real iModel's shape: standard schemas at the bottom, domain + * schemas on top. + * + * Units CoreCustomAttributes + * ^ ^ + * Formats->Units | + * ^ ^ | + * \ \ | + * BisCore -> Units, Formats, CoreCustomAttributes + * ^ ^ + * Generic Functional (both -> BisCore) + */ +function makeSpecs(): EntrySpec[] { + return [ + { name: "Units" }, + { name: "Formats", refs: ["Units"] }, + { name: "CoreCustomAttributes", minorVersion: 1 }, + { name: "BisCore", minorVersion: 15, refs: ["Units", "Formats", "CoreCustomAttributes"] }, + { name: "Generic", minorVersion: 2, refs: ["BisCore"] }, + { name: "Functional", refs: ["BisCore"] }, + ]; +} + +/** Asserts `before` appears at an earlier position than `after` in a load order. */ +function expectOrder(order: readonly string[], before: string, after: string): void { + const beforeIndex = order.indexOf(before); + const afterIndex = order.indexOf(after); + expect(beforeIndex, `${before} should be present`).to.be.greaterThanOrEqual(0); + expect(afterIndex, `${after} should be present`).to.be.greaterThanOrEqual(0); + expect(beforeIndex, `${before} should load before ${after}`).to.be.lessThan(afterIndex); +} + +describe("SchemaManifest", () => { + it("exposes schema count, names, and per-schema identity", () => { + const manifest = makeManifest(makeSpecs()); + + expect(manifest.schemaCount).to.equal(6); + expect(manifest.getAvailableSchemaNames()).to.deep.equal(["Units", "Formats", "CoreCustomAttributes", "BisCore", "Generic", "Functional"]); + + const bisCore = manifest.findByName("BisCore"); + expect(bisCore?.name).to.equal("BisCore"); + expect(bisCore?.minorVersion).to.equal(15); + }); + + it("looks up schemas case-insensitively", () => { + const manifest = makeManifest(makeSpecs()); + expect(manifest.findByName("biscore")?.name).to.equal("BisCore"); + expect(manifest.findByName("BISCORE")?.name).to.equal("BisCore"); + expect(manifest.findByName("NotASchema")).to.be.undefined; + }); + + it("wires references to the referenced entry objects", () => { + const manifest = makeManifest(makeSpecs()); + const bisCore = manifest.findByName("BisCore")!; + const refNames = bisCore.references.map((entry) => entry.name).sort(); + expect(refNames).to.deep.equal(["CoreCustomAttributes", "Formats", "Units"]); + expect(bisCore.references).to.include(manifest.findByName("Units")); + expect(manifest.findByName("Units")!.references).to.deep.equal([]); + }); +}); + +describe("SchemaManifest.getSchemaClosure", () => { + it("returns only the requested schema when it has no references", () => { + const manifest = makeManifest(makeSpecs()); + expect(manifest.getSchemaClosure(["Units"])).to.deep.equal(["Units"]); + }); + + it("returns the requested schema plus its transitive references", () => { + const manifest = makeManifest(makeSpecs()); + const names = manifest.getSchemaClosure(["Generic"]).sort(); + expect(names).to.deep.equal(["BisCore", "CoreCustomAttributes", "Formats", "Generic", "Units"]); + }); + + it("merges the closures of multiple requested schemas without duplicates", () => { + const manifest = makeManifest(makeSpecs()); + const names = manifest.getSchemaClosure(["Generic", "Functional"]); + expect(new Set(names).size).to.equal(names.length); // no duplicates + expect([...names].sort()).to.deep.equal(["BisCore", "CoreCustomAttributes", "Formats", "Functional", "Generic", "Units"]); + }); + + it("ignores requested names the iModel does not contain", () => { + const manifest = makeManifest(makeSpecs()); + const names = manifest.getSchemaClosure(["Generic", "NotASchema"]); + expect(names).to.include("Generic"); + expect(names).to.not.include("NotASchema"); + }); + + it("terminates on a reference cycle rather than looping", () => { + // EC prohibits reference cycles, but the walk must still be defensive. + const manifest = makeManifest([ + { name: "A", refs: ["B"] }, + { name: "B", refs: ["A"] }, + ]); + expect(manifest.getSchemaClosure(["A"]).sort()).to.deep.equal(["A", "B"]); + }); +}); + +describe("SchemaManifest.sortInDependencyOrder", () => { + it("orders each schema after the schemas it references", () => { + const manifest = makeManifest(makeSpecs()); + const order = manifest.sortInDependencyOrder(["Generic", "Units", "Formats", "BisCore", "CoreCustomAttributes"]); + expect(order.slice().sort()).to.deep.equal(["BisCore", "CoreCustomAttributes", "Formats", "Generic", "Units"]); + expectOrder(order, "Units", "Formats"); + expectOrder(order, "Units", "BisCore"); + expectOrder(order, "Formats", "BisCore"); + expectOrder(order, "CoreCustomAttributes", "BisCore"); + expectOrder(order, "BisCore", "Generic"); + }); + + it("returns only the given names, not their references", () => { + const manifest = makeManifest(makeSpecs()); + // BisCore's references (Units, Formats, CoreCustomAttributes) are not in the input, so they are + // not added - only the two requested names come back. + const order = manifest.sortInDependencyOrder(["Generic", "BisCore"]); + expect(order.slice().sort()).to.deep.equal(["BisCore", "Generic"]); + expectOrder(order, "BisCore", "Generic"); + }); + + it("honors dependencies that run through a schema left out of the input", () => { + const manifest = makeManifest(makeSpecs()); + // Generic -> BisCore -> Units. With BisCore excluded, Units must still come before Generic. + const order = manifest.sortInDependencyOrder(["Generic", "Units"]); + expect(order.slice().sort()).to.deep.equal(["Generic", "Units"]); + expectOrder(order, "Units", "Generic"); + }); + + it("emits a schema shared through two omitted intermediates once, ahead of both requesters", () => { + // A -> B -> C and F -> D -> C. With B and D excluded from the input, C is reached along two + // separate paths but must appear exactly once and before both A and F, giving C, A, F or C, F, A. + const manifest = makeManifest([ + { name: "C" }, + { name: "B", refs: ["C"] }, + { name: "D", refs: ["C"] }, + { name: "A", refs: ["B"] }, + { name: "F", refs: ["D"] }, + ]); + const order = manifest.sortInDependencyOrder(["A", "C", "F"]); + expect(order.slice().sort()).to.deep.equal(["A", "C", "F"]); // no duplicate C + expectOrder(order, "C", "A"); + expectOrder(order, "C", "F"); + }); + + it("ignores names the iModel does not contain", () => { + const manifest = makeManifest(makeSpecs()); + const order = manifest.sortInDependencyOrder(["Generic", "NotASchema"]); + expect(order).to.deep.equal(["Generic"]); + }); + + it("terminates on a reference cycle rather than looping", () => { + const manifest = makeManifest([ + { name: "A", refs: ["B"] }, + { name: "B", refs: ["A"] }, + ]); + expect(manifest.sortInDependencyOrder(["A", "B"]).slice().sort()).to.deep.equal(["A", "B"]); + }); +}); + +describe("SchemaManifest.fromRows", () => { + /** Rows as the two ECDbMeta queries would return them for the makeSpecs graph, with deliberately + * non-contiguous ids to catch any assumption that ids are dense or ordered. */ + function makeRows(): { schemaRows: SchemaManifestSchemaRow[], referenceRows: SchemaManifestReferenceRow[] } { + return { + schemaRows: [ + { ecInstanceId: 3, name: "Units", versionMajor: 1, versionWrite: 0, versionMinor: 0 }, + { ecInstanceId: 17, name: "Formats", versionMajor: 1, versionWrite: 0, versionMinor: 0 }, + { ecInstanceId: 5, name: "CoreCustomAttributes", versionMajor: 1, versionWrite: 0, versionMinor: 1 }, + { ecInstanceId: 131, name: "BisCore", versionMajor: 1, versionWrite: 0, versionMinor: 15 }, + { ecInstanceId: 145, name: "Generic", versionMajor: 1, versionWrite: 0, versionMinor: 2 }, + ], + referenceRows: [ + { sourceECInstanceId: 17, targetECInstanceId: 3 }, // Formats -> Units + { sourceECInstanceId: 131, targetECInstanceId: 3 }, // BisCore -> Units + { sourceECInstanceId: 131, targetECInstanceId: 17 }, // BisCore -> Formats + { sourceECInstanceId: 131, targetECInstanceId: 5 }, // BisCore -> CoreCustomAttributes + { sourceECInstanceId: 145, targetECInstanceId: 131 }, // Generic -> BisCore + ], + }; + } + + it("builds entries with identity and version fields from the schema rows", () => { + const { schemaRows, referenceRows } = makeRows(); + const manifest = SchemaManifest.fromRows(schemaRows, referenceRows); + + expect(manifest.schemaCount).to.equal(5); + expect(manifest.getAvailableSchemaNames()).to.deep.equal(["Units", "Formats", "CoreCustomAttributes", "BisCore", "Generic"]); + + const bisCore = manifest.findByName("BisCore")!; + expect(bisCore.readVersion).to.equal(1); + expect(bisCore.writeVersion).to.equal(0); + expect(bisCore.minorVersion).to.equal(15); + }); + + it("wires reference edges by id to the entry objects, carrying no ids into the manifest", () => { + const { schemaRows, referenceRows } = makeRows(); + const manifest = SchemaManifest.fromRows(schemaRows, referenceRows); + + const bisCore = manifest.findByName("BisCore")!; + expect(bisCore.references.map((entry) => entry.name).sort()).to.deep.equal(["CoreCustomAttributes", "Formats", "Units"]); + // Edges point at the same entry objects the manifest exposes, not copies. + expect(bisCore.references).to.include(manifest.findByName("Units")); + // The graph works end to end: closure over the wired edges. + expect(manifest.getSchemaClosure(["Generic"]).sort()).to.deep.equal(["BisCore", "CoreCustomAttributes", "Formats", "Generic", "Units"]); + }); + + it("skips reference rows with unknown endpoints", () => { + const { schemaRows } = makeRows(); + const manifest = SchemaManifest.fromRows(schemaRows, [ + { sourceECInstanceId: 999, targetECInstanceId: 3 }, // unknown source + { sourceECInstanceId: 131, targetECInstanceId: 999 }, // unknown target + { sourceECInstanceId: 131, targetECInstanceId: 3 }, // valid: BisCore -> Units + ]); + expect(manifest.findByName("BisCore")!.references.map((entry) => entry.name)).to.deep.equal(["Units"]); + }); + + it("skips self-referential and duplicate reference rows", () => { + const { schemaRows } = makeRows(); + const manifest = SchemaManifest.fromRows(schemaRows, [ + { sourceECInstanceId: 131, targetECInstanceId: 131 }, // self-reference + { sourceECInstanceId: 131, targetECInstanceId: 3 }, // BisCore -> Units + { sourceECInstanceId: 131, targetECInstanceId: 3 }, // exact duplicate + ]); + expect(manifest.findByName("BisCore")!.references.map((entry) => entry.name)).to.deep.equal(["Units"]); + }); + + it("builds an empty manifest from no rows", () => { + const manifest = SchemaManifest.fromRows([], []); + expect(manifest.schemaCount).to.equal(0); + expect(manifest.getAvailableSchemaNames()).to.deep.equal([]); + expect(manifest.getSchemaClosure(["Anything"])).to.deep.equal([]); + }); +}); diff --git a/core/ecschema-metadata/src/test/SchemaView.test.ts b/core/ecschema-metadata/src/test/SchemaView.test.ts index 54dc2f4c3b80..ca5974f83550 100644 --- a/core/ecschema-metadata/src/test/SchemaView.test.ts +++ b/core/ecschema-metadata/src/test/SchemaView.test.ts @@ -4,8 +4,8 @@ *--------------------------------------------------------------------------------------------*/ import { describe, expect, it } from "vitest"; -import { SchemaView } from "../SchemaView"; -import { schemaViewFormatVersion } from "../SchemaViewInterfaces"; +import { SchemaView } from "../SchemaView/SchemaView"; +import { schemaViewFormatVersion } from "../SchemaView/SchemaViewInterfaces"; /** Build the smallest valid SchemaView blob: header + empty PropertyDefTable, SchemaTable, * EnumTable, KoQTable, PropCatTable, ClassTable, then a string table with one empty entry. */ @@ -167,3 +167,33 @@ describe("SchemaView.fromBinary - malformed blobs", () => { expect(() => SchemaView.fromBinary(buf)).to.throw(/tag/i); }); }); + +describe("SchemaView husk merging", () => { + // These cover the pure-TS husk lifecycle contract (createMergeable / mergeFragment) without + // fabricating a full fragment blob - the writer-to-reader blob contract is exercised end to end + // in the backend tests (core/backend/.../SchemaViewFragmentLoading.test.ts). The empty minimal + // blob is enough to assert the husk's identity, emptiness, and guard behaviors. + + it("createMergeable returns an empty, current view", () => { + const view = SchemaView.createMergeable("husk-token"); + expect(view.schemaCount).to.equal(0); + expect(view.classCount).to.equal(0); + expect(view.isOutdated).to.be.false; + expect(view.schemaToken).to.equal("husk-token"); + }); + + it("merges an (empty) fragment into the same instance without throwing", () => { + const view = SchemaView.createMergeable(); + view.mergeFragment(makeMinimalBlob()); + expect(view.schemaCount).to.equal(0); + // A second empty merge is also a no-op and must not throw. + view.mergeFragment(makeMinimalBlob()); + expect(view.schemaCount).to.equal(0); + }); + + it("throws when merging into a non-mergeable (fromBinary) view", () => { + const view = SchemaView.fromBinary(makeMinimalBlob()); + expect(() => view.mergeFragment(makeMinimalBlob())).to.throw(/not mergeable/i); + }); +}); + diff --git a/core/ecschema-metadata/src/test/SchemaViewManager.test.ts b/core/ecschema-metadata/src/test/SchemaViewManager.test.ts new file mode 100644 index 000000000000..33510cb80c05 --- /dev/null +++ b/core/ecschema-metadata/src/test/SchemaViewManager.test.ts @@ -0,0 +1,80 @@ +/*--------------------------------------------------------------------------------------------- +* Copyright (c) Bentley Systems, Incorporated. All rights reserved. +* See LICENSE.md in the project root for license terms and full copyright notice. +*--------------------------------------------------------------------------------------------*/ + +import { describe, expect, it } from "vitest"; +import { SchemaView, SchemaViewBuilder } from "../SchemaView/SchemaView"; +import { SchemaViewDataProvider, SchemaViewManager } from "../SchemaView/SchemaViewManager"; + +interface Deferred { + promise: Promise; + resolve: (value: T) => void; +} + +function makeDeferred(): Deferred { + let resolve!: (value: T) => void; + const promise = new Promise((res) => { resolve = res; }); + return { promise, resolve }; +} + +/** A provider whose `fetchSchemaToken` blocks until the test releases it, so a `getSchemaView` call + * can be made while an invalidation is waiting on the token. Every other fetch throws: a blob fetch + * only happens if the manager decided to rebuild, which the tests assert on by its message. */ +function makeBlockingProvider(tokenRequested: Deferred, token: Deferred): SchemaViewDataProvider { + return { + fetchFullBlob: async () => { throw new Error("reload attempted"); }, + fetchFragmentBlob: async () => { throw new Error("reload attempted"); }, + fetchManifest: async () => { throw new Error("reload attempted"); }, + fetchSchemaToken: async () => { tokenRequested.resolve(); return token.promise; }, + }; +} + +/** Seeds a manager with `view` already cached, as it would be after a real getSchemaView call. */ +function installCachedView(manager: SchemaViewManager, view: SchemaView): void { + (manager as any)._viewPromise = Promise.resolve(view); +} + +describe("SchemaViewManager.invalidateIfChanged", () => { + it("discards a stale view even when a getSchemaView call arrives while the token is in flight", async () => { + const view = new SchemaViewBuilder().build("token-before"); + const tokenRequested = makeDeferred(); + const token = makeDeferred(); + const manager = new SchemaViewManager(makeBlockingProvider(tokenRequested, token)); + installCachedView(manager, view); + + const invalidated = manager.invalidateIfChanged(); + await tokenRequested.promise; // the token fetch is now in flight + + // A cache read landing mid-check would satisfy itself from the cached view and swap in a new + // promise. The invalidation must still see the token change and drop the view. + const concurrentRead = manager.getSchemaView(); + + token.resolve("token-after"); + await invalidated; + + expect(view.isOutdated).toBe(true); + // The read was serialized behind the invalidation, so it rebuilds instead of returning the + // discarded view. + await expect(concurrentRead).rejects.toThrow("reload attempted"); + }); + + it("keeps the view when the token is unchanged, and a concurrent read returns that same view", async () => { + const view = new SchemaViewBuilder().build("stable-token"); + const tokenRequested = makeDeferred(); + const token = makeDeferred(); + const manager = new SchemaViewManager(makeBlockingProvider(tokenRequested, token)); + installCachedView(manager, view); + + const invalidated = manager.invalidateIfChanged(); + await tokenRequested.promise; + + const concurrentRead = manager.getSchemaView(); + + token.resolve("stable-token"); + await invalidated; + + expect(view.isOutdated).toBe(false); + expect(await concurrentRead).toBe(view); + }); +}); diff --git a/core/frontend/src/IModelConnection.ts b/core/frontend/src/IModelConnection.ts index 714e559b9b51..0a638d6fff9a 100644 --- a/core/frontend/src/IModelConnection.ts +++ b/core/frontend/src/IModelConnection.ts @@ -34,7 +34,7 @@ import { Tiles } from "./Tiles"; import { ViewState } from "./ViewState"; import { _requestSnap } from "./common/internal/Symbols"; import { IpcApp } from "./IpcApp"; -import { SchemaContext, SchemaView, schemaViewFormatVersion } from "@itwin/ecschema-metadata"; +import { type GetSchemaViewArgs, SchemaContext, SchemaManifest, type SchemaManifestReferenceRow, type SchemaManifestSchemaRow, SchemaView, type SchemaViewBlob, type SchemaViewDataProvider, schemaViewFormatVersion, SchemaViewManager } from "@itwin/ecschema-metadata"; import { ECSchemaRpcLocater, RpcIncrementalSchemaLocater } from '@itwin/ecschema-rpcinterface-common'; @@ -160,7 +160,9 @@ export abstract class IModelConnection extends IModel { public fontMap?: FontMap; // eslint-disable-line @typescript-eslint/no-deprecated private _schemaContext?: SchemaContext; - private _schemasPromise?: Promise; + // Created lazily on the first getSchemaView call. Owns the SchemaView's lifetime and does all its + // data access through the SchemaViewDataProvider implemented below. + private _schemaViewManager?: SchemaViewManager; /** Load the FontMap for this IModelConnection. * @returns Returns a Promise that is fulfilled when the FontMap member of this IModelConnection is valid. @@ -659,36 +661,25 @@ export abstract class IModelConnection extends IModel { /** Get the schema view for this iModel. The view is built lazily on * first call by fetching compact binary schema data via `PRAGMA schema_view` through - * the existing queryRows RPC (ConcurrentQuery). Subsequent calls return the cached view. - * Multiple concurrent callers share a single in-flight fetch. + * the existing queryRows RPC (ConcurrentQuery). * * The returned `SchemaView` is a lightweight, read-only, synchronous API for * navigating schema metadata - classes, properties, relationships, enumerations, etc. * It is the recommended default for runtime read-only metadata access and is significantly * faster and lower-memory than [[schemaContext]]. Use [[schemaContext]] for custom-attribute * deserialization or anywhere you need the full ecschema-metadata object graph. + * + * Every call shares one accumulating view instance and concurrent calls are serialized, so a + * caller never observes a partially loaded view. The instance is discarded once the connection + * detects that schemas changed, for example after [[BriefcaseConnection.pullChanges]]; the next + * call builds a new one. See [GetSchemaViewArgs]($ecschema-metadata) for the arguments. * @beta */ - public async getSchemaView(): Promise { - if (this._schemasPromise) { - const ctx = await this._schemasPromise; - if (!ctx.isOutdated) - return ctx; - } - // Capture the in-flight promise locally so the rejection handler only clears - // `_schemasPromise` if it still points at this build. A concurrent invalidation + - // re-fetch could otherwise replace the field before our fetch fails, and a naive - // `_schemasPromise = undefined` would clobber that newer reference. - const inflight = this._fetchSchemas(); - this._schemasPromise = inflight; - inflight.catch(() => { - if (this._schemasPromise === inflight) - this._schemasPromise = undefined; - }); - return inflight; + public async getSchemaView(args?: GetSchemaViewArgs): Promise { + this._schemaViewManager ??= new SchemaViewManager(this._createSchemaViewDataProvider()); + return this._schemaViewManager.getSchemaView(args); } - /** * Checks whether the iModel's schemas have changed since the current cached [[SchemaView]] was * built, and discards the cache only if they have. @@ -699,7 +690,7 @@ export abstract class IModelConnection extends IModel { * returns only the new changeset id, not the list of applied changesets with their types. * Unconditionally discarding the cached [[SchemaView]] after every such operation would cause * unnecessary reloads in the common case where schemas are unchanged. This method avoids that - * cost by fetching a lightweight schema checksum via `PRAGMA checksum(ecdb_schema)` and + * cost by fetching a lightweight schema token via `PRAGMA checksum(schema_token)` and * comparing it against the token stored in the cached view. Only when the token differs is the * cache discarded. * @@ -708,60 +699,63 @@ export abstract class IModelConnection extends IModel { * @internal */ protected async invalidateSchemaViewIfChanged(): Promise { - if (!this._schemasPromise) - return; - const existingPromise = this._schemasPromise; - let existing: SchemaView; - try { - existing = await existingPromise; - } catch { - // The cached promise itself failed; drop it so the next getSchemaView() retries. - if (this._schemasPromise === existingPromise) - this._schemasPromise = undefined; - return; - } - if (!existing.schemaToken) - return; - try { - const reader = this.createQueryReader("PRAGMA checksum(ecdb_schema)"); - const result = await reader.next(); - if (result.done) - throw new Error("PRAGMA checksum(ecdb_schema) returned no rows"); - const liveToken = result.value.sha3_256 as string; - if (liveToken !== existing.schemaToken) { - if (this._schemasPromise === existingPromise) - this._schemasPromise = undefined; - existing.markOutdated(); - } - } catch { - // The checksum check is called right after operations that may have changed schemas - // (e.g., pullChanges). If we cannot verify the cached view is still current, drop it - // rather than risk returning stale metadata indefinitely. The next getSchemaView() call - // will reload. We also mark the existing view outdated so any retained references can - // observe the invalidation. - if (this._schemasPromise === existingPromise) { - this._schemasPromise = undefined; - existing.markOutdated(); - } - } + await this._schemaViewManager?.invalidateIfChanged(); + } + + /** The [SchemaViewDataProvider]($ecschema-metadata) backing this iModel's [[getSchemaView]]: the + * transport-specific half of schema-view loading, issued through the queryRows RPC + * (ConcurrentQuery). The frontend *pins* the blob format version in both pragmas, because the + * backend it talks to can be older or newer; the pin makes it return a blob this code can parse, + * or fail cleanly. + */ + private _createSchemaViewDataProvider(): SchemaViewDataProvider { + return { + fetchFullBlob: async () => this._fetchSchemaBlob(`PRAGMA schema_view(${schemaViewFormatVersion})`), + // Names are ECNames, so a comma can never occur in one. Native re-validates each token as an + // ECName and fails the pragma on an unknown name. The `v;` prefix pins the format version. + fetchFragmentBlob: async (schemaNames) => this._fetchSchemaBlob(`PRAGMA schema_view_fragment('v${schemaViewFormatVersion};${schemaNames.join(",")}')`), + fetchManifest: async () => { + const schemaRows: SchemaManifestSchemaRow[] = []; + const schemaSql = "SELECT ECInstanceId, Name, VersionMajor, VersionWrite, VersionMinor FROM meta.ECSchemaDef"; + for await (const row of this.createQueryReader(schemaSql)) { + // ECInstanceId arrives as a hex Id64String. `ec_` metadata rowids carry no briefcase + // prefix, so the local id is the full value. + schemaRows.push({ ecInstanceId: Id64.getLocalId(row[0]), name: row[1], versionMajor: row[2], versionWrite: row[3], versionMinor: row[4] }); + } + + const referenceRows: SchemaManifestReferenceRow[] = []; + const referenceSql = "SELECT SourceECInstanceId, TargetECInstanceId FROM meta.SchemaHasSchemaReferences"; + for await (const row of this.createQueryReader(referenceSql)) + referenceRows.push({ sourceECInstanceId: Id64.getLocalId(row[0]), targetECInstanceId: Id64.getLocalId(row[1]) }); + + return SchemaManifest.fromRows(schemaRows, referenceRows); + }, + fetchSchemaToken: async () => { + const reader = this.createQueryReader("PRAGMA checksum(schema_token)"); + const result = await reader.next(); + if (result.done) + throw new IModelError(IModelStatus.BadRequest, "PRAGMA checksum(schema_token) returned no rows"); + return result.value.sha3_256 as string; + }, + }; } - private async _fetchSchemas(): Promise { - // PRAGMA returns exactly one row with format, formatVersion, data (binary), schemaToken. - // Important: only call reader.next() once - do NOT use `for await` on PRAGMA results. - // ConcurrentQuery wraps regular ECSQL in LIMIT/OFFSET for pagination but skips this for - // PRAGMAs. If the serialized result exceeds the memory threshold, the response is marked - // "Partial", and a `for await` loop would re-issue the same PRAGMA forever since PRAGMAs - // don't support OFFSET-based pagination. - const reader = this.createQueryReader(`PRAGMA schema_view(${schemaViewFormatVersion})`); + /** Fetch one schema-view blob (full or fragment). Both `PRAGMA schema_view` and + * `PRAGMA schema_view_fragment` return a single row with the same columns. */ + private async _fetchSchemaBlob(pragma: string): Promise { + // Only call reader.next() once - do NOT use `for await` on PRAGMA results. ConcurrentQuery wraps + // regular ECSQL in LIMIT/OFFSET for pagination but skips this for PRAGMAs; if the serialized + // result exceeds the memory threshold, the response is marked "Partial", and a `for await` loop + // would re-issue the same PRAGMA forever since PRAGMAs don't support OFFSET-based pagination. + const reader = this.createQueryReader(pragma); const result = await reader.next(); if (result.done) - throw new IModelError(IModelStatus.BadRequest, "PRAGMA schema_view returned no rows"); + throw new IModelError(IModelStatus.BadRequest, `${pragma} returned no rows`); const data = result.value.data as Uint8Array | undefined; const token = result.value.schemaToken as string | undefined; if (data === undefined || data === null) - throw new IModelError(IModelStatus.BadRequest, "PRAGMA schema_view returned null data column"); - return SchemaView.fromBinary(data, token ?? ""); + throw new IModelError(IModelStatus.BadRequest, `${pragma} returned null data column`); + return { data, schemaToken: token ?? "" }; } } diff --git a/core/frontend/src/test/SchemaViewInvalidation.test.ts b/core/frontend/src/test/SchemaViewInvalidation.test.ts index 0effa5f4495f..b9c9d3ebb063 100644 --- a/core/frontend/src/test/SchemaViewInvalidation.test.ts +++ b/core/frontend/src/test/SchemaViewInvalidation.test.ts @@ -3,7 +3,7 @@ * See LICENSE.md in the project root for license terms and full copyright notice. *--------------------------------------------------------------------------------------------*/ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { SchemaViewBuilder } from "@itwin/ecschema-metadata"; +import { type SchemaView, SchemaViewBuilder, SchemaViewManager } from "@itwin/ecschema-metadata"; import { BriefcaseConnection } from "../BriefcaseConnection"; import { IModelApp } from "../IModelApp"; import { IpcApp } from "../IpcApp"; @@ -28,13 +28,25 @@ async function openMockedConnection(): Promise { } /** - * Returns a minimal mock ECSqlReader whose single row reports the given checksum. - * Used to simulate the response from `PRAGMA checksum(ecdb_schema)`. + * Returns a minimal mock ECSqlReader whose single row reports the given schema token. + * Used to simulate the response from `PRAGMA checksum(schema_token)`. */ -function makeChecksumReader(checksum: string) { - // Property key must match the column name returned by PRAGMA checksum(ecdb_schema). +function makeSchemaTokenReader(token: string) { + // Property key must match the column name returned by PRAGMA checksum(schema_token). // eslint-disable-next-line @typescript-eslint/naming-convention - return { next: vi.fn().mockResolvedValue({ done: false, value: { "sha3_256": checksum } }) } as any; + return { next: vi.fn().mockResolvedValue({ done: false, value: { "sha3_256": token } }) } as any; +} + +/** + * Installs a `SchemaViewManager` on the connection with `view` already cached, mirroring the state + * after a real `getSchemaView` call. Returns the manager so tests can inspect `_viewPromise` after + * `invalidateSchemaViewIfChanged`/`pullChanges` runs. + */ +function installCachedSchemaView(conn: BriefcaseConnection, view: SchemaView): SchemaViewManager { + const manager = new SchemaViewManager((conn as any)._createSchemaViewDataProvider()); + (manager as any)._viewPromise = Promise.resolve(view); + (conn as any)._schemaViewManager = manager; + return manager; } describe("SchemaView frontend cache invalidation", () => { @@ -52,67 +64,67 @@ describe("SchemaView frontend cache invalidation", () => { it("invalidateSchemaViewIfChanged is a no-op when no schema view is cached", async () => { const conn = await openMockedConnection(); - // _schemasPromise is undefined - the method should return immediately without error. + // _schemaViewManager is undefined - the method should return immediately without error. await (conn as any).invalidateSchemaViewIfChanged(); - expect((conn as any)._schemasPromise).toBeUndefined(); + expect((conn as any)._schemaViewManager).toBeUndefined(); }); - it("invalidateSchemaViewIfChanged preserves the cached view when the schema checksum is unchanged", async () => { + it("invalidateSchemaViewIfChanged preserves the cached view when the schema token is unchanged", async () => { const conn = await openMockedConnection(); - const token = "sha3-256-checksum-abc"; + const token = "sha3-256-token-abc"; const view = new SchemaViewBuilder().build(token); - (conn as any)._schemasPromise = Promise.resolve(view); + const manager = installCachedSchemaView(conn, view); - // PRAGMA checksum returns the same token - schemas did not change. - vi.spyOn(conn, "createQueryReader").mockReturnValue(makeChecksumReader(token)); + // PRAGMA checksum(schema_token) returns the same token - schemas did not change. + vi.spyOn(conn, "createQueryReader").mockReturnValue(makeSchemaTokenReader(token)); await (conn as any).invalidateSchemaViewIfChanged(); expect(view.isOutdated).toBe(false); - expect((conn as any)._schemasPromise).not.toBeUndefined(); + expect(await (manager as any)._viewPromise).toBe(view); }); it("invalidateSchemaViewIfChanged marks old view outdated and clears cache when schemas changed", async () => { const conn = await openMockedConnection(); - const view = new SchemaViewBuilder().build("checksum-before"); - (conn as any)._schemasPromise = Promise.resolve(view); + const view = new SchemaViewBuilder().build("token-before"); + const manager = installCachedSchemaView(conn, view); - // PRAGMA checksum returns a different token - schemas changed since the view was built. - vi.spyOn(conn, "createQueryReader").mockReturnValue(makeChecksumReader("checksum-after")); + // PRAGMA checksum(schema_token) returns a different token - schemas changed since the view was built. + vi.spyOn(conn, "createQueryReader").mockReturnValue(makeSchemaTokenReader("token-after")); await (conn as any).invalidateSchemaViewIfChanged(); expect(view.isOutdated).toBe(true); - expect((conn as any)._schemasPromise).toBeUndefined(); + expect(await (manager as any)._viewPromise).toBeUndefined(); }); it("pullChanges invalidates schema view when a schema changeset was pulled", async () => { const conn = await openMockedConnection(); - const view = new SchemaViewBuilder().build("checksum-before"); - (conn as any)._schemasPromise = Promise.resolve(view); + const view = new SchemaViewBuilder().build("token-before"); + const manager = installCachedSchemaView(conn, view); - // Simulate that the pull brought in a schema changeset - checksum differs afterwards. - vi.spyOn(conn, "createQueryReader").mockReturnValue(makeChecksumReader("checksum-after")); + // Simulate that the pull brought in a schema changeset - token differs afterwards. + vi.spyOn(conn, "createQueryReader").mockReturnValue(makeSchemaTokenReader("token-after")); await conn.pullChanges(); expect(view.isOutdated).toBe(true); - expect((conn as any)._schemasPromise).toBeUndefined(); + expect(await (manager as any)._viewPromise).toBeUndefined(); }); it("pullChanges preserves schema view when no schema change was pulled", async () => { const conn = await openMockedConnection(); - const token = "stable-checksum"; + const token = "stable-token"; const view = new SchemaViewBuilder().build(token); - (conn as any)._schemasPromise = Promise.resolve(view); + const manager = installCachedSchemaView(conn, view); - // Checksum is unchanged - a data-only pull should leave the schema view intact. - vi.spyOn(conn, "createQueryReader").mockReturnValue(makeChecksumReader(token)); + // Token is unchanged - a data-only pull should leave the schema view intact. + vi.spyOn(conn, "createQueryReader").mockReturnValue(makeSchemaTokenReader(token)); await conn.pullChanges(); expect(view.isOutdated).toBe(false); - expect((conn as any)._schemasPromise).not.toBeUndefined(); + expect(await (manager as any)._viewPromise).toBe(view); }); }); \ No newline at end of file diff --git a/docs/changehistory/NextVersion.md b/docs/changehistory/NextVersion.md index c2cb663757d0..f884eb701a29 100644 --- a/docs/changehistory/NextVersion.md +++ b/docs/changehistory/NextVersion.md @@ -13,6 +13,10 @@ publish: false - [ChangesetReader.setBatchSize](#changesetreadersetbatchsize) - [@itwin/core-geometry](#itwincore-geometry) - [Region Boolean enhancements](#region-boolean-enhancements) + - [RegionBooleanXYOptions.simplifyUnion](#regionbooleanxyoptionssimplifyunion) + - [RegionBooleanXYOptions.operationGroupA/B](#regionbooleanxyoptionsoperationgroupab) + - [@itwin/ecschema-metadata](#itwinecschema-metadata) + - [SchemaView: load only a subset of schemas, cheaper cache invalidation](#schemaview-load-only-a-subset-of-schemas-cheaper-cache-invalidation) ## Quantity formatting @@ -146,3 +150,25 @@ const result = RegionOps.regionBooleanXY([venn0, venn1, venn2, venn3], outer, Re ![Venn Output Region](./assets/venn-boolean-in-one-go.jpg "Outer loop minus Venn intersection") Note: The same result can also be obtained with `RegionBinaryOpType.BMinusA` instead of `RegionBinaryOpType.Parity`. To perform only the 4-way intersection, pass `undefined` for the second input group. + +## @itwin/ecschema-metadata + +### SchemaView: load only a subset of schemas, cheaper cache invalidation + +[SchemaView](../learning/metadata/SchemaView.md) got two changes to improve performance on iModels with very large schemas. + +`getSchemaView()` on [IModelDb]($backend) and [IModelConnection]($frontend) now takes an optional `schemas` argument to load only a subset: + +```ts +// Unchanged: loads every schema in the iModel. +const full = await iModel.getSchemaView(); + +// New: load only BisCore and its references. +const view = await iModel.getSchemaView({ schemas: ["BisCore"] }); +view.findClass("BisCore.Subject"); // present +view.findClass("Generic.PhysicalObject"); // undefined - not loaded +``` + +The returned view accumulates: a later call with different schemas merges their reference closure into the same view, so schemas loaded by an earlier call stay available, and once the full set is loaded (via a call with no filter), every subsequent call is a synchronous no-op. + +Second, cache invalidation - detecting whether an iModel's schemas changed since a view was cached - switched from hashing the full contents of every schema table to hashing only each schema's name and version. On a large iModel (~30 GB, ~100 schemas), this dropped the check from over a second of CPU time to about a millisecond. The one accepted limitation: a schema whose content changes without a version bump is not detected. This can only happen with dynamic schemas, since ECDb requires a version increment for in-place re-import of any other schema. diff --git a/docs/learning/ECSqlReference/Pragmas.md b/docs/learning/ECSqlReference/Pragmas.md index 40c8e3880cfa..706b459c52d7 100644 --- a/docs/learning/ECSqlReference/Pragmas.md +++ b/docs/learning/ECSqlReference/Pragmas.md @@ -10,7 +10,7 @@ PRAGMA help | pragma | type | descr | | ----------------------------- | ------ | ------------------------------------------------------------------------------- | -| checksum | global | checksum([ecdb_schema\|ecdb_map\|sqlite_schema]) return sha3 checksum for data. | +| checksum | global | checksum([ecdb_schema\|ecdb_map\|sqlite_schema\|schema_token]) return sha3 checksum for data. | | ecdb_ver | global | return current and file profile versions | | experimental_features_enabled | global | enable/disable experimental features | | validate_ecsql_writes | global | enable/disable validation for values in an ecsql statement | @@ -19,6 +19,7 @@ PRAGMA help | integrity_check | global | performs integrity checks on ECDb | | parse_tree | global | parse_tree(ecsql) return parse tree of ecsql. | | schema_view | global | returns a curated subset of schema metadata as a binary blob | +| schema_view_fragment | global | returns a chosen subset of schemas as a binary blob, for incremental loading | | disqualify_type_index | class | set/get disqualify_type_index flag for a given ECClass | ## `PRAGMA checksum` @@ -34,6 +35,7 @@ PRAGMA checksum('ecdb_map') - **ecdb_map** - Includes only the ec mapping tables but not the ec definition tables > `ec_PropertyPath`, `ec_ClassMap`, `ec_Table`, `ec_Column`, `ec_Index`, `ec_IndexColumn`, `ec_PropertyMap` - **sqlite_schema** - Includes information in the `sqlite_master` table +- **schema_token** - A cheap schema-**identity** hash (every schema's name and version only, one tiny row per schema), intended as a cache-invalidation key for [`SchemaView`](../metadata/SchemaView.md). ## `PRAGMA ecdb_ver` @@ -226,7 +228,7 @@ The result is a single row with the following columns: | format | string | Format identifier (currently `binary`) | | formatVersion | integer | The format version of the returned blob | | data | binary | The schema metadata blob | -| schemaToken | string | SHA3-256 hash of the `ecdb_schema` tables, usable as a cache-invalidation key | +| schemaToken | string | Cheap schema-identity hash (see [`checksum(schema_token)`](#using-schema_token-for-cache-invalidation)), usable as a cache-invalidation key | The pragma is read-only. Attempting to set a value returns an error. @@ -239,4 +241,45 @@ PRAGMA schema_view(99) The pragma works against any ECDb profile from `4.0.0.1` onward; older files do not need to be upgraded first. On profile `4.0.0.1` only, `KindOfQuantity` persistence and presentation strings are returned in legacy FUS format rather than EC3.2; all other data is unaffected. See [SchemaViewBinaryFormat - ECDb Profile Compatibility](../metadata/SchemaViewBinaryFormat.md#ecdb-profile-compatibility) for details. +## `PRAGMA schema_view_fragment` + +Returns the same binary format as [`schema_view`](#pragma-schema_view), but for a chosen **subset** of the connection's schemas instead of all of them. This backs incremental loading of a `SchemaView`: a consumer can hydrate only the schemas it needs - for example `BisCore` and its references - rather than every schema in the iModel. A fragment is a content subset of the *identical* format, not a different format, so the blob it returns is parsed exactly like a `schema_view` blob. See [SchemaViewBinaryFormat - Fragments](../metadata/SchemaViewBinaryFormat.md#fragments-partial-blobs). + +The single string argument is a comma-separated list of schema names, optionally prefixed with a `v;` format-version token. Names are matched case-insensitively: + +```sql +-- Latest format version +PRAGMA schema_view_fragment('BisCore,Generic') +``` + +```sql +-- Explicitly request binary format version 1 +PRAGMA schema_view_fragment('v1;BisCore,Generic') +``` + +The pragma does not expand references itself - the caller controls exactly which schemas the blob contains, computed from the schema reference graph (`meta.ECSchemaDef` + `meta.SchemaHasSchemaReferences`). What set to pass depends on how the blob is consumed: + +- **Parsed standalone** (used on its own to build a `SchemaView`): pass a **dependency-closed** set - every schema referenced by a requested schema is also in the list. A referenced schema left out leaves its cross-references unresolved; they parse as "not present", the same as an [excluded schema](../metadata/SchemaView.md). +- **Merged into an existing view** (incremental loading): references already merged from an earlier fragment may be omitted. Cross-references resolve against the accumulated view, so only the still-missing schemas need to be in the list. The incremental loader relies on this to avoid re-fetching schemas it already has. + +The result row has the same columns as [`schema_view`](#pragma-schema_view): `format`, `formatVersion`, `data`, `schemaToken`. + +The pragma is read-only. It fails (returning no blob, not a partial one) on an empty list, a malformed or non-existent schema name, or a malformed or unsupported `v;` prefix. Duplicate names are de-duplicated, not rejected. + +### Why the format version is embedded in the argument string + +The fragment pragma needs two independent inputs - the blob format version and the set of schema names - but the ECSQL pragma infrastructure today only supports exactly **one** scalar argument (`pragma_value` is a single token). Putting the version inside that one string, as an optional self-tagged `v;` prefix, is deliberate. The alternatives are worse or require refactoring the pragma infrastructure. + +The embedded `v;` prefix is self-describing (`v1` reads as "version 1"), cannot collide with the name list (schema names are ECNames, which never contain `;` or `,`), and is read **first** so a future format that changes the list encoding can dispatch on the version before parsing the rest - the role a leading version byte plays in any binary wire format. It also keeps this pragma consistent with `schema_view`, where the format version is likewise the single optional leading argument; the common case `schema_view_fragment('BisCore,Generic')` stays clean and means "latest version." The choice is reversible: if pragmas ever gain real multi-argument support, the prefix can be promoted to a proper second argument while the string form is accepted during a deprecation window. + +## Using `schema_token` for cache invalidation + +`PRAGMA checksum(schema_token)` returns a cheap hash that identifies the current set of schemas - their **names and versions only**. It is intended as a cache-invalidation key for a [`SchemaView`](../metadata/SchemaView.md): hold onto the `schemaToken` column returned by [`schema_view`](#pragma-schema_view) / [`schema_view_fragment`](#pragma-schema_view_fragment), and later compare it against `PRAGMA checksum(schema_token)` to decide whether the cached view is stale. + +```sql +PRAGMA checksum(schema_token) +``` + +Like the other `checksum` keys, the result is a single row whose `sha3_256` column holds the SHA3-256 hash - here, of every schema's name and version digits. + [ECSql Syntax](./index.md) diff --git a/docs/learning/metadata/SchemaView.md b/docs/learning/metadata/SchemaView.md index 5c6b68156f75..1f41b6eb6954 100644 --- a/docs/learning/metadata/SchemaView.md +++ b/docs/learning/metadata/SchemaView.md @@ -1,60 +1,43 @@ # SchemaView -`SchemaView` is a high-performance, read-only schema metadata cache available in both backend and frontend. It loads a curated subset of an iModel's schemas in a single call and provides synchronous access to schemas, classes, properties, enumerations, kinds of quantity, and relationship constraints. +The shape of data in iModels is expressed using [ECSchemas](../../bis/ec/ec-schema.md). +Sometimes, these schemas can grow quite large and complex. It is possible to have a hundred schemas, thousands of classes and hundreds of thousands of properties flat, which expands to millions of properties when you include inherited properties. When that happens, performance and memory consumption suffer. -It lives in `@itwin/ecschema-metadata` and should be the first choice for accessing schema metadata at runtime - for example in presentation rules, property grids, or data-driven UI. +`SchemaView` is the first library primarily aimed at performance and memory consumption. It uses a binary blob to fetch exactly the data it needs from the iModel in as few (async) calls as possible, including string and property deduplication. It is read-only and designed for synchronous access to schema metadata that is held in memory for the lifetime of a connection. + +It lives in `@itwin/ecschema-metadata` and should be the first choice for accessing schema metadata at runtime - for example in presentation layers, property grids, or data-driven UI. For the binary transport format specification, see [SchemaViewBinaryFormat.md](./SchemaViewBinaryFormat.md). -`SchemaView` is a **lossy runtime** projection - it omits units, formats, and custom attribute instances. For the full, authoritative EC schema model and its interchange formats, see [ECSchema XML](../../bis/ec/ec-schema-xml.md) and [ECSchema JSON](../../bis/ec/ec-schema-json.md). +`SchemaView` excludes some information like units, formats, and custom attribute instances. For the full, authoritative EC schema model and its interchange formats, see [ECSchema XML](../../bis/ec/ec-schema-xml.md) and [ECSchema JSON](../../bis/ec/ec-schema-json.md). For formats and units, a separate dedicated API is planned for the near future. For custom attributes, a sidecar will be provided alongside SchemaView which allows getting them on demand directly from the iModel. ## When to use SchemaView -Use `SchemaView` when you need fast, synchronous, repeated lookups at runtime: - -- Property grids and data-driven UI -- IS-A checks and class hierarchy navigation -- Presentation rules and adapter layers -- Iterating properties (including inherited) of a class +Use `SchemaView` when you need fast, synchronous, repeated lookups at runtime without chatty calls into the iModel. Reach for the full-fidelity [SchemaContext]($ecschema-metadata) instead when you are: authoring, validating, serializing to XML/JSON, or accessing data that `SchemaView` deliberately omits (see [What is included](#what-is-included)). `SchemaContext` is the more expensive option - a full object graph with cross-references - use it when its completeness is what you actually need. -| | SchemaView | SchemaContext | -| --------------------- | ----------------------------------------------------------------------- | -------------------------------------------------------- | -| **Loading** | Single binary blob, one RPC call | One async RPC per schema (84 schemas = 84 round-trips) | -| **Memory** | Flat arrays, string dedup, property dedup; 90-95% less memory | Full object graph with cross-references | -| **Parse time** | Very fast (binary decode into typed arrays) | Slow (JSON parse + object construction per schema) | -| **Access** | Synchronous after one async hydration | Async throughout | -| **Mutability** | Read-only snapshot | Mutable; supports editing | -| **Scope** | Curated subset for runtime consumers | Full EC spec | -| **Custom attributes** | Not modeled (selected concepts promoted to first-class - see below) | All custom attribute instances available | - ## What is included `SchemaView` covers what runtime consumers ask for most: schemas, classes (entity, struct, mixin, relationship, custom attribute, view), properties (including inherited, in declaration order), enumerations, kinds of quantity, property categories, and relationship constraints. Class-type checks and downward navigation via `derivedClasses` are also exposed. A small number of widely-used custom attributes are promoted to first-class concepts on the view objects: -- **Views** - entity classes with the `QueryView` custom attribute are surfaced with `ClassType.View`, iterable via `schema.getClasses(ClassType.View)`. +- **Views** - ECViews are surfaced as their own `ClassType.View` (see [Views](#views)). - **Mixin** - the mixin custom attribute is reflected in `classType` and is included in `applies-to`/`is-a` walks. - **Hidden** - hidden flags on schemas, classes, and properties are exposed directly (e.g. `schema.isHidden`). ## What is excluded -The exclusion list is deliberate - it trades a small amount of breadth for a large reduction in transport size and load time. The omitted data is not "never needed", just needed rarely enough that pulling it on demand via `SchemaContext` or [ECDbMeta](../ECDbMeta.ecschema.md) ECSQL queries is a better trade-off than carrying it in every runtime cache. +The exclusion list is a curated set of things we judged to be outside the sweet spot of what runtime consumers usually need. It is not primarily a size optimization - on a typical iModel the excluded data is a small fraction of the schema - it is about keeping the view focused, and keeping out data that has historically caused trouble. Custom attribute instances in particular have been a recurring source of performance problems in the heavier metadata layers: some iModels carry exceptionally large or numerous custom attributes that most apps never look at. Anything omitted is still reachable on demand via `SchemaContext` or [ECDbMeta](../ECDbMeta.ecschema.md) ECSQL queries. At a high level, `SchemaView` omits: -Currently excluded: - -- **Custom attribute instances** on schemas, classes, properties, and relationship constraints. Promote what you need to a first-class concept if it becomes widespread (see above). -- **All "standard" schemas** as defined by ECObjects' `ECSchema::IsStandardSchema`. This covers: - - The EC3 standards: `CoreCustomAttributes`, `Units`, `Formats`, `ECDbMap`, `SchemaLocalizationCustomAttributes`, `EditorCustomAttributes`. `KindOfQuantity` carries only persistence-unit and presentation-format strings; consumers resolve names against the dedicated units/formats APIs (today: `SchemaContext` or ECSQL - see [Resolving format and unit names](#resolving-format-and-unit-names)). - - Legacy EC2-era schemas: `Bentley_Standard_CustomAttributes`, `Bentley_Standard_Classes`, `Bentley_ECSchemaMap`, `Bentley_Common_Classes`, `Dimension_Schema`, `iip_mdb_customAttributes`, `KindOfQuantity_Schema`, `rdl_customAttributes`, `SIUnitSystemDefaults`, `Unit_Attributes`, `Units_Schema`, `USCustomaryUnitSystemDefaults`. These predate EC3.2 and are not referenced structurally by modern domain schemas. -- **ECDb-internal schemas** beyond the standard list - `ECDbSystem`, `ECDbFileInfo`, `ECDbSchemaPolicies`. These describe storage-layer mapping and are not relevant to runtime consumers. Note that `ECDbMeta` is *not* excluded - it remains queryable via ECSQL. -- **Pure custom-attribute schemas** beyond the standard list - `BisCustomAttributes`, `ECv3ConversionAttributes`, `SchemaUpgradeCustomAttributes`. These contain only `CustomAttribute` and `Struct` definitions used for decoration; since CA instances are not transported, the definitions add little value. +- **Custom attribute instances** - the values, not the classes. +- **"Standard" schemas** - units, formats, core custom attributes, and the EC2-era legacy schemas (as defined by ECObjects' `ECSchema::IsStandardSchema`). `KindOfQuantity` still carries its persistence-unit and presentation-format strings; you resolve those names yourself (see [Resolving format and unit names](#resolving-format-and-unit-names)). +- **ECDb-internal and pure custom-attribute schemas** - storage-mapping schemas and decoration-only schemas whose only value is the CA instances that aren't transported anyway. `ECDbMeta` is *not* excluded - it stays queryable via ECSQL. -The authoritative logic lives in `IsExcludedSchema()` in [SchemaViewWriter](https://github.com/iTwin/imodel-native/blob/main/iModelCore/ECDb/ECDb/SchemaViewWriter.cpp), which delegates the standard-schema check to `ECSchema::IsStandardSchema`. +For the exact schema names in each category, see [Full list of excluded schemas](#full-list-of-excluded-schemas). The authoritative logic lives in `IsExcludedSchema()` in [SchemaViewWriter](https://github.com/iTwin/imodel-native/blob/main/iModelCore/ECDb/ECDb/SchemaViewWriter.cpp), which delegates the standard-schema check to `ECSchema::IsStandardSchema`. -When you do need data from an excluded schema, [SchemaContext]($ecschema-metadata) and ECDbMeta queries remain available. Examples of resolving units and formats are in [Resolving format and unit names](#resolving-format-and-unit-names). +The list is subject to debate - if a compelling use case emerges for any excluded schema, we will remove it. When you do need data from an excluded schema, [SchemaContext]($ecschema-metadata) and ECDbMeta queries remain available. ## Obtaining the schema view @@ -66,6 +49,20 @@ The schema view is obtained from [IModelDb]($backend) (backend) or [IModelConnec The schema view is cached for the lifetime of the connection. Schema changes (via `importSchemas` or pulling changesets with schema changes) automatically invalidate the cache. +### Loading only a subset of schemas + +By default `getSchemaView()` loads every (non-excluded) schema in the iModel. On a large iModel with many schemas, a consumer that only needs a few - for example a Models tree that starts from `BisCore` - can ask for just those schemas plus their references: + +```ts +[[include:SchemaView.obtain-subset]] +``` + +We use a single accumulating `SchemaView`, so if multiple callers request different subsets, the union of all requested schemas is loaded. If one party loads the whole set, it will be cached for everybody despite what filters they requested. View the option like a "I care about these schemas" rather than "only return these". + +The trade-off is: you pay only for the schemas you load, but downward schema navigation (`derivedClasses`) will only reflect what is loaded. Reach for the full view when you need a complete picture, but try to limit the scope when possible. + +Additional instructions for backend/frontend developers: We strongly advise against loading the "full" SchemaView automatically on every connection. Try and keep this "on-demand" with limited scope, or else every consumer always pays the cost. That said, on backend, even the worst case scenarios should load in less than a second, asynchronously. + ## Navigating schemas and classes All lookups are synchronous and case-insensitive. @@ -116,7 +113,7 @@ Properties can reference a kind of quantity (KoQ) or a property category. KoQs c ### Resolving format and unit names -Presentation format names use schema aliases (e.g. `"f"` for `Formats`, `"u"` for `Units`). The Units and Formats schemas are excluded from `SchemaView` because they will be accessed through a separate dedicated API in the future. +Presentation format names use schema aliases (e.g. `"f"` for `Formats`, `"u"` for `Units`). The Units and Formats schemas themselves are excluded from `SchemaView` (see [What is excluded](#what-is-excluded)), so you resolve these names yourself. > **Pre-EC3.2 iModels:** on the rare iModel still on ECDb profile `4.0.0.1` (predates the 2018 EC3.2 Units/Formats migration), `KindOfQuantity.persistenceUnit` and `presentationFormats` are returned in legacy FUS format and will not parse with the alias-qualified resolution patterns below. The fix is to upgrade the iModel's ECDb profile. See [SchemaViewBinaryFormat - ECDb Profile Compatibility](./SchemaViewBinaryFormat.md#ecdb-profile-compatibility). @@ -178,7 +175,7 @@ for await (const row of iModel.createQueryReader( ## Views -ECViews (entity classes with a `QueryView` custom attribute) are included in the runtime blob. They show up as classes with `ClassType.View` - use `schema.getClasses(ClassType.View)` to iterate just views, or `findClass(...)` + `isView()` to look one up by qualified name. Views expose their own properties but do not participate in class inheritance. +ECViews (entity classes with a `QueryView` custom attribute) are included in the SchemaView as their own type distinct from entity classes. They show up as classes with `ClassType.View` - use `schema.getClasses(ClassType.View)` to iterate just views, or `findClass(...)` + `isView()` to look one up by qualified name. Views expose their own properties but do not participate in class inheritance. ```ts [[include:SchemaView.views]] @@ -210,7 +207,7 @@ You can iterate every schema, class, and property in the schema view efficiently ## Sync/async contract -All schema, class, and property access is **synchronous** - the data is fully loaded from the binary blob on first hydration. This is a key difference from ecschema-metadata, where loading schemas and resolving cross-references requires async calls. +All schema, class, and property access is **synchronous**. `getSchemaView()` is the only asynchronous/IO step - it loads the binary blob once - and every read after that is synchronous. This is a key difference from ecschema-metadata, where loading schemas and resolving cross-references requires async calls and results in unpredictable loading behavior. ## View objects and allocation @@ -241,6 +238,20 @@ At the time of writing, some concepts are not exposed through ECDbMeta, and some +## Full list of excluded schemas + +
+Exact schema names excluded from SchemaView + +- **Custom attribute instances** (all). +- **All "standard" schemas** as defined by ECObjects' `ECSchema::IsStandardSchema`: + - EC3 standards: `CoreCustomAttributes`, `Units`, `Formats`, `ECDbMap`, `SchemaLocalizationCustomAttributes`, `EditorCustomAttributes`. + - Legacy EC2-era schemas: `Bentley_Standard_CustomAttributes`, `Bentley_Standard_Classes`, `Bentley_ECSchemaMap`, `Bentley_Common_Classes`, `Dimension_Schema`, `iip_mdb_customAttributes`, `KindOfQuantity_Schema`, `rdl_customAttributes`, `SIUnitSystemDefaults`, `Unit_Attributes`, `Units_Schema`, `USCustomaryUnitSystemDefaults`. These predate EC3.2 and are not referenced structurally by modern domain schemas. +- **ECDb-internal schemas** beyond the standard list - `ECDbSystem`, `ECDbFileInfo`, `ECDbSchemaPolicies`. These describe storage-layer mapping and are not relevant to runtime consumers. +- **Pure custom-attribute schemas** beyond the standard list - `BisCustomAttributes`, `ECv3ConversionAttributes`, `SchemaUpgradeCustomAttributes`. These contain only `CustomAttribute` and `Struct` definitions used for decoration; since CA instances are not transported, the definitions add little value. + +
+ ## Schema Localization To use schema localization for SchemaView items, see [SchemaLocalization.md](./SchemaLocalization.md). diff --git a/docs/learning/metadata/SchemaViewBinaryFormat.md b/docs/learning/metadata/SchemaViewBinaryFormat.md index ac41e1505f7b..778c39982332 100644 --- a/docs/learning/metadata/SchemaViewBinaryFormat.md +++ b/docs/learning/metadata/SchemaViewBinaryFormat.md @@ -12,9 +12,9 @@ Foreign-key references to `ec_` table rows use uint32 row IDs. A value of 0 mean ## Version History -| Version | Description | -| ------- | ------------------------------------------------------------------------------------------------ | -| 1 | Initial format. Flat count-prefixed tables with string interning and property-def deduplication. | +| Version | Description | +| ------- | ------------------------------------------------------------------------------------------------- | +| 1 | Initial format. Flat count-prefixed tables with string interning and property-def deduplication. The same format also carries [fragments](#fragments-partial-blobs) (a subset of schemas) - a fragment is a content variation, not a format change. | --- @@ -73,7 +73,7 @@ Each PropertyDef record: | uint8 | isHidden | 1 if HiddenProperty CA is present (with Show != true), 0 otherwise | | uint32 | descriptionSid | Description (SRef) | -Record size: **38 bytes** fixed. +Record size: **46 bytes** fixed. ### SchemaTable (tag 0x10) @@ -109,16 +109,16 @@ Each enumeration record: | Type | Field | Description | | ------ | -------------- | --------------------------------------- | -| uint32 | schemaEcId | ec_Schema.Id of the owning schema | +| uint32 | schemaECId | ec_Schema.Id of the owning schema | | uint32 | nameSid | Enumeration name (SRef) | -| uint8 | primitiveType | Underlying primitive type | +| uint16 | primitiveType | Underlying primitive type | | uint8 | isStrict | 1 if strict, 0 otherwise | | uint32 | labelSid | Display label (SRef) | | uint32 | descriptionSid | Description (SRef) | | uint32 | enumValuesSid | Enumerator values as JSON string (SRef) | | uint32 | ecInstanceId | ec_Enumeration.Id | -Record size: **24 bytes** fixed. +Record size: **26 bytes** fixed. The `enumValuesSid` field references a JSON array string stored in the string table. Each element has the shape: `{"Name":"...","IntValue":...,"StringValue":"...","DisplayLabel":"...","Description":"..."}`. The reader is responsible for parsing this JSON to extract individual enumerators. @@ -133,7 +133,7 @@ Each KoQ record: | Type | Field | Description | | ------- | ---------------------- | ---------------------------------- | -| uint32 | schemaEcId | ec_Schema.Id of the owning schema | +| uint32 | schemaECId | ec_Schema.Id of the owning schema | | uint32 | nameSid | KoQ name (SRef) | | uint32 | labelSid | Display label (SRef) | | uint32 | descriptionSid | Description (SRef) | @@ -155,7 +155,7 @@ Each property category record: | Type | Field | Description | | ------ | -------------- | --------------------------------- | -| uint32 | schemaEcId | ec_Schema.Id of the owning schema | +| uint32 | schemaECId | ec_Schema.Id of the owning schema | | uint32 | nameSid | Category name (SRef) | | uint32 | labelSid | Display label (SRef) | | uint32 | descriptionSid | Description (SRef) | @@ -177,7 +177,7 @@ Each class record: | Type | Field | Description | | ------ | ----------------- | ---------------------------------------------------------------------- | -| uint32 | schemaEcId | ec_Schema.Id of the owning schema | +| uint32 | schemaECId | ec_Schema.Id of the owning schema | | uint32 | nameSid | Class name (SRef) | | uint8 | classType | 0=Entity, 1=Relationship, 2=Struct, 3=CustomAttribute, 4=Mixin, 5=View | | uint8 | modifier | 0=None, 1=Abstract, 2=Sealed | @@ -284,6 +284,34 @@ Index 0 is always the empty string. --- +## Fragments (partial blobs) + +A **fragment** is a blob carrying only a requested subset of the iModel's schemas rather than all of them, so a consumer can load just what it needs (e.g. `BisCore` and its references) instead of hydrating every schema up front. This is what makes incremental loading of a `SchemaView` possible. + +A fragment is **not a different format** - it uses the exact same layout described above, byte for byte. "Whole-schema" versus "fragment" is a difference in *which schemas the blob contains*, not in how it is encoded, so there is no format-version distinction and the reader needs no flag in the blob to parse it. What differs is **reference resolution** (see below), and the consumer already knows which kind it is holding because it chose which pragma to call. + +No encoding change is needed because the format already addresses every cross-schema reference globally: row-id references carry `ec_` row ids (unique within the iModel) and class references carry schema-name/class-name string pairs. Neither is a blob-local index, so both already point at a row in any blob, whole or partial. The one blob-local index, `defIdx`, never crosses a fragment boundary (see below). + +### Producing a fragment + +- `PRAGMA schema_view(N)` emits a whole-schema blob: every non-excluded schema in the iModel. +- `PRAGMA schema_view_fragment('name,name,...')` emits a fragment containing the named schemas. For a standalone fragment, callers typically pass the dependency-closed set computed from the schema reference graph (`meta.ECSchemaDef` + `meta.SchemaHasSchemaReferences`). For incremental SchemaView loading, it is valid to omit schemas that are already present in the accumulating view; references to those schemas resolve against previously merged data. + +Both pragmas produce identical byte layout; a fragment that happens to contain every schema is byte-for-byte the same as the whole-schema blob. + +### Fragment containment (no leakage) + +A fragment contains only the rows **owned** by the requested schemas - their classes, properties, enumerations, KoQs, categories, and constraints. It contains **zero** rows owned by a schema that is referenced but not requested. References that cross the fragment boundary use the encodings the format already defines for any cross-schema reference: + +- **Row-id references** (`schemaECId`, `structClassRowId`, `enumRowId`, `koqRowId`, `catRowId`, `navRelClassRowId`) carry the target's `ec_` row id. Row ids are unique within the iModel, so they address a row in any blob unambiguously. +- **Name references** (base classes, constraint classes, abstract constraint classes) carry the target's schema-name and class-name string refs. + +Neither encoding marks a target as in-fragment versus cross-fragment, and it does not need to - resolution is the consumer's responsibility (see below). + +`defIdx` (the PropertyRef index into the PropertyDefTable) is the one **blob-local** index. A class and its deduplicated property definitions always ship in the same blob, so `defIdx` is always resolvable within the blob it came from and never crosses a boundary. A consumer merging fragments must **re-base / re-deduplicate** these defs into its accumulated PropertyDefTable as it applies each fragment: the `defIdx` value is relative to its own blob's table, so the merge step maps it to the corresponding index in the combined table. This is the one place fragment merging needs care. + +--- + ## Excluded Schemas The writer silently skips schemas that provide no value at runtime. Excluded schemas and all their items (classes, properties, enumerations, etc.) are omitted from the blob. The exclusion is the union of two sets: @@ -313,7 +341,13 @@ Note: **ECDbMeta is NOT excluded** - consumers use it for metadata ECSQL queries ## Cross-Reference Resolution -Row IDs (`schemaEcId`, `enumRowId`, `structClassRowId`, etc.) and name-based references (base classes, constraint classes) must be resolved by the reader after parsing. The writer does not scrub references to excluded schemas. Readers should handle unresolved references gracefully - see the TS reader for the resolution strategy (drop properties with broken structural refs, skip dangling mixin/constraint entries). +Row IDs (`schemaECId`, `enumRowId`, `structClassRowId`, etc.) and name-based references (base classes, constraint classes) must be resolved by the reader after parsing. The writer does not scrub references to excluded schemas. The resolution strategy depends on whether the blob is a whole-schema blob or a [fragment](#fragments-partial-blobs) - and the consumer always knows which, because it chose the pragma. + +**Whole-schema blob** is self-contained: every non-excluded schema is present, so references resolve against the blob's own row-id and name maps. A reference whose target is absent points at an excluded schema and is dropped gracefully - drop properties with broken structural refs, skip dangling mixin/constraint entries. See the TS reader for the resolution strategy. + +**Fragment** is not self-contained: a reference may target a schema carried by another fragment. The consumer applies fragments into one accumulated view and resolves references against the **merged** row-id and name maps spanning every fragment applied so far, not against the single blob. Fragments are applied in dependency order - a fragment is never applied before the schemas it references - so a cross-fragment target is already present when its reference is resolved. A target still absent after the requested closure has been applied points at an *excluded* schema and is dropped, exactly as for a whole-schema blob. Cross-fragment references and excluded-schema references thus share a single resolution path: look the target up in the merged maps, drop on a miss. + +Because the byte layout is identical, the reader cannot - and need not - tell a fragment from a whole-schema blob by inspecting it. The caller selects the resolution mode (resolve standalone vs. resolve against accumulated maps) from which pragma it issued. ## ECDb Profile Compatibility @@ -326,5 +360,5 @@ In practice any iModel that has been opened by iTwin.js since mid-2018 has alrea ## Implementation References - **Writer (C++)**: `iModelCore/ECDb/ECDb/SchemaViewWriter.cpp` / `.h` -- **Reader (TS)**: `core/ecschema-metadata/src/SchemaViewBinaryReader.ts` -- **Pragma handler**: `PRAGMA schema_view(N)` where N is the requested format version +- **Reader (TS)**: `core/ecschema-metadata/src/SchemaView/SchemaViewBinaryReader.ts` +- **Pragma handlers**: `PRAGMA schema_view(N)` (whole-schema blob) and `PRAGMA schema_view_fragment('name,name,...')` (fragment - same format, subset of schemas) diff --git a/example-code/snippets/src/backend/SchemaViewExamples.test.ts b/example-code/snippets/src/backend/SchemaViewExamples.test.ts index d4426f44419b..fda5cbf8a978 100644 --- a/example-code/snippets/src/backend/SchemaViewExamples.test.ts +++ b/example-code/snippets/src/backend/SchemaViewExamples.test.ts @@ -34,6 +34,22 @@ describe("SchemaView Examples", () => { // __PUBLISH_EXTRACT_END__ }); + it("obtaining a subset schema view", async () => { + // __PUBLISH_EXTRACT_START__ SchemaView.obtain-subset + // Load only a subset of schemas (plus their reference closure) instead of every schema. + // This is an accumulating view instance that initially contains only the schemas you ask for. + const bisOnly = await iModel.getSchemaView({ schemas: ["BisCore"] }); + assert.isDefined(bisOnly.getSchema("BisCore")); + assert.isDefined(bisOnly.findClass("BisCore:Element")); + + // A later request for more schemas merges them into the SAME accumulating instance - schemas + // requested earlier remain available. Schemas that were never requested are simply absent. + const withGeneric = await iModel.getSchemaView({ schemas: ["Generic"] }); + assert.strictEqual(withGeneric, bisOnly); + assert.isDefined(withGeneric.getSchema("BisCore")); // still here + // __PUBLISH_EXTRACT_END__ + }); + it("navigating schemas and classes", async () => { const view = await iModel.getSchemaView();