diff --git a/change/@itwin-imodel-transformer-e69e7651-40c7-4476-9e9e-6026f430c36c.json b/change/@itwin-imodel-transformer-e69e7651-40c7-4476-9e9e-6026f430c36c.json new file mode 100644 index 00000000..4fde42de --- /dev/null +++ b/change/@itwin-imodel-transformer-e69e7651-40c7-4476-9e9e-6026f430c36c.json @@ -0,0 +1,7 @@ +{ + "type": "prerelease", + "comment": "Add new APIs to support providing custom changes ( not found in a changeset ) to the transformer", + "packageName": "@itwin/imodel-transformer", + "email": "22119573+nick4598@users.noreply.github.com, JulijaRamoskiene@users.noreply.github.com", + "dependentChangeType": "patch" +} diff --git a/common/api/imodel-transformer.api.md b/common/api/imodel-transformer.api.md index 5bf12471..e2c5a14c 100644 --- a/common/api/imodel-transformer.api.md +++ b/common/api/imodel-transformer.api.md @@ -22,6 +22,7 @@ import { EntityReference } from '@itwin/core-common'; import { ExternalSourceAspect } from '@itwin/core-backend'; import { ExternalSourceAspectProps } from '@itwin/core-common'; import { FontProps } from '@itwin/core-common'; +import { Id64Arg } from '@itwin/core-bentley'; import { Id64Array } from '@itwin/core-bentley'; import { Id64Set } from '@itwin/core-bentley'; import { Id64String } from '@itwin/core-bentley'; @@ -36,11 +37,18 @@ import { Relationship } from '@itwin/core-backend'; import { RelationshipProps } from '@itwin/core-backend'; import { Schema } from '@itwin/ecschema-metadata'; import { SchemaKey } from '@itwin/ecschema-metadata'; +import { SqliteChangeOp } from '@itwin/core-backend'; // @public export class ChangedInstanceIds { constructor(db: IModelDb); addChange(change: ChangedECInstance): Promise; + // @beta + addCustomAspectChange(changeType: SqliteChangeOp, ids: Id64Arg): void; + // @beta + addCustomElementChange(changeType: SqliteChangeOp, ids: Id64Arg): Promise; + // @beta + addCustomModelChange(changeType: SqliteChangeOp, ids: Id64Arg): Promise; // (undocumented) aspect: ChangedInstanceOps; // (undocumented) @@ -49,6 +57,7 @@ export class ChangedInstanceIds { element: ChangedInstanceOps; // (undocumented) font: ChangedInstanceOps; + get hasChanges(): boolean; static initialize(opts: ChangedInstanceIdsInitOptions): Promise; // (undocumented) model: ChangedInstanceOps; @@ -68,6 +77,7 @@ export class ChangedInstanceOps { deleteIds: Set; // (undocumented) insertIds: Set; + get isEmpty(): boolean; // (undocumented) updateIds: Set; } @@ -232,6 +242,7 @@ export interface IModelImportOptions { // @beta export class IModelTransformer extends IModelExportHandler { constructor(source: IModelDb | IModelExporter, target: IModelDb | IModelImporter, options?: IModelTransformOptions); + protected addCustomChanges(_sourceDbChanges: ChangedInstanceIds): Promise; combineElements(sourceElementIds: Id64Array, targetElementId: Id64String): void; // (undocumented) protected completePartiallyCommittedAspects(): void; diff --git a/packages/test-app/src/IModelHubUtils.ts b/packages/test-app/src/IModelHubUtils.ts index f7706722..ab6da6db 100644 --- a/packages/test-app/src/IModelHubUtils.ts +++ b/packages/test-app/src/IModelHubUtils.ts @@ -97,12 +97,14 @@ export namespace IModelHubUtils { ): Promise { return ( // eslint-disable-next-line @itwin/no-internal - await IModelHost.hubAccess.queryChangeset({ - accessToken, - iModelId, - changeset: { index: changesetIndex }, - }) - ).id; + ( + await IModelHost.hubAccess.queryChangeset({ + accessToken, + iModelId, + changeset: { index: changesetIndex }, + }) + ).id + ); } /** Temporarily needed to convert from the legacy ChangesetId to the now preferred ChangeSetIndex. @@ -201,8 +203,8 @@ export namespace IModelHubUtils { return BriefcaseDb.open({ fileName: briefcaseProps.fileName, readonly: briefcaseArg.briefcaseId - // eslint-disable-next-line @typescript-eslint/no-unsafe-enum-comparison - ? briefcaseArg.briefcaseId === BriefcaseIdValue.Unassigned + ? // eslint-disable-next-line @typescript-eslint/no-unsafe-enum-comparison + briefcaseArg.briefcaseId === BriefcaseIdValue.Unassigned : false, }); } diff --git a/packages/transformer/src/IModelExporter.ts b/packages/transformer/src/IModelExporter.ts index 5bda0d8f..fbd5ca84 100644 --- a/packages/transformer/src/IModelExporter.ts +++ b/packages/transformer/src/IModelExporter.ts @@ -33,6 +33,9 @@ import { import { assert, DbResult, + Id64, + Id64Arg, + Id64Set, Id64String, IModelStatus, Logger, @@ -44,6 +47,7 @@ import { FontProps, IModel, IModelError, + QueryBinder, } from "@itwin/core-common"; import { ECVersion, @@ -289,9 +293,10 @@ export class IModelExporter { private _progressCounter: number = 0; /** Optionally cached entity change information */ private _sourceDbChanges?: ChangedInstanceIds; + /** * Retrieve the cached entity change information. - * @note This will only be initialized after [IModelExporter.exportChanges] is invoked. + * @note This will only be initialized after [IModelExporter.exportChanges] is invoked or [IModelExporter.initialize] is called. */ public get sourceDbChanges(): ChangedInstanceIds | undefined { return this._sourceDbChanges; @@ -444,7 +449,6 @@ export class IModelExporter { const initOpts: ExporterInitOptions = { startChangeset: { id: startChangeset?.id }, }; - await this.initialize(initOpts); // _sourceDbChanges are initialized in this.initialize nodeAssert( @@ -1027,6 +1031,18 @@ export class ChangedInstanceOps { val.delete.forEach((id: Id64String) => this.deleteIds.add(id)); } } + + /** + * Checks if empty. + * @returns true if there no ids in the ChangedInstanceOps object. + */ + public get isEmpty(): boolean { + return ( + 0 === this.insertIds.size && + 0 === this.updateIds.size && + 0 === this.deleteIds.size + ); + } } /** @@ -1045,6 +1061,8 @@ export class ChangedInstanceIds { private _elementSubclassIds?: Set; private _aspectSubclassIds?: Set; private _relationshipSubclassIds?: Set; + private _relationshipSubclassIdsToSkip?: Set; + private _db: IModelDb; public constructor(db: IModelDb) { this._db = db; @@ -1056,6 +1074,7 @@ export class ChangedInstanceIds { this._elementSubclassIds = new Set(); this._aspectSubclassIds = new Set(); this._relationshipSubclassIds = new Set(); + this._relationshipSubclassIdsToSkip = new Set(); const addECClassIdsToSet = async ( setToModify: Set, @@ -1080,6 +1099,10 @@ export class ChangedInstanceIds { this._relationshipSubclassIds, "BisCore.ElementRefersToElements" ), + addECClassIdsToSet( + this._relationshipSubclassIdsToSkip, + "BisCore.ElementDrivesElement" + ), ]; await Promise.all(promises); } @@ -1090,7 +1113,8 @@ export class ChangedInstanceIds { this._modelSubclassIds && this._elementSubclassIds && this._aspectSubclassIds && - this._relationshipSubclassIds + this._relationshipSubclassIds && + this._relationshipSubclassIdsToSkip ); } @@ -1114,6 +1138,20 @@ export class ChangedInstanceIds { return this._elementSubclassIds?.has(ecClassId); } + /** Checks if there are any changes. + * @returns true if there are any changes in the ChangedInstanceIds object. + */ + public get hasChanges(): boolean { + return ( + !this.codeSpec.isEmpty || + !this.model.isEmpty || + !this.element.isEmpty || + !this.aspect.isEmpty || + !this.relationship.isEmpty || + !this.font.isEmpty + ); + } + /** * Adds the provided [[ChangedECInstance]] to the appropriate set of changes by class type (codeSpec, model, element, aspect, or relationship) maintained by this instance of ChangedInstanceIds. * If the same ECInstanceId is seen multiple times, the changedInstanceIds will be modified accordingly, i.e. if an id 'x' was updated but now we see 'x' was deleted, we will remove 'x' @@ -1132,6 +1170,7 @@ export class ChangedInstanceIds { throw new Error( `ChangeType was undefined for id: ${change.ECInstanceId}.` ); + if (this._relationshipSubclassIdsToSkip?.has(ecClassId)) return; if (this.isRelationship(ecClassId)) this.handleChange(this.relationship, changeType, change.ECInstanceId); @@ -1145,6 +1184,154 @@ export class ChangedInstanceIds { this.handleChange(this.element, changeType, change.ECInstanceId); } + /** + * This method should only be called inside [[IModelTransformer.addCustomChanges]]. + * It adds the provided change to the element changes maintained by this instance of ChangedInstanceIds. + * If the same ECInstanceId is seen multiple times, the changedInstanceIds will be modified accordingly, i.e. if an id 'x' was updated but now we see 'x' was deleted, we will remove 'x' + * from the set of updatedIds and add it to the set of deletedIds for the appropriate class type. + * @note Custom element 'Insert' and 'Update' will mark element's parent model hierarchy and their modeled elements as 'Updated' in [[ChangedInstanceIds.model]] and [[ChangedInstanceIds.element]]. Parent models have to be marked as 'Updated' to make sure that added change is not skipped by transformer. Transformer starts processing elements from RepositoryModel and then visits all child models. Modeled elements hierarchy is marked as updated to trigger their inserts in case a new model (or its parent) needs to be inserted. + * @note Custom element 'Insert' will also mark element aspects and all element relationships as inserted. + * @note It is the responsibility of the caller to ensure that the provided id is, in fact an element. + * @note In most cases, this method does not need to be called. Its only for consumers to mimic changes as if they were found in a changeset, which should only be useful in certain cases such as the changing of filter criteria for a preexisting master branch relationship. + * @note In data processing with filter criteria scenarios it is important to consistently filter out models and their modeled elements that were previously removed from target via [[addCustomModelChange]] or [[shouldExportElement]] apis. + * @beta + */ + public async addCustomElementChange( + changeType: SqliteChangeOp, + ids: Id64Arg + ): Promise { + if (Id64.sizeOf(ids) === 0) { + return; + } + + for (const id of Id64.iterable(ids)) { + this.handleChange(this.element, changeType, id); + } + + if (changeType === "Deleted") { + return; + } + + const idsSet = Id64.toIdSet(ids); + // Parent models have to be marked as 'Updated' to make sure that added change is not skipped by transformer. Transformer starts processing elements from RepositoryModel and then visits all child models. + // Transformer handles update as insert if element is not found in target, for this reason modeled elements will be also marked as updated to trigger their inserts in case a new model (or its parent) needs to be inserted. Otherwise error would be thrown about missing modeled element while inserting new model. + const parentModelIds = await this.markParentModelsAsUpdated(idsSet); + + // Aspects and relationships of inserted data needs to be marked as inserted otherwise those would not be exported + if (changeType === "Inserted") { + // Adding parents as well as we are not sure if those were inserted or updated + parentModelIds.forEach((parentId) => { + idsSet.add(parentId); + }); + + await this.markElementAspectsAsInserted(idsSet); + // Marking only ElementRefersToElements.classFullName as only those are exported in exportRelationships() + await this.markElementRelationshipsAsInserted( + ElementRefersToElements.classFullName, + idsSet + ); + } + } + + /** + * This method should only be called inside [IModelTransformer.addCustomChanges]. + * Adds the provided change to the model changes maintained by this instance of ChangedInstanceIds. + * If the same ECInstanceId is seen multiple times, the changedInstanceIds will be modified accordingly, i.e. if an id 'x' was updated but now we see 'x' was deleted, we will remove 'x' + * from the set of updatedIds and add it to the set of deletedIds for the appropriate class type. + * Will add same change to the model's modeledElement by calling [[ChangedInstanceIds.addCustomElementChange]] which will register more needed changes. This is to ensure the changes from the model and its modeledElement get exported together. + * @note It is the responsibility of the caller to ensure that the provided id is, in fact a model. + * @note In most cases, this method does not need to be called. Its only for consumers to mimic changes as if they were found in a changeset, which should only be useful in certain cases such as the changing of filter criteria for a preexisting master branch relationship. + * @note In data processing with filter criteria scenarios it is important to consistently filter out models and their modeled elements that were previously removed from target via [[addCustomModelChange]] or [[shouldExportElement]] apis. + * @beta + */ + public async addCustomModelChange( + changeType: SqliteChangeOp, + ids: Id64Arg + ): Promise { + // Also add the model's modeledElement to the element changes. The modeledElement and model go hand in hand and have the same id. + await this.addCustomElementChange(changeType, ids); + for (const id of Id64.iterable(ids)) { + this.handleChange(this.model, changeType, id); + } + } + + /** + * This method should only be called inside [IModelTransformer.addCustomChanges]. + * Adds the provided change to the aspect changes maintained by this instance of ChangedInstanceIds + * If the same ECInstanceId is seen multiple times, the changedInstanceIds will be modified accordingly, i.e. if an id 'x' was updated but now we see 'x' was deleted, we will remove 'x' + * from the set of updatedIds and add it to the set of deletedIds for the appropriate class type. + * @note It is the responsibility of the caller to ensure that the provided id is, in fact an aspect. + * @note In most cases, this method does not need to be called. Its only for consumers to mimic changes as if they were found in a changeset, which should only be useful in certain cases such as the changing of filter criteria for a preexisting master branch relationship. + * @beta + */ + public addCustomAspectChange(changeType: SqliteChangeOp, ids: Id64Arg): void { + for (const id of Id64.iterable(ids)) { + this.handleChange(this.aspect, changeType, id); + } + } + + /** + * There is an optimization in [IModelExporter.exportModelContents] which doesn't try to export elements within a model unless the model itself is marked as `Updated` or 'Inserted' in sourceDbChanges. This method is used in [[addCustomElementChange]] and [[addCustomModelChange]] to add the parent model hierarchy to the 'updatedIds' so that the custom element changes are exported. + * Transformer will insert 'Updated' model to target if it does not exist there already. To handle such case, modeled elements of parent models are also marked as updated. This is done, because model can not be inserted without it's modeled element. + */ + private async markParentModelsAsUpdated(elementIds: Id64Set) { + const params = new QueryBinder().bindIdSet("elementIds", elementIds); + + const ecQuery = ` + WITH RECURSIVE hierarchy (parentId) AS ( + SELECT Model.Id FROM bis.Element WHERE InVirtualSet(:elementIds, ECInstanceId) + UNION + SELECT ParentModel.id + FROM bis.Model e + INNER JOIN hierarchy h ON h.parentId = e.ECInstanceId + ) + SELECT parentId FROM hierarchy where parentId is not null + `; + const parentModelIds = new Set(); + for await (const row of this._db.createQueryReader(ecQuery, params)) { + // Transformer handles update as insert when element does not exist in target. + // Which means that in scenario where child and parent model are filtered out from target, + // and child element is inserted trough custom change, its parent model will be marked as updated. + // Transformer then will: + // 1. Handle parent update as insert (since it does not exist in target). + // 2. Will insert child element (otherwise this insert would be ignored due to missing parent). + this.handleChange(this.model, "Updated", row.parentId); + this.handleChange(this.element, "Updated", row.parentId); + parentModelIds.add(row.parentId); + } + return parentModelIds; + } + + private async markElementRelationshipsAsInserted( + relationshipClassName: string, + elementIds: Id64Set + ) { + const ecQuery = `SELECT ECInstanceId FROM ${relationshipClassName} + WHERE InVirtualSet(:elementIds, TargetECInstanceId) + OR InVirtualSet(:elementIds, SourceECInstanceId)`; + + const queryBinder = new QueryBinder().bindIdSet("elementIds", elementIds); + const queryReader = this._db.createQueryReader(ecQuery, queryBinder); + + for await (const row of queryReader) { + this.handleChange(this.relationship, "Inserted", row.ECInstanceId); + } + } + + private async markElementAspectsAsInserted(elementIds: Id64Set) { + for (const aspectClassName of [ + ElementUniqueAspect.classFullName, + ElementMultiAspect.classFullName, + ]) { + const ecQuery = `Select ECInstanceId from ${aspectClassName} where InVirtualSet(:elementIds, Element.Id)`; + const queryBinder = new QueryBinder().bindIdSet("elementIds", elementIds); + const queryReader = this._db.createQueryReader(ecQuery, queryBinder); + for await (const row of queryReader) { + this.addCustomAspectChange("Inserted", row.toArray()[0]); + } + } + } + private handleChange( changedInstanceOps: ChangedInstanceOps, changeType: SqliteChangeOp, @@ -1228,12 +1415,6 @@ export class ChangedInstanceIds { if (csFileProps === undefined) return undefined; const changedInstanceIds = new ChangedInstanceIds(opts.iModel); - const relationshipECClassIdsToSkip = new Set(); - for await (const row of opts.iModel.createQueryReader( - "SELECT ECInstanceId FROM ECDbMeta.ECClassDef where ECInstanceId IS (BisCore.ElementDrivesElement)" - )) { - relationshipECClassIdsToSkip.add(row.ECInstanceId); - } for (const csFile of csFileProps) { const csReader = SqliteChangesetReader.openFile({ @@ -1249,11 +1430,6 @@ export class ChangedInstanceIds { const changes: ChangedECInstance[] = [...ecChangeUnifier.instances]; for (const change of changes) { - if ( - change.ECClassId !== undefined && - relationshipECClassIdsToSkip.has(change.ECClassId) - ) - continue; await changedInstanceIds.addChange(change); } csReader.close(); diff --git a/packages/transformer/src/IModelTransformer.ts b/packages/transformer/src/IModelTransformer.ts index e4c57b63..01b9de37 100644 --- a/packages/transformer/src/IModelTransformer.ts +++ b/packages/transformer/src/IModelTransformer.ts @@ -93,6 +93,7 @@ import { SourceAndTarget, } from "@itwin/core-common"; import { + ChangedInstanceIds, ExportChangesOptions, ExporterInitOptions, ExportSchemaResult, @@ -2788,8 +2789,19 @@ export class IModelTransformer extends IModelExportHandler { this.context.remapElement(sourceElementId, targetElementId); } ); - if (this._csFileProps === undefined || this._csFileProps.length === 0) - return; + if (this.exporter.sourceDbChanges) + await this.addCustomChanges(this.exporter.sourceDbChanges); + + if (this._csFileProps === undefined || this._csFileProps.length === 0) { + if ( + this.exporter.sourceDbChanges === undefined || + !this.exporter.sourceDbChanges.hasChanges + ) + return; + // our sourcedbChanges aren't empty (probably due to someone adding custom changes), change our sourceChangeDataState to has-changes + if (this._sourceChangeDataState === "no-changes") + this._sourceChangeDataState = "has-changes"; + } const relationshipECClassIdsToSkip = new Set(); for await (const row of this.sourceDb.createQueryReader( @@ -2825,9 +2837,10 @@ export class IModelTransformer extends IModelExportHandler { alreadyImportedModelInserts.add(targetModelId); } ); + this._deletedSourceRelationshipData = new Map(); - for (const csFile of this._csFileProps) { + for (const csFile of this._csFileProps ?? []) { const csReader = SqliteChangesetReader.openFile({ fileName: csFile.pathname, db: this.sourceDb, @@ -2858,7 +2871,6 @@ export class IModelTransformer extends IModelExportHandler { elemIdToScopeEsa.set(change.Element.Id, change); } } - // Loop to process deletes. for (const change of changes) { const changeType: SqliteChangeOp | undefined = change.$meta?.op; @@ -2889,6 +2901,19 @@ export class IModelTransformer extends IModelExportHandler { } return; } + + /** + * This will be called when transformer is called with [[IModelTransformOptions.argsForProcessChanges]] to process changes. + * It will be executed after changes in changesets are populated into `sourceDbChanges` and before data processing begins. + * Remap table between the source and target iModels will be built at that time, meaning that functions like [[IModelTransformer.context.findTargetElementId]] will return meaningful results. + * This function should be used to modify the `sourceDbChanges`, if necessary, using `add custom change` methods in [[ChangedInstanceIds]], such as [[ChangedInstanceIds.addCustomElementChange]], [[ChangedInstanceIds.addCustomModelChange]] and other. + * @param sourceDbChanges the ChangedInstanceIds already populated by the exporter with the changes in source changesets, if any, passed to the transformer. + * @note Its expected that this function be overridden by a subclass of transformer if it needs to modify sourceDbChanges. + */ + protected async addCustomChanges( + _sourceDbChanges: ChangedInstanceIds + ): Promise {} + /** * Helper function for processChangesets. Remaps the id of element deleted found in the 'change' to an element in the targetDb. * @param change the change to process, must be of changeType "Deleted" @@ -2909,7 +2934,9 @@ export class IModelTransformer extends IModelExportHandler { // we need a connected iModel with changes to remap elements with deletions const notConnectedModel = this.sourceDb.iTwinId === undefined; const noChanges = - this.synchronizationVersion.index === this.sourceDb.changeset.index; + this.synchronizationVersion.index === this.sourceDb.changeset.index && + (this.exporter.sourceDbChanges === undefined || + !this.exporter.sourceDbChanges.hasChanges); if (notConnectedModel || noChanges) return; /** diff --git a/packages/transformer/src/test/IModelTransformerUtils.ts b/packages/transformer/src/test/IModelTransformerUtils.ts index a1aab993..472b18f5 100644 --- a/packages/transformer/src/test/IModelTransformerUtils.ts +++ b/packages/transformer/src/test/IModelTransformerUtils.ts @@ -139,6 +139,55 @@ export class IModelTransformerTestUtils extends TestUtils.IModelTestUtils { return iModelDb; } + /** Returns path to a schema which contains a multiAspect TestSchema2:MyMultiAspect. + * The schema is created in the output directory. + * The multi aspect has a prop 'MyProp1'. + * Users should import this schema in order to insert multi aspects. + */ + public static getPathToSchemaWithMultiAspect(): string { + const testSchema1Path = IModelTransformerTestUtils.prepareOutputFile( + "IModelTransformer", + "TestSchema2.ecschema.xml" + ); + IModelJsFs.writeFileSync( + testSchema1Path, + ` + + + + bis:ElementMultiAspect + + + ` + ); + return testSchema1Path; + } + + /** Returns path to a schema which contains a UniqueAspect TestSchema1:MyUniqueAspect. + * The schema is created in the output directory. + * the only two ElementUniqueAspect's in bis are ignored by the transformer, so we can add our own to test their export + * The unique aspect has a prop 'MyProp1'. + * Users should import this schema in order to insert unique aspects. + */ + public static getPathToSchemaWithUniqueAspect(): string { + const testSchema1Path = IModelTransformerTestUtils.prepareOutputFile( + "IModelTransformer", + "TestSchema1.ecschema.xml" + ); + IModelJsFs.writeFileSync( + testSchema1Path, + ` + + + + bis:ElementUniqueAspect + + + ` + ); + return testSchema1Path; + } + public static populateTeamIModel( teamDb: IModelDb, teamName: string, diff --git a/packages/transformer/src/test/TestUtils/IModelTestUtils.ts b/packages/transformer/src/test/TestUtils/IModelTestUtils.ts index 5fe17552..a0d0d13a 100644 --- a/packages/transformer/src/test/TestUtils/IModelTestUtils.ts +++ b/packages/transformer/src/test/TestUtils/IModelTestUtils.ts @@ -1208,6 +1208,21 @@ export class IModelTestUtils { ); } + public static queryModelIddByModeledElementCodeValue( + iModelDb: IModelDb, + codeValue: string + ): Id64String { + return iModelDb.withPreparedStatement( + `SELECT ECInstanceId FROM ${Model.classFullName} WHERE ModeledElement.Id in (Select ECInstanceId from Bis.Element where CodeValue=:codeValue)`, + (statement: ECSqlStatement): Id64String => { + statement.bindString("codeValue", codeValue); + return DbResult.BE_SQLITE_ROW === statement.step() + ? statement.getValue(0).getId() + : Id64.invalid; + } + ); + } + public static insertRepositoryLink( iModelDb: IModelDb, codeValue: string, diff --git a/packages/transformer/src/test/standalone/ChangedInstanceIds.test.ts b/packages/transformer/src/test/standalone/ChangedInstanceIds.test.ts new file mode 100644 index 00000000..2f63353e --- /dev/null +++ b/packages/transformer/src/test/standalone/ChangedInstanceIds.test.ts @@ -0,0 +1,478 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ +import path = require("path"); +import { KnownTestLocations } from "../TestUtils"; +import { + DocumentListModel, + Drawing, + ElementGroupsMembers, + ElementOwnsExternalSourceAspects, + ExternalSourceAspect, + IModelDb, + IModelJsFs, + SnapshotDb, + Subject, +} from "@itwin/core-backend"; +import { IModelTransformerTestUtils } from "../IModelTransformerUtils"; +import { Id64String } from "@itwin/core-bentley"; +import { + ElementProps, + ExternalSourceAspectProps, + IModel, +} from "@itwin/core-common"; +import { ChangedInstanceIds, ChangedInstanceOps } from "../../IModelExporter"; +import { expect } from "chai"; + +describe("ChangedInstanceIds", () => { + const outputDir = path.join( + KnownTestLocations.outputDir, + "IModelTransformer" + ); + + let sourceDb: SnapshotDb; + + let documentListModel: Id64String; + let parentDrawing: ElementProps; + let childDrawing1: ElementProps; + let childDrawing2: ElementProps; + let aspect1Id: Id64String; + let aspect2Id: Id64String; + let parentAspect1Id: Id64String; + let parentRelationshipId: Id64String; + let relationshipId: Id64String; + + before(async () => { + if (!IModelJsFs.existsSync(KnownTestLocations.outputDir)) { + IModelJsFs.mkdirSync(KnownTestLocations.outputDir); + } + if (!IModelJsFs.existsSync(outputDir)) { + IModelJsFs.mkdirSync(outputDir); + } + + sourceDb = prepareSnapshotDb("ChangedInstanceIds"); + // add data to source iModel + const sourceSubjectId = Subject.insert( + sourceDb, + IModel.rootSubjectId, + "S1" + ); + documentListModel = DocumentListModel.insert( + sourceDb, + sourceSubjectId, + "DL" + ); + parentDrawing = insertDrawingElement( + sourceDb, + documentListModel, + "ParentDrawing" + ); + childDrawing1 = insertDrawingElement( + sourceDb, + parentDrawing.id!, + "ChildDrawing1" + ); + childDrawing2 = insertDrawingElement( + sourceDb, + parentDrawing.id!, + "ChildDrawing2" + ); + const parentDrawing2 = insertDrawingElement( + sourceDb, + documentListModel, + "ParentDrawing2" + ); + insertDrawingElement(sourceDb, parentDrawing2.id!, "ChildDrawing3"); + parentAspect1Id = insertElementAspect( + sourceDb, + sourceSubjectId, + parentDrawing.id!, + "TestParentAspect1" + ); + aspect1Id = insertElementAspect( + sourceDb, + sourceSubjectId, + childDrawing1.id!, + "TestChildAspect1" + ); + aspect2Id = insertElementAspect( + sourceDb, + sourceSubjectId, + childDrawing2.id!, + "TestChildAspect2" + ); + relationshipId = ElementGroupsMembers.create( + sourceDb, + childDrawing1.id!, + childDrawing2.id!, + 0 + ).insert(); + + parentRelationshipId = ElementGroupsMembers.create( + sourceDb, + parentDrawing.id!, + parentDrawing2.id!, + 0 + ).insert(); + + sourceDb.saveChanges(); + }); + + after(() => { + sourceDb.close(); + }); + + function prepareSnapshotDb(name: string) { + const sourceDbPath = IModelTransformerTestUtils.prepareOutputFile( + "ChangedInstanceIds", + `${name}.bim` + ); + return SnapshotDb.createEmpty(sourceDbPath, { + rootSubject: { name }, + }); + } + + function insertDrawingElement( + iModel: IModelDb, + documentListModelId: Id64String, + drawingName: string + ): ElementProps { + const id = Drawing.insert(iModel, documentListModelId, drawingName); + return iModel.elements.getElementProps(id); + } + + function insertElementAspect( + iModel: IModelDb, + scopeId: Id64String, + elementId: Id64String, + identifier: string + ): Id64String { + const aspectProps: ExternalSourceAspectProps = { + classFullName: ExternalSourceAspect.classFullName, + kind: "something", + scope: { id: scopeId }, + element: { + id: elementId, + relClassName: ElementOwnsExternalSourceAspects.classFullName, + }, + identifier, + }; + + return iModel.elements.insertAspect(aspectProps); + } + + function assertHasValues( + instanceOps: ChangedInstanceOps, + propertyName: string, + expectedInserted: Id64String[], + expectedUpdated: Id64String[], + expectedDeleted: Id64String[] + ) { + expect([...instanceOps.insertIds]).to.have.all.members( + expectedInserted, + `'${propertyName}.insertIds' contains different values than expected` + ); + expect([...instanceOps.updateIds]).to.have.all.members( + expectedUpdated, + `'${propertyName}.updateIds' contains different values than expected` + ); + expect([...instanceOps.deleteIds]).to.have.all.members( + expectedDeleted, + `'${propertyName}.deleteIds' contains different values than expected` + ); + } + describe("addCustomElementChange", async function () { + it("should add changes for related entities when element is Inserted", async function () { + const sourceDbChanges = new ChangedInstanceIds(sourceDb); + await sourceDbChanges.addCustomElementChange( + "Inserted", + childDrawing1.id! + ); + + assertHasValues( + sourceDbChanges.element, + "element", + [childDrawing1.id!], + ["0x1", documentListModel, parentDrawing.id!], + [] + ); + assertHasValues( + sourceDbChanges.model, + "model", + [], + ["0x1", documentListModel, parentDrawing.id!], + [] + ); + assertHasValues( + sourceDbChanges.aspect, + "aspect", + [aspect1Id, parentAspect1Id], + [], + [] + ); + assertHasValues( + sourceDbChanges.relationship, + "relationship", + [relationshipId, parentRelationshipId], + [], + [] + ); + }); + + it("should add changes for related entities when element is Updated", async function () { + const sourceDbChanges = new ChangedInstanceIds(sourceDb); + await sourceDbChanges.addCustomElementChange( + "Updated", + childDrawing1.id! + ); + + assertHasValues( + sourceDbChanges.element, + "element", + [], + ["0x1", documentListModel, parentDrawing.id!, childDrawing1.id!], + [] + ); + assertHasValues( + sourceDbChanges.model, + "model", + [], + ["0x1", documentListModel, parentDrawing.id!], + [] + ); + assertHasValues(sourceDbChanges.aspect, "aspect", [], [], []); + assertHasValues(sourceDbChanges.relationship, "relationship", [], [], []); + }); + + it("should add changes for related entities when multiple elements are updated", async function () { + const sourceDbChanges = new ChangedInstanceIds(sourceDb); + await sourceDbChanges.addCustomElementChange("Updated", [ + childDrawing1.id!, + childDrawing2.id!, + ]); + + assertHasValues( + sourceDbChanges.element, + "element", + [], + [ + "0x1", + documentListModel, + parentDrawing.id!, + childDrawing1.id!, + childDrawing2.id!, + ], + [] + ); + assertHasValues( + sourceDbChanges.model, + "model", + [], + ["0x1", documentListModel, parentDrawing.id!], + [] + ); + assertHasValues(sourceDbChanges.aspect, "aspect", [], [], []); + assertHasValues(sourceDbChanges.relationship, "relationship", [], [], []); + }); + + it("should add changes for related entities when element is Deleted", async function () { + const sourceDbChanges = new ChangedInstanceIds(sourceDb); + await sourceDbChanges.addCustomElementChange( + "Deleted", + childDrawing1.id! + ); + + assertHasValues( + sourceDbChanges.element, + "element", + [], + [], + [childDrawing1.id!] + ); + assertHasValues(sourceDbChanges.model, "model", [], [], []); + assertHasValues(sourceDbChanges.aspect, "aspect", [], [], []); + assertHasValues(sourceDbChanges.relationship, "relationship", [], [], []); + }); + + it("should not add changes when empty array is passed for custom element change ", async function () { + const sourceDbChanges = new ChangedInstanceIds(sourceDb); + await sourceDbChanges.addCustomElementChange("Inserted", []); + + assertHasValues(sourceDbChanges.element, "element", [], [], []); + assertHasValues(sourceDbChanges.model, "model", [], [], []); + assertHasValues(sourceDbChanges.aspect, "aspect", [], [], []); + assertHasValues(sourceDbChanges.relationship, "relationship", [], [], []); + }); + }); + + describe("addCustomModelChange", async function () { + it("should add custom changes when one model is inserted", async function () { + const sourceDbChanges = new ChangedInstanceIds(sourceDb); + await sourceDbChanges.addCustomModelChange("Inserted", parentDrawing.id!); + // Act + assertHasValues( + sourceDbChanges.element, + "element", + [parentDrawing.id!], + [documentListModel, "0x1"], + [] + ); + assertHasValues( + sourceDbChanges.model, + "model", + [parentDrawing.id!], + [documentListModel, "0x1"], + [] + ); + assertHasValues( + sourceDbChanges.aspect, + "aspect", + [parentAspect1Id], + [], + [] + ); + assertHasValues( + sourceDbChanges.relationship, + "relationship", + [parentRelationshipId], + [], + [] + ); + }); + + it("should add custom changes when multiple models are inserted", async function () { + // Arrange + const sourceDbChanges = new ChangedInstanceIds(sourceDb); + await sourceDbChanges.addCustomModelChange("Inserted", [ + childDrawing1.id!, + childDrawing2.id!, + ]); + + // Act + assertHasValues( + sourceDbChanges.element, + "element", + [childDrawing1.id!, childDrawing2.id!], + ["0x1", documentListModel, parentDrawing.id!], + [] + ); + assertHasValues( + sourceDbChanges.model, + "model", + [childDrawing1.id!, childDrawing2.id!], + ["0x1", documentListModel, parentDrawing.id!], + [] + ); + assertHasValues( + sourceDbChanges.aspect, + "aspect", + [aspect1Id, aspect2Id, parentAspect1Id], + [], + [] + ); + assertHasValues( + sourceDbChanges.relationship, + "relationship", + [relationshipId, parentRelationshipId], + [], + [] + ); + }); + + it("should add custom changes when model is Updated", async function () { + const sourceDbChanges = new ChangedInstanceIds(sourceDb); + await sourceDbChanges.addCustomModelChange("Updated", parentDrawing.id!); + // Act + assertHasValues( + sourceDbChanges.element, + "element", + [], + [documentListModel, "0x1", parentDrawing.id!], + [] + ); + assertHasValues( + sourceDbChanges.model, + "model", + [], + [documentListModel, "0x1", parentDrawing.id!], + [] + ); + assertHasValues(sourceDbChanges.aspect, "aspect", [], [], []); + assertHasValues(sourceDbChanges.relationship, "relationship", [], [], []); + }); + + it("should add custom changes when model is Deleted", async function () { + const sourceDbChanges = new ChangedInstanceIds(sourceDb); + await sourceDbChanges.addCustomModelChange("Deleted", parentDrawing.id!); + // Act + assertHasValues( + sourceDbChanges.element, + "element", + [], + [], + [parentDrawing.id!] + ); + assertHasValues( + sourceDbChanges.model, + "model", + [], + [], + [parentDrawing.id!] + ); + assertHasValues(sourceDbChanges.aspect, "aspect", [], [], []); + assertHasValues(sourceDbChanges.relationship, "relationship", [], [], []); + }); + + it("should not add changes when empty array is passed for custom model change ", async function () { + const sourceDbChanges = new ChangedInstanceIds(sourceDb); + await sourceDbChanges.addCustomModelChange("Inserted", []); + + assertHasValues(sourceDbChanges.element, "element", [], [], []); + assertHasValues(sourceDbChanges.model, "model", [], [], []); + assertHasValues(sourceDbChanges.aspect, "aspect", [], [], []); + assertHasValues(sourceDbChanges.relationship, "relationship", [], [], []); + }); + }); + + describe("addCustomAspectChange", async function () { + it("should add custom changes when aspect is Inserted", async function () { + const sourceDbChanges = new ChangedInstanceIds(sourceDb); + sourceDbChanges.addCustomAspectChange("Inserted", aspect1Id); + // Act + assertHasValues(sourceDbChanges.element, "element", [], [], []); + assertHasValues(sourceDbChanges.model, "model", [], [], []); + assertHasValues(sourceDbChanges.aspect, "aspect", [aspect1Id], [], []); + assertHasValues(sourceDbChanges.relationship, "relationship", [], [], []); + }); + + it("should add custom changes when aspect is Updated", async function () { + const sourceDbChanges = new ChangedInstanceIds(sourceDb); + sourceDbChanges.addCustomAspectChange("Updated", aspect1Id); + // Act + assertHasValues(sourceDbChanges.element, "element", [], [], []); + assertHasValues(sourceDbChanges.model, "model", [], [], []); + assertHasValues(sourceDbChanges.aspect, "aspect", [], [aspect1Id], []); + assertHasValues(sourceDbChanges.relationship, "relationship", [], [], []); + }); + + it("should add custom changes when aspect is Deleted", async function () { + const sourceDbChanges = new ChangedInstanceIds(sourceDb); + sourceDbChanges.addCustomAspectChange("Deleted", aspect1Id); + // Act + assertHasValues(sourceDbChanges.element, "element", [], [], []); + assertHasValues(sourceDbChanges.model, "model", [], [], []); + assertHasValues(sourceDbChanges.aspect, "aspect", [], [], [aspect1Id]); + assertHasValues(sourceDbChanges.relationship, "relationship", [], [], []); + }); + + it("should not add changes when empty array is passed for custom aspect change ", async function () { + const sourceDbChanges = new ChangedInstanceIds(sourceDb); + sourceDbChanges.addCustomAspectChange("Inserted", []); + assertHasValues(sourceDbChanges.element, "element", [], [], []); + assertHasValues(sourceDbChanges.model, "model", [], [], []); + assertHasValues(sourceDbChanges.aspect, "aspect", [], [], []); + assertHasValues(sourceDbChanges.relationship, "relationship", [], [], []); + }); + }); +}); diff --git a/packages/transformer/src/test/standalone/IModelTransformer.test.ts b/packages/transformer/src/test/standalone/IModelTransformer.test.ts index 75292349..d88c4ffb 100644 --- a/packages/transformer/src/test/standalone/IModelTransformer.test.ts +++ b/packages/transformer/src/test/standalone/IModelTransformer.test.ts @@ -3086,24 +3086,9 @@ describe("IModelTransformer", () => { rootSubject: { name: "deferred-element-with-aspects" }, }); - const testSchema1Path = IModelTransformerTestUtils.prepareOutputFile( - "IModelTransformer", - "TestSchema1.ecschema.xml" - ); - // the only two ElementUniqueAspect's in bis are ignored by the transformer, so we add our own to test their export - IModelJsFs.writeFileSync( - testSchema1Path, - ` - - - - bis:ElementUniqueAspect - - - ` - ); - - await sourceDb.importSchemas([testSchema1Path]); + const testSchemaPath = + IModelTransformerTestUtils.getPathToSchemaWithUniqueAspect(); + await sourceDb.importSchemas([testSchemaPath]); const myPhysicalModelId = PhysicalModel.insert( sourceDb, diff --git a/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts b/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts index 814caa23..ed40b5c1 100644 --- a/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts +++ b/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts @@ -17,6 +17,9 @@ import { DefinitionPartition, deleteElementTree, DisplayStyle3d, + DocumentListModel, + Drawing, + DrawingModel, ECSqlStatement, // eslint-disable-next-line @typescript-eslint/no-redeclare Element, @@ -26,6 +29,7 @@ import { ElementRefersToElements, ExternalSourceAspect, GenericSchema, + GeometricModel, HubMock, IModelDb, IModelHost, @@ -78,6 +82,7 @@ import { IModelExporter, IModelImporter, IModelTransformer, + IModelTransformOptions, ProcessChangesOptions, TransformerLoggerCategory, } from "../../imodel-transformer"; @@ -1302,7 +1307,9 @@ describe("IModelTransformerHub", () => { master: { sync: [ "branch1", - { initTransformer: setForceOldRelationshipProvenanceMethod }, + { + initTransformer: setForceOldRelationshipProvenanceMethod, + }, ], }, }, // first master<-branch1 reverse sync picking up new relationship from branch imodel @@ -1346,7 +1353,9 @@ describe("IModelTransformerHub", () => { branch1: { sync: [ "master", - { initTransformer: setForceOldRelationshipProvenanceMethod }, + { + initTransformer: setForceOldRelationshipProvenanceMethod, + }, ], }, }, // forward sync master->branch1 to pick up delete of relationship @@ -4829,4 +4838,1090 @@ describe("IModelTransformerHub", () => { const { tearDown } = await runTimeline(timeline, { iTwinId, accessToken }); await tearDown(); }); + + describe("addCustomChanges", () => { + let sourceDb: BriefcaseDb; + let targetDb: BriefcaseDb; + + beforeEach(async () => { + sourceDb = await prepareBriefcase("source"); + targetDb = await prepareBriefcase("target"); + }); + + afterEach(async () => { + await closeAndDeleteBriefcase(sourceDb); + await closeAndDeleteBriefcase(targetDb); + }); + + async function prepareBriefcase(name: string) { + const iModelId = await HubWrappers.createIModel( + accessToken, + iTwinId, + name + ); + + const newBriefcase = await HubWrappers.downloadAndOpenBriefcase({ + accessToken: await IModelHost.getAccessToken(), + iTwinId, + iModelId, + asOf: IModelVersion.latest().toJSON(), + }); + await newBriefcase.locks.acquireLocks({ + shared: "0x10", + exclusive: "0x1", + }); + return newBriefcase; + } + + async function closeAndDeleteBriefcase(iModel: BriefcaseDb) { + await HubWrappers.closeAndDeleteBriefcaseDb(accessToken, iModel); + // eslint-disable-next-line @itwin/no-internal + await IModelHost.hubAccess.deleteIModel({ + iTwinId, + iModelId: iModel.iModelId, + }); + } + + async function pushChanges(iModel: BriefcaseDb, description: string) { + iModel.saveChanges(); + await iModel.pushChanges({ description, retainLocks: true }); + } + class CustomChangesTransformer extends IModelTransformer { + constructor( + source: IModelDb, + target: IModelDb, + isChangeProcessing: boolean + ) { + const options: IModelTransformOptions = { + includeSourceProvenance: true, + }; + if (isChangeProcessing) { + options.argsForProcessChanges = {}; + } + const exporter = new IModelExporter( + source, + DetachedExportElementAspectsStrategy + ); + super(exporter, target, options); + } + + public override async addCustomChanges( + _sourceDbChanges: ChangedInstanceIds + ) {} + public override shouldExportElement(sourceElement: Element) { + return super.shouldExportElement(sourceElement); + } + } + + it("should call addCustomChanges when processing changes after source and target id map is populated", async () => { + // set up source + const sourceModelId0 = PhysicalModel.insert( + sourceDb, + IModel.rootSubjectId, + "M0" + ); + await pushChanges(sourceDb, "Initial source data"); + + // process all + let transformer = new CustomChangesTransformer(sourceDb, targetDb, false); + let addChangesStub = sinon.stub(transformer, "addCustomChanges"); + await transformer.process(); + await pushChanges(targetDb, "target changes for transformation 1"); + expect(addChangesStub.calledOnce).to.be.false; + + // process changes + transformer = new CustomChangesTransformer(sourceDb, targetDb, true); + addChangesStub = sinon + .stub(transformer, "addCustomChanges") + .callsFake(async (_sourceDbChanges) => { + const targetId = + transformer.context.findTargetElementId(sourceModelId0); + expect( + targetId, + "addCustomChanges should be called only after elements are mapped in clone context" + ).to.not.be.equal(Id64.invalid); + }); + await transformer.process(); + await pushChanges(targetDb, "target changes for transformation 2"); + expect(addChangesStub.calledOnce).to.be.true; + }); + + it("should update data in target correctly when custom changes are registered for models", async () => { + // Arrange + const sourceSubjectId = Subject.insert( + sourceDb, + IModel.rootSubjectId, + "S1" + ); + // Create Drawing model hierarchy + const documentListModel = DocumentListModel.insert( + sourceDb, + sourceSubjectId, + "DL" + ); + const parentDrawing = insertDrawingElement( + sourceDb, + documentListModel, + "DrawingParent" + ); + const childDrawing = insertDrawingElement( + sourceDb, + parentDrawing.id!, + "DrawingChild" + ); + // Create physical model + const physicalModel1Id = PhysicalModel.insert( + sourceDb, + sourceSubjectId, + "PM1" + ); + const categoryId1 = SpatialCategory.insert( + sourceDb, + IModel.dictionaryId, + "C1", + {} + ); + const physicalElem1 = insertPhysicalElement( + sourceDb, + physicalModel1Id, + categoryId1, + "PhysicalOne" + ); + await pushChanges(sourceDb, "Initial changes"); + + // === Transformation 1: Run `process all` transformation === + let transformer = new CustomChangesTransformer(sourceDb, targetDb, false); + sinon + .stub(transformer, "shouldExportElement") + .callsFake((sourceElement) => { + // Exclude all drawings + return sourceElement.id !== documentListModel; + }); + await transformer.process(); + transformer.updateSynchronizationVersion({ + initializeReverseSyncVersion: true, + }); + await pushChanges(targetDb, "Transformation 1: Process All"); + + // Assert + expect( + IModelTestUtils.count(targetDb, GeometricModel.classFullName) + ).to.be.equal(1); + expect( + IModelTestUtils.count(targetDb, DrawingModel.classFullName) + ).to.be.equal(0); + expect(IModelTestUtils.queryByCodeValue(targetDb, "PM1")).to.not.be.equal( + Id64.invalid + ); + assertElementsExistByCode(targetDb, [physicalElem1]); + assertElementsDoNotExistByCode(targetDb, [parentDrawing, childDrawing]); + + // === Transformation 2: `process changes` transformation to insert excluded parent model === + transformer = new CustomChangesTransformer(sourceDb, targetDb, true); + sinon + .stub(transformer, "shouldExportElement") + .callsFake((_sourceElement) => true); + sinon + .stub(transformer, "addCustomChanges") + .callsFake(async (sourceDbChanges) => { + expect( + sourceDbChanges.hasChanges, + "there should be only custom changes" + ).to.be.false; + await sourceDbChanges.addCustomModelChange( + "Inserted", + parentDrawing.id! + ); + }); + await transformer.process(); + await pushChanges( + targetDb, + "Transformation 2: inserted previously excluded model" + ); + // Assert + expect( + IModelTestUtils.count(targetDb, GeometricModel.classFullName) + ).to.be.equal(2); + assertModelExistsByName(targetDb, ["PM1", "DL", "DrawingParent"]); + expect( + IModelTestUtils.count(targetDb, DrawingModel.classFullName) + ).to.be.equal(1); + assertElementsExistByCode(targetDb, [physicalElem1, parentDrawing]); + assertElementsDoNotExistByCode(targetDb, [childDrawing]); + + // === Transformation 3: `process changes` transformation to include newly added model === + // Act + const physicalModel2Id = PhysicalModel.insert( + sourceDb, + sourceSubjectId, + "PM2" + ); + const physicalElem2 = insertPhysicalElement( + sourceDb, + physicalModel2Id, + categoryId1, + "PhysicalTwo" + ); + await pushChanges(sourceDb, "Added new physical model"); + + transformer = new CustomChangesTransformer(sourceDb, targetDb, true); + sinon + .stub(transformer, "shouldExportElement") + .callsFake((_sourceElement) => true); + sinon + .stub(transformer, "addCustomChanges") + .callsFake(async (sourceDbChanges) => { + await sourceDbChanges.addCustomModelChange( + "Inserted", + physicalModel2Id + ); + }); + await transformer.process(); + await pushChanges( + targetDb, + "Transformation 3: inserted newly created model" + ); + // Assert + expect( + IModelTestUtils.count(targetDb, GeometricModel.classFullName) + ).to.be.equal(3); + expect( + IModelTestUtils.count(targetDb, DrawingModel.classFullName) + ).to.be.equal(1); + assertModelExistsByName(targetDb, ["PM1", "DL", "DrawingParent", "PM2"]); + assertElementsExistByCode(targetDb, [ + physicalElem1, + physicalElem2, + parentDrawing, + ]); + assertElementsDoNotExistByCode(targetDb, [childDrawing]); + + // === Transformation 4: `process changes` transformation to delete existing model === + transformer = new CustomChangesTransformer(sourceDb, targetDb, true); + sinon + .stub(transformer, "shouldExportElement") + .callsFake((_sourceElement) => true); + sinon + .stub(transformer, "addCustomChanges") + .callsFake(async (sourceDbChanges) => { + expect( + sourceDbChanges.hasChanges, + "there should be only custom changes" + ).to.be.false; + await sourceDbChanges.addCustomModelChange( + "Deleted", + physicalModel1Id + ); + await sourceDbChanges.addCustomModelChange( + "Deleted", + parentDrawing.id! + ); + }); + await transformer.process(); + await pushChanges(targetDb, "Transformation 4: delete exported model"); + // Assert + expect( + IModelTestUtils.count(targetDb, GeometricModel.classFullName) + ).to.be.equal(1); + expect( + IModelTestUtils.count(targetDb, DrawingModel.classFullName) + ).to.be.equal(0); + assertModelExistsByName(targetDb, ["DL", "PM2"]); + assertModelDoesNotExistsByName(targetDb, ["PM1", "DrawingParent"]); + assertElementsExistByCode(targetDb, [physicalElem2]); + assertElementsDoNotExistByCode(targetDb, [ + physicalElem1, + parentDrawing, + childDrawing, + ]); + // === Transformation 5: `process changes` transformation to delete existing model with newly added elements === + const physicalElem3 = insertPhysicalElement( + sourceDb, + physicalModel2Id, + categoryId1, + "PhysicalThree" + ); + await pushChanges(sourceDb, "Added new physical element into PM2"); + transformer = new CustomChangesTransformer(sourceDb, targetDb, true); + sinon + .stub(transformer, "shouldExportElement") + .callsFake((_sourceElement) => true); + sinon + .stub(transformer, "addCustomChanges") + .callsFake(async (sourceDbChanges) => { + await sourceDbChanges.addCustomModelChange( + "Deleted", + physicalModel2Id + ); + }); + await transformer.process(); + await pushChanges( + targetDb, + "Transformation 5: delete model with newly added elements" + ); + // Assert + expect( + IModelTestUtils.count(targetDb, GeometricModel.classFullName) + ).to.be.equal(0); + assertModelDoesNotExistsByName(targetDb, ["PM2"]); + assertElementsDoNotExistByCode(targetDb, [physicalElem2, physicalElem3]); + }); + + it("should update modeled element and its related data when custom changes are added for it's sub model", async function () { + // === Transformation 1: Run `process all` transformation === + // Arrange + const sourceSubjectId = Subject.insert( + sourceDb, + IModel.rootSubjectId, + "S1" + ); + const documentListModel = DocumentListModel.insert( + sourceDb, + sourceSubjectId, + "DL" + ); + const parentDrawing1 = insertDrawingElement( + sourceDb, + documentListModel, + "ParentDrawing1" + ); + const parentDrawing2 = insertDrawingElement( + sourceDb, + documentListModel, + "ParentDrawing2" + ); + const childDrawing1 = insertDrawingElement( + sourceDb, + parentDrawing1.id!, + "ChildDrawing1" + ); + const childDrawing2 = insertDrawingElement( + sourceDb, + parentDrawing1.id!, + "ChildDrawing2" + ); + insertElementAspect( + sourceDb, + sourceSubjectId, + parentDrawing1.id!, + "ParentAspect1" + ); + insertElementAspect( + sourceDb, + sourceSubjectId, + childDrawing1.id!, + "TestAspect1" + ); + insertElementAspect( + sourceDb, + sourceSubjectId, + childDrawing2.id!, + "TestAspect2" + ); + insertElementGroupsElementsRelationship( + sourceDb, + parentDrawing1.id!, + parentDrawing2.id! + ); + + insertElementGroupsElementsRelationship( + sourceDb, + childDrawing1.id!, + childDrawing2.id! + ); + await pushChanges(sourceDb, "Initial changes"); + // Act + let transformer = new CustomChangesTransformer(sourceDb, targetDb, false); + sinon + .stub(transformer, "shouldExportElement") + .callsFake((sourceElement) => { + // Exclude all drawings + return sourceElement.id !== parentDrawing1.id!; + }); + await transformer.process(); + transformer.updateSynchronizationVersion({ + initializeReverseSyncVersion: true, + }); + await pushChanges(targetDb, "Transformation 1: Process All"); + + assertModelExistsByName(targetDb, ["DL", "ParentDrawing2"]); + assertModelDoesNotExistsByName(targetDb, [ + "ParentDrawing1", + "ChildDrawing1", + "ChildDrawing2", + ]); + assertElementsDoNotExistByCode(targetDb, [ + parentDrawing1, + childDrawing1, + childDrawing2, + ]); + + // === Transformation 2: `process changes` transformation to include first child element's sub model === + // Act + // insert first child and keep excluding second child + transformer = new CustomChangesTransformer(sourceDb, targetDb, true); + sinon + .stub(transformer, "shouldExportElement") + .callsFake((sourceElement) => sourceElement.id !== childDrawing2.id!); + + sinon + .stub(transformer, "addCustomChanges") + .callsFake(async (sourceDbChanges) => { + expect( + sourceDbChanges.hasChanges, + "there should be only custom changes" + ).to.be.false; + await sourceDbChanges.addCustomModelChange( + "Inserted", + childDrawing1.id! + ); + }); + await transformer.process(); + await pushChanges( + targetDb, + "Transformation 2: add first previously excluded child element" + ); + + assertModelExistsByName(targetDb, [ + "DL", + "ParentDrawing1", + "ParentDrawing2", + "ChildDrawing1", + ]); + assertModelDoesNotExistsByName(targetDb, ["ChildDrawing2"]); + assertElementsExistByCode(targetDb, [parentDrawing1, childDrawing1]); + assertElementsDoNotExistByCode(targetDb, [childDrawing2]); + assertElementHasExpectedAspectCount( + targetDb, + childDrawing1.federationGuid!, + 1 + ); + assertElementHasExpectedAspectCount( + targetDb, + parentDrawing1.federationGuid!, + 1 + ); + expect( + IModelTestUtils.count(targetDb, ElementGroupsMembers.classFullName) + ).to.be.equal(1); + + // === Transformation 3: `process changes` transformation to include second child element's sub model === + transformer = new CustomChangesTransformer(sourceDb, targetDb, true); + sinon + .stub(transformer, "shouldExportElement") + .callsFake((_sourceElement) => true); + sinon + .stub(transformer, "addCustomChanges") + .callsFake(async (sourceDbChanges) => { + expect( + sourceDbChanges.hasChanges, + "there should be only custom changes" + ).to.be.false; + await sourceDbChanges.addCustomModelChange( + "Inserted", + childDrawing2.id! + ); + }); + await transformer.process(); + await pushChanges( + targetDb, + "Transformation 2: add second previously excluded child element" + ); + // Assert + assertModelExistsByName(targetDb, [ + "DL", + "ParentDrawing1", + "ParentDrawing2", + "ChildDrawing1", + "ChildDrawing2", + ]); + assertElementsExistByCode(targetDb, [ + parentDrawing1, + childDrawing1, + childDrawing2, + ]); + assertElementHasExpectedAspectCount( + targetDb, + childDrawing2.federationGuid!, + 1 + ); + expect( + IModelTestUtils.count(targetDb, ElementGroupsMembers.classFullName) + ).to.be.equal(2); + + // === Transformation 4: `process changes` transformation to delete first child element's sub model === + transformer = new CustomChangesTransformer(sourceDb, targetDb, true); + sinon + .stub(transformer, "shouldExportElement") + .callsFake((_sourceElement) => true); + sinon + .stub(transformer, "addCustomChanges") + .callsFake(async (sourceDbChanges) => { + expect( + sourceDbChanges.hasChanges, + "there should be only custom changes" + ).to.be.false; + await sourceDbChanges.addCustomModelChange( + "Deleted", + childDrawing1.id! + ); + }); + await transformer.process(); + await pushChanges( + targetDb, + "Transformation 3: delete first child element's submodel" + ); + assertModelExistsByName(targetDb, [ + "DL", + "ParentDrawing1", + "ParentDrawing2", + "ChildDrawing2", + ]); + assertElementsExistByCode(targetDb, [parentDrawing1, childDrawing2]); + assertElementsDoNotExistByCode(targetDb, [childDrawing1]); + expect( + IModelTestUtils.count(targetDb, ElementGroupsMembers.classFullName) + ).to.be.equal(1); + }); + + it("should update exported data correctly when custom changes are registered for elements", async function () { + // Prepare source + const sourceSubjectId = Subject.insert( + sourceDb, + IModel.rootSubjectId, + "S1" + ); + const categoryId1 = SpatialCategory.insert( + sourceDb, + IModel.dictionaryId, + "C1", + {} + ); + const physicalModel1Id = PhysicalModel.insert( + sourceDb, + sourceSubjectId, + "PM1" + ); + const physicalModel2Id = PhysicalModel.insert( + sourceDb, + sourceSubjectId, + "PM2" + ); + const physicalElem1 = insertPhysicalElement( + sourceDb, + physicalModel1Id, + categoryId1, + "PhysicalOne" + ); + const physicalElem2 = insertPhysicalElement( + sourceDb, + physicalModel2Id, + categoryId1, + "PhysicalTwo" + ); + insertElementAspect( + sourceDb, + sourceSubjectId, + physicalElem1.id!, + "TestAspect1" + ); + insertElementAspect( + sourceDb, + sourceSubjectId, + physicalElem2.id!, + "TestAspect2" + ); + insertElementGroupsElementsRelationship( + sourceDb, + physicalElem1.id!, + physicalElem2.id! + ); + await pushChanges(sourceDb, "Initial changes"); + + // === Transformation 1: Run `process all` transformation === + let transformer = new CustomChangesTransformer(sourceDb, targetDb, false); + sinon + .stub(transformer, "shouldExportElement") + .callsFake((sourceElement) => { + // will exclude 'PM2' + return sourceElement.id !== physicalModel2Id; + }); + await transformer.process(); + transformer.updateSynchronizationVersion({ + initializeReverseSyncVersion: true, + }); + await pushChanges(targetDb, "Transformation 1: Process All"); + + expect( + IModelTestUtils.count(targetDb, GeometricModel.classFullName) + ).to.be.equal(1); + expect( + IModelTestUtils.count(targetDb, ElementGroupsMembers.classFullName) + ).to.be.equal(0); + assertModelExistsByName(targetDb, ["PM1"]); + assertModelDoesNotExistsByName(targetDb, ["PM2"]); + assertElementsExistByCode(targetDb, [physicalElem1]); + assertElementsDoNotExistByCode(targetDb, [physicalElem2]); + assertElementHasExpectedAspectCount( + targetDb, + physicalElem1.federationGuid!, + 1 + ); + + // === Transformation 2: `process changes` transformation to include excluded element === + transformer = new CustomChangesTransformer(sourceDb, targetDb, true); + sinon + .stub(transformer, "shouldExportElement") + .callsFake((_sourceElement) => true); + sinon + .stub(transformer, "addCustomChanges") + .callsFake(async (sourceDbChanges) => { + expect( + sourceDbChanges.hasChanges, + "there should be only custom changes" + ).to.be.false; + await sourceDbChanges.addCustomElementChange( + "Inserted", + physicalElem2.id! + ); + }); + await transformer.process(); + await pushChanges( + targetDb, + "Transformation 2: include previously excluded element" + ); + + expect( + IModelTestUtils.count(targetDb, GeometricModel.classFullName) + ).to.be.equal(2); + expect( + IModelTestUtils.count(targetDb, ElementGroupsMembers.classFullName) + ).to.be.equal(1); + assertModelExistsByName(targetDb, ["PM1", "PM2"]); + assertElementsExistByCode(targetDb, [physicalElem1, physicalElem2]); + assertElementHasExpectedAspectCount( + targetDb, + physicalElem2.federationGuid!, + 1 + ); + + // === Transformation 3: `process changes` transformation to include newly added element === + const physicalModel3Id = PhysicalModel.insert( + sourceDb, + sourceSubjectId, + "PM3" + ); + const physicalElem3 = insertPhysicalElement( + sourceDb, + physicalModel3Id, + categoryId1, + "PhysicalThree" + ); + await pushChanges(sourceDb, "Added new model and physical element"); + + transformer = new CustomChangesTransformer(sourceDb, targetDb, true); + sinon + .stub(transformer, "shouldExportElement") + .callsFake((_sourceElement) => true); + sinon + .stub(transformer, "addCustomChanges") + .callsFake(async (sourceDbChanges) => { + await sourceDbChanges.addCustomElementChange( + "Inserted", + physicalElem3.id! + ); + }); + await transformer.process(); + await pushChanges( + targetDb, + "Transformation 3: include newly added element" + ); + + expect( + IModelTestUtils.count(targetDb, GeometricModel.classFullName) + ).to.be.equal(3); + expect( + IModelTestUtils.count(targetDb, ElementGroupsMembers.classFullName) + ).to.be.equal(1); + assertModelExistsByName(targetDb, ["PM1", "PM2", "PM3"]); + assertElementsExistByCode(targetDb, [ + physicalElem1, + physicalElem2, + physicalElem3, + ]); + + // === Transformation 4: `process changes` transformation to delete exported element === + transformer = new CustomChangesTransformer(sourceDb, targetDb, true); + sinon + .stub(transformer, "shouldExportElement") + .callsFake((_sourceElement) => true); + sinon + .stub(transformer, "addCustomChanges") + .callsFake(async (sourceDbChanges) => { + expect( + sourceDbChanges.hasChanges, + "there should be only custom changes" + ).to.be.false; + await sourceDbChanges.addCustomElementChange( + "Deleted", + physicalElem1.id! + ); + }); + await transformer.process(); + await pushChanges(targetDb, "Transformation 4: delete exported element"); + // Assert + expect( + IModelTestUtils.count(targetDb, GeometricModel.classFullName) + ).to.be.equal(3); + expect( + IModelTestUtils.count(targetDb, ElementGroupsMembers.classFullName) + ).to.be.equal(0); + assertModelExistsByName(targetDb, ["PM1", "PM2", "PM3"]); + assertElementsExistByCode(targetDb, [physicalElem2, physicalElem3]); + assertElementsDoNotExistByCode(targetDb, [physicalElem1]); + }); + + it("should reset element values when custom changes to update element are added", async function () { + // Arrange + const sourceSubjectId = Subject.insert( + sourceDb, + IModel.rootSubjectId, + "S1" + ); + const categoryId1 = SpatialCategory.insert( + sourceDb, + IModel.dictionaryId, + "C1", + {} + ); + + const physicalModel1Id = PhysicalModel.insert( + sourceDb, + sourceSubjectId, + "PM1" + ); + const physicalModel2Id = PhysicalModel.insert( + sourceDb, + sourceSubjectId, + "PM2" + ); + const physicalElem1 = insertPhysicalElement( + sourceDb, + physicalModel1Id, + categoryId1, + "PhysicalOne" + ); + const physicalElem2 = insertPhysicalElement( + sourceDb, + physicalModel2Id, + categoryId1, + "PhysicalTwo" + ); + await pushChanges(sourceDb, "Initial changes"); + + // === Transformation 1: Run `process all` transformation === + let transformer = new CustomChangesTransformer(sourceDb, targetDb, false); + sinon + .stub(transformer, "shouldExportElement") + .callsFake((_sourceElement) => true); + + await transformer.process(); + transformer.updateSynchronizationVersion({ + initializeReverseSyncVersion: true, + }); + await pushChanges(targetDb, "Transformation 1: Process All"); + + // === Transformation 2: `process changes` transformation to update other element === + // Update element in target + const physicalElem1InTargetProps = targetDb.elements.getElementProps( + physicalElem1.federationGuid! + ); + physicalElem1InTargetProps.userLabel = "Updated"; + targetDb.elements.updateElement(physicalElem1InTargetProps); + + transformer = new CustomChangesTransformer(sourceDb, targetDb, true); + sinon + .stub(transformer, "shouldExportElement") + .callsFake((_sourceElement) => true); + sinon + .stub(transformer, "addCustomChanges") + .callsFake(async (sourceDbChanges) => { + expect( + sourceDbChanges.hasChanges, + "there should be only custom changes" + ).to.be.false; + await sourceDbChanges.addCustomElementChange( + "Updated", + physicalElem2.id! + ); + }); + await transformer.process(); + await pushChanges(targetDb, "Transformation 2: update other element"); + + let physicalElem1InTarget = targetDb.elements.tryGetElement( + physicalElem1.federationGuid! + ); + expect(physicalElem1InTarget).to.not.be.undefined; + expect(physicalElem1InTarget!.userLabel).to.be.equal("Updated"); + + // === Transformation 3: `process changes` transformation to update changed element === + transformer = new CustomChangesTransformer(sourceDb, targetDb, true); + sinon + .stub(transformer, "shouldExportElement") + .callsFake((_sourceElement) => true); + sinon + .stub(transformer, "addCustomChanges") + .callsFake(async (sourceDbChanges) => { + expect( + sourceDbChanges.hasChanges, + "there should be only custom changes" + ).to.be.false; + await sourceDbChanges.addCustomElementChange( + "Updated", + physicalElem1.id! + ); + }); + await transformer.process(); + await pushChanges(targetDb, "Transformation 2: update changed element"); + + physicalElem1InTarget = targetDb.elements.tryGetElement( + physicalElem1.federationGuid! + ); + expect(physicalElem1InTarget).to.not.be.undefined; + expect( + physicalElem1InTarget!.userLabel, + "updated value should be reverted" + ).to.be.equal("PhysicalOne"); + }); + + it("should delete recreated model when custom delete change is registered for it", async () => { + const constSubjectFedGuid = Guid.createValue(); + const originalSubjectId = sourceDb.elements.insertElement({ + classFullName: Subject.classFullName, + code: Code.createEmpty(), + model: IModel.repositoryModelId, + parent: new SubjectOwnsSubjects(IModel.rootSubjectId), + federationGuid: constSubjectFedGuid, + userLabel: "A", + }); + + const constPartitionFedGuid = Guid.createValue(); + const originalPartitionId = sourceDb.elements.insertElement({ + model: IModel.repositoryModelId, + code: PhysicalPartition.createCode( + sourceDb, + IModel.rootSubjectId, + "original partition" + ), + classFullName: PhysicalPartition.classFullName, + federationGuid: constPartitionFedGuid, + parent: new SubjectOwnsPartitionElements(IModel.rootSubjectId), + }); + const originalModelId = sourceDb.models.insertModel({ + classFullName: PhysicalModel.classFullName, + modeledElement: { id: originalPartitionId }, + isPrivate: true, + }); + + await pushChanges(sourceDb, "Initial changes"); + + // === Transformation 1: Run `process all` transformation === + let transformer = new CustomChangesTransformer(sourceDb, targetDb, false); + await transformer.process(); + transformer.updateSynchronizationVersion({ + initializeReverseSyncVersion: true, + }); + await pushChanges(targetDb, "Transformation 1: Process All"); + + // Assert + expect(targetDb.elements.tryGetElement(constSubjectFedGuid)).to.not.be + .undefined; + expect(targetDb.elements.tryGetElement(constPartitionFedGuid)).to.not.be + .undefined; + expect( + IModelTestUtils.count(targetDb, PhysicalModel.classFullName) + ).to.be.equal(1); + assertModelExistsByName(targetDb, ["original partition"]); + + // === Transformation 1: Run `process all` transformation === + sourceDb.elements.deleteElement(originalSubjectId); + const secondCopyOfSubjectId = sourceDb.elements.insertElement({ + classFullName: Subject.classFullName, + code: Code.createEmpty(), + model: IModel.repositoryModelId, + parent: new SubjectOwnsSubjects(IModel.rootSubjectId), + federationGuid: constSubjectFedGuid, + userLabel: "B", + }); + + sourceDb.models.deleteModel(originalModelId); + sourceDb.elements.deleteElement(originalPartitionId); + const recreatedPartitionId = sourceDb.elements.insertElement({ + model: IModel.repositoryModelId, + code: PhysicalPartition.createCode( + sourceDb, + IModel.rootSubjectId, + "recreated partition" + ), + classFullName: PhysicalPartition.classFullName, + federationGuid: constPartitionFedGuid, + parent: new SubjectOwnsPartitionElements(IModel.rootSubjectId), + }); + sourceDb.models.insertModel({ + classFullName: PhysicalModel.classFullName, + modeledElement: { id: recreatedPartitionId }, + isPrivate: false, + }); + + await pushChanges(sourceDb, "Recreated elements"); + + transformer = new CustomChangesTransformer(sourceDb, targetDb, true); + sinon + .stub(transformer, "addCustomChanges") + .callsFake(async (sourceDbChanges) => { + await sourceDbChanges.addCustomModelChange( + "Deleted", + recreatedPartitionId + ); + await sourceDbChanges.addCustomElementChange( + "Deleted", + secondCopyOfSubjectId + ); + }); + await transformer.process(); + await pushChanges( + targetDb, + "Transformation 2: inserted previously excluded model" + ); + expect(targetDb.elements.tryGetElement(constSubjectFedGuid)).to.be + .undefined; + expect(targetDb.elements.tryGetElement(constPartitionFedGuid)).to.be + .undefined; + expect( + IModelTestUtils.count(targetDb, PhysicalModel.classFullName) + ).to.be.equal(0); + }); + + function insertDrawingElement( + iModel: IModelDb, + documentListModelId: Id64String, + drawingName: string + ): ElementProps { + const id = Drawing.insert(iModel, documentListModelId, drawingName); + return iModel.elements.getElementProps(id); + } + + function insertPhysicalElement( + iModel: IModelDb, + modelId: Id64String, + categoryId: Id64String, + uniqueName: string + ): ElementProps { + const code = new Code({ scope: "0x1", spec: "0x1", value: uniqueName }); + const element: PhysicalElementProps = { + classFullName: PhysicalObject.classFullName, + model: modelId, + category: categoryId, + code, + userLabel: uniqueName, + }; + + iModel.elements.insertElement(element); + // re-read element to populate federationGuid value + return iModel.elements.getElementProps(element.id!); + } + + function insertElementAspect( + iModel: IModelDb, + scopeId: Id64String, + elementId: Id64String, + identifier: string + ): Id64String { + const aspectProps: ExternalSourceAspectProps = { + classFullName: ExternalSourceAspect.classFullName, + kind: "something", + scope: { id: scopeId }, + element: { + id: elementId, + relClassName: ElementOwnsExternalSourceAspects.classFullName, + }, + identifier, + }; + + return iModel.elements.insertAspect(aspectProps); + } + + function insertElementGroupsElementsRelationship( + iModel: IModelDb, + sourceId: Id64String, + targetId: Id64String + ) { + const rel = ElementGroupsMembers.create(iModel, sourceId, targetId, 0); + const id = rel.insert(); + return iModel.relationships.getInstance( + ElementGroupsMembers.classFullName, + id + ); + } + + function assertElementsExistByCode( + iModel: IModelDb, + properties: ElementProps[] + ) { + properties.forEach((elemProp) => { + expect(elemProp.code.value).to.not.be.undefined; + expect( + IModelTestUtils.queryByCodeValue(iModel, elemProp.code.value!), + `Element '${elemProp.code.value}' should exist in iModel.` + ).to.not.be.equal(Id64.invalid); + }); + } + + function assertModelExistsByName(iModel: IModelDb, names: string[]) { + names.forEach((name) => { + expect( + IModelTestUtils.queryModelIddByModeledElementCodeValue(iModel, name), + `Model '${name}' should exist in iModel.` + ).to.not.be.equal(Id64.invalid); + }); + } + + function assertModelDoesNotExistsByName(iModel: IModelDb, names: string[]) { + names.forEach((name) => { + expect( + IModelTestUtils.queryModelIddByModeledElementCodeValue(iModel, name), + `Model '${name}' should not exist in iModel.` + ).to.be.equal(Id64.invalid); + }); + } + + function assertElementsDoNotExistByCode( + iModel: IModelDb, + properties: ElementProps[] + ) { + properties.forEach((elemProp) => { + expect(elemProp.code.value).to.not.be.undefined; + expect( + IModelTestUtils.queryByCodeValue(iModel, elemProp.code.value!), + `Element '${elemProp.code.value}' should not exist in iModel.` + ).to.be.equal(Id64.invalid); + }); + } + + function assertElementHasExpectedAspectCount( + iModel: IModelDb, + federationGuid: GuidString, + expectedAspectCount: number + ) { + const element = iModel.elements.tryGetElement(federationGuid); + expect( + element, + `Could not locate element with federationGuid: ${federationGuid}` + ).to.not.be.undefined; + expect(iModel.elements.getAspects(element!.id).length).to.be.equal( + expectedAspectCount, + "Aspect count is different than expected." + ); + } + }); });