From 192a7db8faca5894780bb846a37989163a2a64af Mon Sep 17 00:00:00 2001 From: "nick.tessier" <22119573+nick4598@users.noreply.github.com> Date: Tue, 7 Jan 2025 11:58:41 -0500 Subject: [PATCH 1/7] wip test is failing --- .../IModelBranchingOperations.test.ts | 302 ++++++++++++++++++ 1 file changed, 302 insertions(+) create mode 100644 packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts diff --git a/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts b/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts new file mode 100644 index 000000000..0001046de --- /dev/null +++ b/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts @@ -0,0 +1,302 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "fs"; +import * as path from "path"; +import * as TestUtils from "../TestUtils"; +import { + BriefcaseDb, + BriefcaseManager, + ExternalSource, + ExternalSourceIsInRepository, + HubMock, + IModelDb, + IModelHost, + PhysicalModel, + PhysicalObject, + PhysicalPartition, + RepositoryLink, + SnapshotDb, + SpatialCategory, +} from "@itwin/core-backend"; +import { + HubWrappers, + IModelTransformerTestUtils, +} from "../IModelTransformerUtils"; +import { AccessToken } from "@itwin/core-bentley"; +import { + Code, + ExternalSourceProps, + IModel, + PhysicalElementProps, + RepositoryLinkProps, + SubCategoryAppearance, +} from "@itwin/core-common"; +import { Point3d, YawPitchRollAngles } from "@itwin/core-geometry"; +import { IModelTransformer } from "../../IModelTransformer"; +process.env.TRANSFORMER_NO_STRICT_DEP_CHECK = "1"; // allow this monorepo's dev versions of core libs in transformer + +// some json will be required later, but we don't want an eslint-disable line in the example code, so just disable for the file +/* eslint-disable @typescript-eslint/no-require-imports */ + +async function initializeBranch( + myITwinId: string, + masterIModelId: string, + myAccessToken: AccessToken +) { + // __PUBLISH_EXTRACT_START__ IModelBranchingOperations_initialize + // download and open master + const masterDbProps = await BriefcaseManager.downloadBriefcase({ + accessToken: myAccessToken, + iTwinId: myITwinId, + iModelId: masterIModelId, + }); + const masterDb = await BriefcaseDb.open({ fileName: masterDbProps.fileName }); + + // create a duplicate of master as a good starting point for our branch + const branchIModelId = await IModelHost.hubAccess.createNewIModel({ + iTwinId: myITwinId, + iModelName: "my-branch-imodel", + version0: masterDb.pathName, + noLocks: true, // you may prefer locks for your application + }); + + // download and open the new branch + const branchDbProps = await BriefcaseManager.downloadBriefcase({ + accessToken: myAccessToken, + iTwinId: myITwinId, + iModelId: branchIModelId, + }); + const branchDb = await BriefcaseDb.open({ fileName: branchDbProps.fileName }); + + // create an external source and owning repository link to use as our *Target Scope Element* for future synchronizations + const masterLinkRepoId = branchDb + .constructEntity({ + classFullName: RepositoryLink.classFullName, + code: RepositoryLink.createCode( + branchDb, + IModelDb.repositoryModelId, + "example-code-value" + ), + model: IModelDb.repositoryModelId, + url: "https://wherever-you-got-your-imodel.net", + format: "iModel", + repositoryGuid: masterDb.iModelId, + description: "master iModel repository", + }) + .insert(); + + const masterExternalSourceId = branchDb + .constructEntity({ + classFullName: ExternalSource.classFullName, + model: IModelDb.rootSubjectId, + code: Code.createEmpty(), + repository: new ExternalSourceIsInRepository(masterLinkRepoId), + connectorName: "iModel Transformer", + connectorVersion: require("@itwin/imodel-transformer/package.json") + .version, + }) + .insert(); + + // initialize the branch provenance + const branchInitializer = new IModelTransformer(masterDb, branchDb, { + // tells the transformer that we have a raw copy of a source and the target should receive + // provenance from the source that is necessary for performing synchronizations in the future + wasSourceIModelCopiedToTarget: true, + // store the synchronization provenance in the scope of our representation of the external source, master + targetScopeElementId: masterExternalSourceId, + }); + await branchInitializer.process(); + branchInitializer.dispose(); + + // save+push our changes to whatever hub we're using + const description = "initialized branch iModel"; + branchDb.saveChanges(description); + await branchDb.pushChanges({ + accessToken: myAccessToken, + description, + }); + // __PUBLISH_EXTRACT_END__ + + return { masterDb, branchDb }; +} + +// we assume masterDb and branchDb have already been opened (see the first example) +async function forwardSyncMasterToBranch( + masterDb: BriefcaseDb, + branchDb: BriefcaseDb, + myAccessToken: AccessToken +) { + // __PUBLISH_EXTRACT_START__ IModelBranchingOperations_forwardSync + const masterExternalSourceId = branchDb.elements.queryElementIdByCode( + RepositoryLink.createCode( + masterDb, + IModelDb.repositoryModelId, + "example-code-value" + ) + ); + const synchronizer = new IModelTransformer(masterDb, branchDb, { + // read the synchronization provenance in the scope of our representation of the external source, master + targetScopeElementId: masterExternalSourceId, + // Presence of argsForProcessChanges even if empty is required to have process run 'processChanges' internally. + argsForProcessChanges: {}, + }); + + await synchronizer.process(); + synchronizer.dispose(); + // save and push + const description = "updated branch with recent master changes"; + branchDb.saveChanges(description); + await branchDb.pushChanges({ + accessToken: myAccessToken, + description, + }); + // __PUBLISH_EXTRACT_END__ +} + +async function reverseSyncBranchToMaster( + branchDb: BriefcaseDb, + masterDb: BriefcaseDb, + myAccessToken: AccessToken +) { + // __PUBLISH_EXTRACT_START__ IModelBranchingOperations_reverseSync + // we assume masterDb and branchDb have already been opened (see the first example) + const masterExternalSourceId = branchDb.elements.queryElementIdByCode( + RepositoryLink.createCode( + masterDb, + IModelDb.repositoryModelId, + "example-code-value" + ) + ); + const reverseSynchronizer = new IModelTransformer(branchDb, masterDb, { + // read the synchronization provenance in the scope of our representation of the external source, master + // "isReverseSynchronization" actually causes the provenance (and therefore the targetScopeElementId) to + // be searched for from the source + targetScopeElementId: masterExternalSourceId, + // Presence of argsForProcessChanges even if empty is required to have process run 'processChanges' internally. + argsForProcessChanges: {}, + }); + + await reverseSynchronizer.process(); + reverseSynchronizer.dispose(); + // save and push + const description = "merged changes from branch into master"; + masterDb.saveChanges(description); + await masterDb.pushChanges({ + accessToken: myAccessToken, + description, + }); + // __PUBLISH_EXTRACT_END__ +} + +async function arbitraryEdit( + db: BriefcaseDb, + myAccessToken: AccessToken, + description: string +) { + const spatialCategoryCode = SpatialCategory.createCode( + db, + IModel.dictionaryId, + "SpatialCategory1" + ); + const physicalModelCode = PhysicalPartition.createCode( + db, + IModel.rootSubjectId, + "PhysicalModel1" + ); + let spatialCategoryId = db.elements.queryElementIdByCode(spatialCategoryCode); + let physicalModelId = db.elements.queryElementIdByCode(physicalModelCode); + if (physicalModelId === undefined || spatialCategoryId === undefined) { + spatialCategoryId = SpatialCategory.insert( + db, + IModel.dictionaryId, + "SpatialCategory1", + new SubCategoryAppearance() + ); + physicalModelId = PhysicalModel.insert( + db, + IModel.rootSubjectId, + "PhysicalModel1" + ); + } + const physicalObjectProps: PhysicalElementProps = { + classFullName: PhysicalObject.classFullName, + model: physicalModelId, + category: spatialCategoryId, + code: new Code({ + spec: IModelDb.rootSubjectId, + scope: IModelDb.rootSubjectId, + value: `${arbitraryEdit.editCounter}`, + }), + userLabel: `${arbitraryEdit.editCounter}`, + geom: IModelTransformerTestUtils.createBox(Point3d.create(1, 1, 1)), + placement: { + origin: Point3d.create( + arbitraryEdit.editCounter, + arbitraryEdit.editCounter, + 0 + ), + angles: YawPitchRollAngles.createDegrees(0, 0, 0), + }, + }; + arbitraryEdit.editCounter++; + db.elements.insertElement(physicalObjectProps); + db.saveChanges(); + await db.pushChanges({ + accessToken: myAccessToken, + description, + }); +} + +namespace arbitraryEdit { + // eslint-disable-next-line prefer-const + export let editCounter = 0; +} + +describe.only("IModelBranchingOperations", () => { + const version0Path = path.join( + TestUtils.KnownTestLocations.outputDir, + "branching-ops.bim" + ); + + before(async () => { + HubMock.startup( + "IModelBranchingOperations", + TestUtils.KnownTestLocations.outputDir + ); + if (fs.existsSync(version0Path)) fs.unlinkSync(version0Path); + SnapshotDb.createEmpty(version0Path, { + rootSubject: { name: "branching-ops" }, + }).close(); + }); + + after(() => { + HubMock.shutdown(); + }); + + it("run branching operations", async () => { + const myAccessToken = await HubWrappers.getAccessToken( + TestUtils.TestUserType.Regular + ); + const myITwinId = HubMock.iTwinId; + const masterIModelId = await IModelHost.hubAccess.createNewIModel({ + iTwinId: myITwinId, + iModelName: "my-branch-imodel", + version0: version0Path, + noLocks: true, + }); + const { masterDb, branchDb } = await initializeBranch( + myITwinId, + masterIModelId, + myAccessToken + ); + await arbitraryEdit(masterDb, myAccessToken, "edit master"); + await forwardSyncMasterToBranch(masterDb, branchDb, myAccessToken); + await arbitraryEdit(branchDb, myAccessToken, "edit branch"); + await reverseSyncBranchToMaster(branchDb, masterDb, myAccessToken); + masterDb.close(); + branchDb.close(); + }); +}); From 03c6e8bd1a7f41f7efe46f1ebccb6679445db701 Mon Sep 17 00:00:00 2001 From: "nick.tessier" <22119573+nick4598@users.noreply.github.com> Date: Tue, 7 Jan 2025 12:46:58 -0500 Subject: [PATCH 2/7] test is passing now --- .../IModelBranchingOperations.test.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts b/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts index 0001046de..6c6b69224 100644 --- a/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts +++ b/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts @@ -31,6 +31,7 @@ import { ExternalSourceProps, IModel, PhysicalElementProps, + QueryBinder, RepositoryLinkProps, SubCategoryAppearance, } from "@itwin/core-common"; @@ -130,13 +131,20 @@ async function forwardSyncMasterToBranch( myAccessToken: AccessToken ) { // __PUBLISH_EXTRACT_START__ IModelBranchingOperations_forwardSync - const masterExternalSourceId = branchDb.elements.queryElementIdByCode( + const repositoryLinkId = branchDb.elements.queryElementIdByCode( RepositoryLink.createCode( masterDb, IModelDb.repositoryModelId, "example-code-value" ) ); + let masterExternalSourceId; + for await (const row of branchDb.createQueryReader( + `SELECT ECInstanceId FROM ${ExternalSource.classFullName} WHERE Repository.Id=:id`, + QueryBinder.from({ id: repositoryLinkId }) + )) { + masterExternalSourceId = row.ECInstanceId; + } const synchronizer = new IModelTransformer(masterDb, branchDb, { // read the synchronization provenance in the scope of our representation of the external source, master targetScopeElementId: masterExternalSourceId, @@ -163,13 +171,20 @@ async function reverseSyncBranchToMaster( ) { // __PUBLISH_EXTRACT_START__ IModelBranchingOperations_reverseSync // we assume masterDb and branchDb have already been opened (see the first example) - const masterExternalSourceId = branchDb.elements.queryElementIdByCode( + const repositoryLinkId = branchDb.elements.queryElementIdByCode( RepositoryLink.createCode( masterDb, IModelDb.repositoryModelId, "example-code-value" ) ); + let masterExternalSourceId; + for await (const row of branchDb.createQueryReader( + `SELECT ECInstanceId FROM ${ExternalSource.classFullName} WHERE Repository.Id=:id`, + QueryBinder.from({ id: repositoryLinkId }) + )) { + masterExternalSourceId = row.ECInstanceId; + } const reverseSynchronizer = new IModelTransformer(branchDb, masterDb, { // read the synchronization provenance in the scope of our representation of the external source, master // "isReverseSynchronization" actually causes the provenance (and therefore the targetScopeElementId) to From 6115919b547cca885a9eb10f9d069d9586c5e6d2 Mon Sep 17 00:00:00 2001 From: "nick.tessier" <22119573+nick4598@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:41:46 -0500 Subject: [PATCH 3/7] small changes to docs --- docs/learning/transformer/branching-imodels.md | 4 ++-- docs/learning/transformer/index.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/learning/transformer/branching-imodels.md b/docs/learning/transformer/branching-imodels.md index cce069087..c8c58a552 100644 --- a/docs/learning/transformer/branching-imodels.md +++ b/docs/learning/transformer/branching-imodels.md @@ -90,7 +90,7 @@ forward synchronizations. Conflicts during a transformation are resolved in favor of the element which was modified most recently, as stored in the `LastMod` property of an element. Elements in transformations are considered in conflict when their [code](/bis/fundamentals/foundation/codes) is the same. -You can override the method [`IModelTransformer.hasElementChanged`](/reference/core-transformer/imodels/imodeltransformer/haselementchanged/) +You can override the method [`IModelTransformer.hasElementChanged`](/reference/imodel-transformer/imodels/imodeltransformer/haselementchanged/) in your transformer implementation to use more specific logic for determining if an element should be considered changed. Some other data in the iModel follows more specific rules for conflicts: @@ -124,4 +124,4 @@ Synchronization conflicts are not to be confused with concurrent edit conflicts ### Synchronization workflow examples -More in depth samples exist in the [tests](https://github.com/iTwin/itwinjs-core/blob/master/core/transformer/src/test/standalone/IModelTransformerHub.test.ts) for the `@itwin/core-transformer` package. +More in depth samples exist in the [tests](https://github.com/iTwin/imodel-transformer/blob/main/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts) for the `@itwin/imodel-transformer` package. diff --git a/docs/learning/transformer/index.md b/docs/learning/transformer/index.md index 4cb8e04d8..4fe829358 100644 --- a/docs/learning/transformer/index.md +++ b/docs/learning/transformer/index.md @@ -1,6 +1,6 @@ # iModel Transformation and Data Exchange -The `@itwin/core-transformer` package provides some classes that implement [Extract, Transform, and Load](https://en.wikipedia.org/wiki/Extract,_transform,_load) (ETL) functionality: +The `@itwin/imodel-transformer` package provides some classes that implement [Extract, Transform, and Load](https://en.wikipedia.org/wiki/Extract,_transform,_load) (ETL) functionality: - [IModelExporter]($transformer) and [IModelExportHandler]($transformer) are the base classes that implement the *extract* (or *export*) part of ETL functionality. - [IModelTransformer]($transformer) is the base class that implements the *transform* part of ETL functionality. From d5c0e8b36a00ea5ee51c88b0e9443e42e4d6fe0d Mon Sep 17 00:00:00 2001 From: "nick.tessier" <22119573+nick4598@users.noreply.github.com> Date: Tue, 7 Jan 2025 15:01:39 -0500 Subject: [PATCH 4/7] copy learning folder over to build folder (docs output) --- packages/transformer/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/transformer/package.json b/packages/transformer/package.json index ff7a5f4c8..373e4ff14 100644 --- a/packages/transformer/package.json +++ b/packages/transformer/package.json @@ -13,8 +13,9 @@ "build:ci": "npm run -s build", "build:cjs": "tsc 1>&2 --outDir lib/cjs", "clean": "rimraf lib", - "docs": "npm run -s docs:extract && npm run -s docs:reference && npm run -s docs:changelog", + "docs": "npm run -s docs:extract && npm run -s docs:reference && npm run -s docs:changelog && npm run -s docs:copyLearning", "docs:changelog": "cpx ./CHANGELOG.md ../../build/docs/reference/imodel-transformer", + "docs:copyLearning": "cpx \"../../docs/learning/transformer/**/*\" ../../build/docs/learning/transformer", "# env var is workaround, need to contribute a better rush-less root-package.json detector to betools": "", "docs:reference": "cross-env RUSHSTACK_FILE_ERROR_BASE_FOLDER='../..' betools docs --includes=../../build/docs/extract --json=../../build/docs/reference/imodel-transformer/file.json --tsIndexFile=imodel-transformer.ts --onlyJson", "docs:extract": "betools extract --fileExt=ts --extractFrom=./src/test --recursive --out=../../build/docs/extract", From ed02cd5bca818abae0dd50c9935a67a9518d3e60 Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Date: Wed, 3 Dec 2025 15:02:49 -0500 Subject: [PATCH 5/7] Change files --- ...l-transformer-1298e016-279a-4f30-9552-3ce40e14203e.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 change/@itwin-imodel-transformer-1298e016-279a-4f30-9552-3ce40e14203e.json diff --git a/change/@itwin-imodel-transformer-1298e016-279a-4f30-9552-3ce40e14203e.json b/change/@itwin-imodel-transformer-1298e016-279a-4f30-9552-3ce40e14203e.json new file mode 100644 index 000000000..b203187ea --- /dev/null +++ b/change/@itwin-imodel-transformer-1298e016-279a-4f30-9552-3ce40e14203e.json @@ -0,0 +1,7 @@ +{ + "type": "patch", + "comment": "copy learning folder over to build folder (docs output)", + "packageName": "@itwin/imodel-transformer", + "email": "'DanRod1999@users.noreply.github.com'", + "dependentChangeType": "patch" +} From b28156e50a24ce4420a3a3b33fe7d40cbbe44a9c Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Date: Wed, 3 Dec 2025 15:18:56 -0500 Subject: [PATCH 6/7] lint --- .../src/test/example-code/IModelBranchingOperations.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts b/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts index 6c6b69224..19abb9b15 100644 --- a/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts +++ b/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts @@ -96,6 +96,7 @@ async function initializeBranch( code: Code.createEmpty(), repository: new ExternalSourceIsInRepository(masterLinkRepoId), connectorName: "iModel Transformer", + // eslint-disable-next-line @typescript-eslint/no-var-requires connectorVersion: require("@itwin/imodel-transformer/package.json") .version, }) From eb500c39c622c17b7131e97ededbf62b73b8dcf3 Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Date: Thu, 4 Dec 2025 13:32:31 -0500 Subject: [PATCH 7/7] update url path for IModelCloneContext --- docs/learning/transformer/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/learning/transformer/index.md b/docs/learning/transformer/index.md index 4fe829358..21cb8ab1f 100644 --- a/docs/learning/transformer/index.md +++ b/docs/learning/transformer/index.md @@ -71,7 +71,7 @@ Either of the non-false autoExtendProjectExtents options are useful for consolid ## IModelCloneContext -The [IModelCloneContext]($backend) class provides the core *cloning* capability required for iModel transformation. +The [IModelCloneContext]($transformer) class provides the core *cloning* capability required for iModel transformation. It also maintains the **sourceId --> targetId** mapping which is required to successfully *clone* [Entity]($backend) instances from the source iModel into the target iModel. iModel entities are highly related to each other. Therefore, *cloning* an entity means copying a *graph* of objects and remapping their source references (Ids) to other target entities.