diff --git a/change/@itwin-imodel-transformer-1298e016-279a-4f30-9552-3ce40e14203e.json b/change/@itwin-imodel-transformer-1298e016-279a-4f30-9552-3ce40e14203e.json new file mode 100644 index 000000000..b203187ea --- /dev/null +++ b/change/@itwin-imodel-transformer-1298e016-279a-4f30-9552-3ce40e14203e.json @@ -0,0 +1,7 @@ +{ + "type": "patch", + "comment": "copy learning folder over to build folder (docs output)", + "packageName": "@itwin/imodel-transformer", + "email": "'DanRod1999@users.noreply.github.com'", + "dependentChangeType": "patch" +} diff --git a/docs/learning/transformer/branching-imodels.md b/docs/learning/transformer/branching-imodels.md index cce069087..c8c58a552 100644 --- a/docs/learning/transformer/branching-imodels.md +++ b/docs/learning/transformer/branching-imodels.md @@ -90,7 +90,7 @@ forward synchronizations. Conflicts during a transformation are resolved in favor of the element which was modified most recently, as stored in the `LastMod` property of an element. Elements in transformations are considered in conflict when their [code](/bis/fundamentals/foundation/codes) is the same. -You can override the method [`IModelTransformer.hasElementChanged`](/reference/core-transformer/imodels/imodeltransformer/haselementchanged/) +You can override the method [`IModelTransformer.hasElementChanged`](/reference/imodel-transformer/imodels/imodeltransformer/haselementchanged/) in your transformer implementation to use more specific logic for determining if an element should be considered changed. Some other data in the iModel follows more specific rules for conflicts: @@ -124,4 +124,4 @@ Synchronization conflicts are not to be confused with concurrent edit conflicts ### Synchronization workflow examples -More in depth samples exist in the [tests](https://github.com/iTwin/itwinjs-core/blob/master/core/transformer/src/test/standalone/IModelTransformerHub.test.ts) for the `@itwin/core-transformer` package. +More in depth samples exist in the [tests](https://github.com/iTwin/imodel-transformer/blob/main/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts) for the `@itwin/imodel-transformer` package. diff --git a/docs/learning/transformer/index.md b/docs/learning/transformer/index.md index 4cb8e04d8..21cb8ab1f 100644 --- a/docs/learning/transformer/index.md +++ b/docs/learning/transformer/index.md @@ -1,6 +1,6 @@ # iModel Transformation and Data Exchange -The `@itwin/core-transformer` package provides some classes that implement [Extract, Transform, and Load](https://en.wikipedia.org/wiki/Extract,_transform,_load) (ETL) functionality: +The `@itwin/imodel-transformer` package provides some classes that implement [Extract, Transform, and Load](https://en.wikipedia.org/wiki/Extract,_transform,_load) (ETL) functionality: - [IModelExporter]($transformer) and [IModelExportHandler]($transformer) are the base classes that implement the *extract* (or *export*) part of ETL functionality. - [IModelTransformer]($transformer) is the base class that implements the *transform* part of ETL functionality. @@ -71,7 +71,7 @@ Either of the non-false autoExtendProjectExtents options are useful for consolid ## IModelCloneContext -The [IModelCloneContext]($backend) class provides the core *cloning* capability required for iModel transformation. +The [IModelCloneContext]($transformer) class provides the core *cloning* capability required for iModel transformation. It also maintains the **sourceId --> targetId** mapping which is required to successfully *clone* [Entity]($backend) instances from the source iModel into the target iModel. iModel entities are highly related to each other. Therefore, *cloning* an entity means copying a *graph* of objects and remapping their source references (Ids) to other target entities. diff --git a/packages/transformer/package.json b/packages/transformer/package.json index fac9d27e0..95e0a4ea3 100644 --- a/packages/transformer/package.json +++ b/packages/transformer/package.json @@ -13,8 +13,9 @@ "build:ci": "npm run -s build", "build:cjs": "tsc 1>&2 --outDir lib/cjs", "clean": "rimraf lib", - "docs": "npm run -s docs:extract && npm run -s docs:reference && npm run -s docs:changelog", + "docs": "npm run -s docs:extract && npm run -s docs:reference && npm run -s docs:changelog && npm run -s docs:copyLearning", "docs:changelog": "cpx ./CHANGELOG.md ../../build/docs/reference/imodel-transformer", + "docs:copyLearning": "cpx \"../../docs/learning/transformer/**/*\" ../../build/docs/learning/transformer", "# env var is workaround, need to contribute a better rush-less root-package.json detector to betools": "", "docs:reference": "cross-env RUSHSTACK_FILE_ERROR_BASE_FOLDER='../..' betools docs --includes=../../build/docs/extract --json=../../build/docs/reference/imodel-transformer/file.json --tsIndexFile=imodel-transformer.ts --onlyJson", "docs:extract": "betools extract --fileExt=ts --extractFrom=./src/test --recursive --out=../../build/docs/extract", diff --git a/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts b/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts new file mode 100644 index 000000000..19abb9b15 --- /dev/null +++ b/packages/transformer/src/test/example-code/IModelBranchingOperations.test.ts @@ -0,0 +1,318 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Bentley Systems, Incorporated. All rights reserved. + * See LICENSE.md in the project root for license terms and full copyright notice. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from "fs"; +import * as path from "path"; +import * as TestUtils from "../TestUtils"; +import { + BriefcaseDb, + BriefcaseManager, + ExternalSource, + ExternalSourceIsInRepository, + HubMock, + IModelDb, + IModelHost, + PhysicalModel, + PhysicalObject, + PhysicalPartition, + RepositoryLink, + SnapshotDb, + SpatialCategory, +} from "@itwin/core-backend"; +import { + HubWrappers, + IModelTransformerTestUtils, +} from "../IModelTransformerUtils"; +import { AccessToken } from "@itwin/core-bentley"; +import { + Code, + ExternalSourceProps, + IModel, + PhysicalElementProps, + QueryBinder, + RepositoryLinkProps, + SubCategoryAppearance, +} from "@itwin/core-common"; +import { Point3d, YawPitchRollAngles } from "@itwin/core-geometry"; +import { IModelTransformer } from "../../IModelTransformer"; +process.env.TRANSFORMER_NO_STRICT_DEP_CHECK = "1"; // allow this monorepo's dev versions of core libs in transformer + +// some json will be required later, but we don't want an eslint-disable line in the example code, so just disable for the file +/* eslint-disable @typescript-eslint/no-require-imports */ + +async function initializeBranch( + myITwinId: string, + masterIModelId: string, + myAccessToken: AccessToken +) { + // __PUBLISH_EXTRACT_START__ IModelBranchingOperations_initialize + // download and open master + const masterDbProps = await BriefcaseManager.downloadBriefcase({ + accessToken: myAccessToken, + iTwinId: myITwinId, + iModelId: masterIModelId, + }); + const masterDb = await BriefcaseDb.open({ fileName: masterDbProps.fileName }); + + // create a duplicate of master as a good starting point for our branch + const branchIModelId = await IModelHost.hubAccess.createNewIModel({ + iTwinId: myITwinId, + iModelName: "my-branch-imodel", + version0: masterDb.pathName, + noLocks: true, // you may prefer locks for your application + }); + + // download and open the new branch + const branchDbProps = await BriefcaseManager.downloadBriefcase({ + accessToken: myAccessToken, + iTwinId: myITwinId, + iModelId: branchIModelId, + }); + const branchDb = await BriefcaseDb.open({ fileName: branchDbProps.fileName }); + + // create an external source and owning repository link to use as our *Target Scope Element* for future synchronizations + const masterLinkRepoId = branchDb + .constructEntity({ + classFullName: RepositoryLink.classFullName, + code: RepositoryLink.createCode( + branchDb, + IModelDb.repositoryModelId, + "example-code-value" + ), + model: IModelDb.repositoryModelId, + url: "https://wherever-you-got-your-imodel.net", + format: "iModel", + repositoryGuid: masterDb.iModelId, + description: "master iModel repository", + }) + .insert(); + + const masterExternalSourceId = branchDb + .constructEntity({ + classFullName: ExternalSource.classFullName, + model: IModelDb.rootSubjectId, + code: Code.createEmpty(), + repository: new ExternalSourceIsInRepository(masterLinkRepoId), + connectorName: "iModel Transformer", + // eslint-disable-next-line @typescript-eslint/no-var-requires + connectorVersion: require("@itwin/imodel-transformer/package.json") + .version, + }) + .insert(); + + // initialize the branch provenance + const branchInitializer = new IModelTransformer(masterDb, branchDb, { + // tells the transformer that we have a raw copy of a source and the target should receive + // provenance from the source that is necessary for performing synchronizations in the future + wasSourceIModelCopiedToTarget: true, + // store the synchronization provenance in the scope of our representation of the external source, master + targetScopeElementId: masterExternalSourceId, + }); + await branchInitializer.process(); + branchInitializer.dispose(); + + // save+push our changes to whatever hub we're using + const description = "initialized branch iModel"; + branchDb.saveChanges(description); + await branchDb.pushChanges({ + accessToken: myAccessToken, + description, + }); + // __PUBLISH_EXTRACT_END__ + + return { masterDb, branchDb }; +} + +// we assume masterDb and branchDb have already been opened (see the first example) +async function forwardSyncMasterToBranch( + masterDb: BriefcaseDb, + branchDb: BriefcaseDb, + myAccessToken: AccessToken +) { + // __PUBLISH_EXTRACT_START__ IModelBranchingOperations_forwardSync + const repositoryLinkId = branchDb.elements.queryElementIdByCode( + RepositoryLink.createCode( + masterDb, + IModelDb.repositoryModelId, + "example-code-value" + ) + ); + let masterExternalSourceId; + for await (const row of branchDb.createQueryReader( + `SELECT ECInstanceId FROM ${ExternalSource.classFullName} WHERE Repository.Id=:id`, + QueryBinder.from({ id: repositoryLinkId }) + )) { + masterExternalSourceId = row.ECInstanceId; + } + const synchronizer = new IModelTransformer(masterDb, branchDb, { + // read the synchronization provenance in the scope of our representation of the external source, master + targetScopeElementId: masterExternalSourceId, + // Presence of argsForProcessChanges even if empty is required to have process run 'processChanges' internally. + argsForProcessChanges: {}, + }); + + await synchronizer.process(); + synchronizer.dispose(); + // save and push + const description = "updated branch with recent master changes"; + branchDb.saveChanges(description); + await branchDb.pushChanges({ + accessToken: myAccessToken, + description, + }); + // __PUBLISH_EXTRACT_END__ +} + +async function reverseSyncBranchToMaster( + branchDb: BriefcaseDb, + masterDb: BriefcaseDb, + myAccessToken: AccessToken +) { + // __PUBLISH_EXTRACT_START__ IModelBranchingOperations_reverseSync + // we assume masterDb and branchDb have already been opened (see the first example) + const repositoryLinkId = branchDb.elements.queryElementIdByCode( + RepositoryLink.createCode( + masterDb, + IModelDb.repositoryModelId, + "example-code-value" + ) + ); + let masterExternalSourceId; + for await (const row of branchDb.createQueryReader( + `SELECT ECInstanceId FROM ${ExternalSource.classFullName} WHERE Repository.Id=:id`, + QueryBinder.from({ id: repositoryLinkId }) + )) { + masterExternalSourceId = row.ECInstanceId; + } + const reverseSynchronizer = new IModelTransformer(branchDb, masterDb, { + // read the synchronization provenance in the scope of our representation of the external source, master + // "isReverseSynchronization" actually causes the provenance (and therefore the targetScopeElementId) to + // be searched for from the source + targetScopeElementId: masterExternalSourceId, + // Presence of argsForProcessChanges even if empty is required to have process run 'processChanges' internally. + argsForProcessChanges: {}, + }); + + await reverseSynchronizer.process(); + reverseSynchronizer.dispose(); + // save and push + const description = "merged changes from branch into master"; + masterDb.saveChanges(description); + await masterDb.pushChanges({ + accessToken: myAccessToken, + description, + }); + // __PUBLISH_EXTRACT_END__ +} + +async function arbitraryEdit( + db: BriefcaseDb, + myAccessToken: AccessToken, + description: string +) { + const spatialCategoryCode = SpatialCategory.createCode( + db, + IModel.dictionaryId, + "SpatialCategory1" + ); + const physicalModelCode = PhysicalPartition.createCode( + db, + IModel.rootSubjectId, + "PhysicalModel1" + ); + let spatialCategoryId = db.elements.queryElementIdByCode(spatialCategoryCode); + let physicalModelId = db.elements.queryElementIdByCode(physicalModelCode); + if (physicalModelId === undefined || spatialCategoryId === undefined) { + spatialCategoryId = SpatialCategory.insert( + db, + IModel.dictionaryId, + "SpatialCategory1", + new SubCategoryAppearance() + ); + physicalModelId = PhysicalModel.insert( + db, + IModel.rootSubjectId, + "PhysicalModel1" + ); + } + const physicalObjectProps: PhysicalElementProps = { + classFullName: PhysicalObject.classFullName, + model: physicalModelId, + category: spatialCategoryId, + code: new Code({ + spec: IModelDb.rootSubjectId, + scope: IModelDb.rootSubjectId, + value: `${arbitraryEdit.editCounter}`, + }), + userLabel: `${arbitraryEdit.editCounter}`, + geom: IModelTransformerTestUtils.createBox(Point3d.create(1, 1, 1)), + placement: { + origin: Point3d.create( + arbitraryEdit.editCounter, + arbitraryEdit.editCounter, + 0 + ), + angles: YawPitchRollAngles.createDegrees(0, 0, 0), + }, + }; + arbitraryEdit.editCounter++; + db.elements.insertElement(physicalObjectProps); + db.saveChanges(); + await db.pushChanges({ + accessToken: myAccessToken, + description, + }); +} + +namespace arbitraryEdit { + // eslint-disable-next-line prefer-const + export let editCounter = 0; +} + +describe.only("IModelBranchingOperations", () => { + const version0Path = path.join( + TestUtils.KnownTestLocations.outputDir, + "branching-ops.bim" + ); + + before(async () => { + HubMock.startup( + "IModelBranchingOperations", + TestUtils.KnownTestLocations.outputDir + ); + if (fs.existsSync(version0Path)) fs.unlinkSync(version0Path); + SnapshotDb.createEmpty(version0Path, { + rootSubject: { name: "branching-ops" }, + }).close(); + }); + + after(() => { + HubMock.shutdown(); + }); + + it("run branching operations", async () => { + const myAccessToken = await HubWrappers.getAccessToken( + TestUtils.TestUserType.Regular + ); + const myITwinId = HubMock.iTwinId; + const masterIModelId = await IModelHost.hubAccess.createNewIModel({ + iTwinId: myITwinId, + iModelName: "my-branch-imodel", + version0: version0Path, + noLocks: true, + }); + const { masterDb, branchDb } = await initializeBranch( + myITwinId, + masterIModelId, + myAccessToken + ); + await arbitraryEdit(masterDb, myAccessToken, "edit master"); + await forwardSyncMasterToBranch(masterDb, branchDb, myAccessToken); + await arbitraryEdit(branchDb, myAccessToken, "edit branch"); + await reverseSyncBranchToMaster(branchDb, masterDb, myAccessToken); + masterDb.close(); + branchDb.close(); + }); +});