From 917e2cf00f0a377c96581289ec8e8863284d95f4 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 11 Sep 2024 16:20:06 +0530 Subject: [PATCH 01/55] Move methods in util class --- .../org/apache/atlas/web/rest/ModelREST.java | 156 +--------------- .../org/apache/atlas/web/util/ModelUtil.java | 173 ++++++++++++++++++ 2 files changed, 177 insertions(+), 152 deletions(-) create mode 100644 webapp/src/main/java/org/apache/atlas/web/util/ModelUtil.java diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/ModelREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/ModelREST.java index dd83cfc77f..d962b670e3 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/ModelREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/ModelREST.java @@ -16,6 +16,7 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.atlas.utils.AtlasPerfTracer; +import org.apache.atlas.web.util.ModelUtil; import org.apache.atlas.web.util.Servlets; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -41,13 +42,8 @@ @Produces({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) public class ModelREST { - private static final String BUSINESS_DATE = "dMDataModelBusinessDate"; - private static final String EXPIRED_BUSINESS_DATE = "dMDataModelExpiredAtBusinessDate"; - private static final String LESSER_THAN_EQUAL_TO = "lte"; - private static final String SYSTEM_DATE = "dMDataModelSystemDate"; - private static final String EXPIRED_SYSTEM_DATE = "dMDataModelExpiredAtSystemDate"; - private static final String NAMESPACE = "dMDataModelNamespace"; - private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.DiscoveryREST"); + + private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.ModelREST"); private static final Logger LOG = LoggerFactory.getLogger(DiscoveryREST.class); @Context @@ -91,7 +87,7 @@ public AtlasSearchResult dataSearch(@QueryParam("namespace") String namespace, parameters = parameters == null ? new IndexSearchParams() : parameters; - String queryStringUsingFiltersAndUserDSL = createQueryStringUsingFiltersAndUserDSL(namespace, + String queryStringUsingFiltersAndUserDSL = ModelUtil.createQueryStringUsingFiltersAndUserDSL(namespace, businessDate, systemDate, parameters.getQuery()); @@ -149,148 +145,4 @@ public AtlasSearchResult dataSearch(@QueryParam("namespace") String namespace, } } - /*** - * combines user query/dsl along with business parameters - * - * creates query as following : - * {"query":{"bool":{"must":[{"bool":{"filter":[{"match":{"namespace":"{namespace}"}},{"bool":{"must":[{"range":{"businessDate":{"lte":"businessDate"}}},{"bool":{"should":[{"range":{"expiredAtBusinessDate":{"gt":"{businessDate}"}}},{"bool":{"must_not":[{"exists":{"field":"expiredAtBusiness"}}]}}],"minimum_should_match":1}}]}}]}},{"wrapper":{"query":"user query"}}]}}} - * @param namespace - * @param businessDate - * @param dslString - * @return - */ - private String createQueryStringUsingFiltersAndUserDSL(final String namespace, - final String businessDate, - final String systemDate, - final String dslString) { - try { - AtlasPerfMetrics.MetricRecorder addBusinessFiltersToSearchQueryMetric = RequestContext.get().startMetricRecord("createQueryStringUsingFiltersAndUserDSL"); - // Create an ObjectMapper instance - ObjectMapper objectMapper = new ObjectMapper(); - - // Create the root 'query' node - ObjectNode rootNode = objectMapper.createObjectNode(); - ObjectNode queryNode = objectMapper.createObjectNode(); - ObjectNode boolNode = objectMapper.createObjectNode(); - ArrayNode mustArray = objectMapper.createArrayNode(); - - // Create the first 'bool' object inside 'must' - ObjectNode firstBoolNode = objectMapper.createObjectNode(); - ObjectNode filterBoolNode = objectMapper.createObjectNode(); - ArrayNode filterArray = objectMapper.createArrayNode(); - - // Create 'match' object - ObjectNode matchNode = objectMapper.createObjectNode(); - matchNode.put(NAMESPACE.concat(".keyword"), namespace); - - // Add 'match' object to filter - ObjectNode matchWrapper = objectMapper.createObjectNode(); - matchWrapper.set("term", matchNode); - filterArray.add(matchWrapper); - - // add 'businessDateValidation' - ObjectNode businessDateWrapper = dateValidation(businessDate, true, objectMapper); - filterArray.add(businessDateWrapper); - - // add 'systemDateValidation' - if (!StringUtils.isEmpty(systemDate)) { - ObjectNode systemDateWrapper = dateValidation(systemDate, false, objectMapper); - filterArray.add(systemDateWrapper); - } - - // Add filter to firstBool - filterBoolNode.set("filter", filterArray); - firstBoolNode.set("bool", filterBoolNode); - - // Add firstBool to must array - mustArray.add(firstBoolNode); - - // process user query - if (!StringUtils.isEmpty(dslString)) { - JsonNode node = new ObjectMapper().readTree(dslString); - JsonNode userQueryNode = node.get("query"); - ObjectNode wrapperNode = objectMapper.createObjectNode(); - String userQueryString = userQueryNode.toString(); - String userQueryBase64 = Base64.getEncoder().encodeToString(userQueryString.getBytes()); - wrapperNode.put("query", userQueryBase64); - // Add wrapper to must array - ObjectNode wrapperWrapper = objectMapper.createObjectNode(); - wrapperWrapper.set("wrapper", wrapperNode); - mustArray.add(wrapperWrapper); - } - - - // Add must array to bool node - boolNode.set("must", mustArray); - - // Add bool to query - queryNode.set("bool", boolNode); - - rootNode.set("query", queryNode); - - // Print the JSON representation of the query - return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(rootNode); - } catch (Exception e) { - LOG.error("Error -> createQueryStringUsingFiltersAndUserDSL!", e); - } - return ""; - } - - private ObjectNode dateValidation(final String date, final boolean isBusinessDate, ObjectMapper objectMapper) { - - String condition = LESSER_THAN_EQUAL_TO, dateType = BUSINESS_DATE, expiredDateType = EXPIRED_BUSINESS_DATE; - - if (!isBusinessDate) { - dateType = SYSTEM_DATE; - expiredDateType = EXPIRED_SYSTEM_DATE; - } - // Create the nested 'bool' object inside filter - ObjectNode nestedBoolNode = objectMapper.createObjectNode(); - ArrayNode nestedMustArray = objectMapper.createArrayNode(); - ObjectNode rangeBusinessDateNode = objectMapper.createObjectNode(); - rangeBusinessDateNode.put(condition, date); - - // Add 'range' object to nestedMust - ObjectNode rangeBusinessDateWrapper = objectMapper.createObjectNode(); - rangeBusinessDateWrapper.set("range", objectMapper.createObjectNode().set(dateType, rangeBusinessDateNode)); - nestedMustArray.add(rangeBusinessDateWrapper); - - - // Create 'bool' object for 'should' - ObjectNode shouldBoolNodeWrapper = objectMapper.createObjectNode(); - ObjectNode shouldBoolNode = objectMapper.createObjectNode(); - ArrayNode shouldArray = objectMapper.createArrayNode(); - - // Create 'range' object for 'expiredAtBusinessDate' - ObjectNode rangeExpiredAtNode = objectMapper.createObjectNode(); - rangeExpiredAtNode.put("gt", date); - - // Add 'range' object to should array - ObjectNode rangeExpiredAtWrapper = objectMapper.createObjectNode(); - rangeExpiredAtWrapper.set("range", objectMapper.createObjectNode().set(expiredDateType, rangeExpiredAtNode)); - shouldArray.add(rangeExpiredAtWrapper); - - // add 'term' object to should array - ObjectNode termNode = objectMapper.createObjectNode(); - termNode.put(expiredDateType, 0); - ObjectNode termNodeWrapper = objectMapper.createObjectNode(); - termNodeWrapper.set("term", termNode); - shouldArray.add(termNodeWrapper); - - // Add 'should' to should array - shouldBoolNode.set("should", shouldArray); - shouldBoolNode.put("minimum_should_match", 1); - shouldBoolNodeWrapper.set("bool", shouldBoolNode); - - // Add shouldBoolNodeWrapper to nestedMust - nestedMustArray.add(shouldBoolNodeWrapper); - - // Add nestedMust to nestedBool - nestedBoolNode.set("must", nestedMustArray); - - // Add nestedBool to filter - ObjectNode nestedBoolWrapper = objectMapper.createObjectNode(); - nestedBoolWrapper.set("bool", nestedBoolNode); - return nestedBoolWrapper; - } } \ No newline at end of file diff --git a/webapp/src/main/java/org/apache/atlas/web/util/ModelUtil.java b/webapp/src/main/java/org/apache/atlas/web/util/ModelUtil.java new file mode 100644 index 0000000000..d8e95d1b40 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/util/ModelUtil.java @@ -0,0 +1,173 @@ +package org.apache.atlas.web.util; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.atlas.utils.AtlasPerfTracer; +import org.apache.atlas.web.rest.DiscoveryREST; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Base64; + +public class ModelUtil { + + private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.ModelUtil"); + private static final Logger LOG = LoggerFactory.getLogger(DiscoveryREST.class); + + private static final String BUSINESS_DATE = "dMDataModelBusinessDate"; + private static final String EXPIRED_BUSINESS_DATE = "dMDataModelExpiredAtBusinessDate"; + private static final String LESSER_THAN_EQUAL_TO = "lte"; + private static final String SYSTEM_DATE = "dMDataModelSystemDate"; + private static final String EXPIRED_SYSTEM_DATE = "dMDataModelExpiredAtSystemDate"; + private static final String NAMESPACE = "dMDataModelNamespace"; + + /*** + * combines user query/dsl along with business parameters + * + * creates query as following : + * {"query":{"bool":{"must":[{"bool":{"filter":[{"match":{"namespace":"{namespace}"}},{"bool":{"must":[{"range":{"businessDate":{"lte":"businessDate"}}},{"bool":{"should":[{"range":{"expiredAtBusinessDate":{"gt":"{businessDate}"}}},{"bool":{"must_not":[{"exists":{"field":"expiredAtBusiness"}}]}}],"minimum_should_match":1}}]}}]}},{"wrapper":{"query":"user query"}}]}}} + * @param namespace + * @param businessDate + * @param dslString + * @return + */ + public static String createQueryStringUsingFiltersAndUserDSL(final String namespace, + final String businessDate, + final String systemDate, + final String dslString) { + try { + AtlasPerfMetrics.MetricRecorder addBusinessFiltersToSearchQueryMetric = RequestContext.get().startMetricRecord("createQueryStringUsingFiltersAndUserDSL"); + // Create an ObjectMapper instance + ObjectMapper objectMapper = new ObjectMapper(); + + // Create the root 'query' node + ObjectNode rootNode = objectMapper.createObjectNode(); + ObjectNode queryNode = objectMapper.createObjectNode(); + ObjectNode boolNode = objectMapper.createObjectNode(); + ArrayNode mustArray = objectMapper.createArrayNode(); + + // Create the first 'bool' object inside 'must' + ObjectNode firstBoolNode = objectMapper.createObjectNode(); + ObjectNode filterBoolNode = objectMapper.createObjectNode(); + ArrayNode filterArray = objectMapper.createArrayNode(); + + // Create 'match' object + ObjectNode matchNode = objectMapper.createObjectNode(); + matchNode.put(NAMESPACE.concat(".keyword"), namespace); + + // Add 'match' object to filter + ObjectNode matchWrapper = objectMapper.createObjectNode(); + matchWrapper.set("term", matchNode); + filterArray.add(matchWrapper); + + // add 'businessDateValidation' + ObjectNode businessDateWrapper = dateValidation(businessDate, true, objectMapper); + filterArray.add(businessDateWrapper); + + // add 'systemDateValidation' + if (!StringUtils.isEmpty(systemDate)) { + ObjectNode systemDateWrapper = dateValidation(systemDate, false, objectMapper); + filterArray.add(systemDateWrapper); + } + + // Add filter to firstBool + filterBoolNode.set("filter", filterArray); + firstBoolNode.set("bool", filterBoolNode); + + // Add firstBool to must array + mustArray.add(firstBoolNode); + + // process user query + if (!StringUtils.isEmpty(dslString)) { + JsonNode node = new ObjectMapper().readTree(dslString); + JsonNode userQueryNode = node.get("query"); + ObjectNode wrapperNode = objectMapper.createObjectNode(); + String userQueryString = userQueryNode.toString(); + String userQueryBase64 = Base64.getEncoder().encodeToString(userQueryString.getBytes()); + wrapperNode.put("query", userQueryBase64); + // Add wrapper to must array + ObjectNode wrapperWrapper = objectMapper.createObjectNode(); + wrapperWrapper.set("wrapper", wrapperNode); + mustArray.add(wrapperWrapper); + } + + + // Add must array to bool node + boolNode.set("must", mustArray); + + // Add bool to query + queryNode.set("bool", boolNode); + + rootNode.set("query", queryNode); + + // Print the JSON representation of the query + return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(rootNode); + } catch (Exception e) { + LOG.error("Error -> createQueryStringUsingFiltersAndUserDSL!", e); + } + return ""; + } + + private static ObjectNode dateValidation(final String date, final boolean isBusinessDate, ObjectMapper objectMapper) { + + String condition = LESSER_THAN_EQUAL_TO, dateType = BUSINESS_DATE, expiredDateType = EXPIRED_BUSINESS_DATE; + + if (!isBusinessDate) { + dateType = SYSTEM_DATE; + expiredDateType = EXPIRED_SYSTEM_DATE; + } + // Create the nested 'bool' object inside filter + ObjectNode nestedBoolNode = objectMapper.createObjectNode(); + ArrayNode nestedMustArray = objectMapper.createArrayNode(); + ObjectNode rangeBusinessDateNode = objectMapper.createObjectNode(); + rangeBusinessDateNode.put(condition, date); + + // Add 'range' object to nestedMust + ObjectNode rangeBusinessDateWrapper = objectMapper.createObjectNode(); + rangeBusinessDateWrapper.set("range", objectMapper.createObjectNode().set(dateType, rangeBusinessDateNode)); + nestedMustArray.add(rangeBusinessDateWrapper); + + + // Create 'bool' object for 'should' + ObjectNode shouldBoolNodeWrapper = objectMapper.createObjectNode(); + ObjectNode shouldBoolNode = objectMapper.createObjectNode(); + ArrayNode shouldArray = objectMapper.createArrayNode(); + + // Create 'range' object for 'expiredAtBusinessDate' + ObjectNode rangeExpiredAtNode = objectMapper.createObjectNode(); + rangeExpiredAtNode.put("gt", date); + + // Add 'range' object to should array + ObjectNode rangeExpiredAtWrapper = objectMapper.createObjectNode(); + rangeExpiredAtWrapper.set("range", objectMapper.createObjectNode().set(expiredDateType, rangeExpiredAtNode)); + shouldArray.add(rangeExpiredAtWrapper); + + // add 'term' object to should array + ObjectNode termNode = objectMapper.createObjectNode(); + termNode.put(expiredDateType, 0); + ObjectNode termNodeWrapper = objectMapper.createObjectNode(); + termNodeWrapper.set("term", termNode); + shouldArray.add(termNodeWrapper); + + // Add 'should' to should array + shouldBoolNode.set("should", shouldArray); + shouldBoolNode.put("minimum_should_match", 1); + shouldBoolNodeWrapper.set("bool", shouldBoolNode); + + // Add shouldBoolNodeWrapper to nestedMust + nestedMustArray.add(shouldBoolNodeWrapper); + + // Add nestedMust to nestedBool + nestedBoolNode.set("must", nestedMustArray); + + // Add nestedBool to filter + ObjectNode nestedBoolWrapper = objectMapper.createObjectNode(); + nestedBoolWrapper.set("bool", nestedBoolNode); + return nestedBoolWrapper; + } +} From 70360ac07b5d31a26356981a6eb9d492b37769f9 Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 13 Sep 2024 17:27:52 +0530 Subject: [PATCH 02/55] expose model atttributes --- .../apache/atlas/repository/Constants.java | 16 ++ .../model/AbstractModelPreProcessor.java | 2 + .../model/DMAttributePreprocessor.java | 2 + .../model/DMEntityPreProcessor.java | 227 ++++++++++++++++++ 4 files changed, 247 insertions(+) create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 9d3c7a3fde..f3a4d75a41 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -201,6 +201,13 @@ public final class Constants { public static final String PROCESS_OUTPUTS = "__Process.outputs"; public static final String PROCESS_INPUTS = "__Process.inputs"; + /*** + * DataModel + */ + + public static final String ATLAS_DM_ENTITY_TYPE = "DMEntity"; + + public static final String ATLAS_DM_ATTRIBUTE_TYPE = "DMAttribute"; public static String[] PROCESS_EDGE_LABELS = {PROCESS_OUTPUTS, PROCESS_INPUTS}; /** @@ -372,6 +379,15 @@ public final class Constants { public static final String ASSET_POLICY_GUIDS = "assetPolicyGUIDs"; public static final String ASSET_POLICIES_COUNT = "assetPoliciesCount"; + /*** + * Model related constants + * + */ + public static final String MODEL_EXPIRED_AT_SYSTEM_DATE= "dMDataModelExpiredAtSystemDate"; + public static final String MODEL_EXPIRED_AT_BUSINESS_DATE= "dMDataModelExpiredAtBusinessDate"; + public static final String MODEL_SYSTEM_DATE= "dMDataModelSystemDate"; + public static final String MODEL_BUSINESS_DATE= "dMDataModelBusinessDate"; + /* diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java new file mode 100644 index 0000000000..5f986ede98 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -0,0 +1,2 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.model;public class AbstractModelPreProcessor { +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java new file mode 100644 index 0000000000..a0d96f296f --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java @@ -0,0 +1,2 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.model;public class DMAttributePreprocessor { +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java new file mode 100644 index 0000000000..f6905cccd7 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -0,0 +1,227 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.model; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.discovery.EntityDiscoveryService; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.*; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; +import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.CategoryPreProcessor; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.lang.StringUtils; +import org.apache.poi.ss.formula.functions.Na; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; + +public class ModelPreProcessor implements PreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(CategoryPreProcessor.class); + private static final String ENTITY_TYPE = "DMEntity"; + private static final String ATTRIBUTE_TYPE = "DMAttribute"; + protected final AtlasTypeRegistry typeRegistry; + protected final EntityGraphRetriever entityRetriever; + protected EntityGraphMapper entityGraphMapper; + protected EntityDiscoveryService discovery; + protected AtlasRelationshipStore atlasRelationshipStore; + + public ModelPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, EntityGraphMapper entityGraphMapper, EntityDiscoveryService discovery, AtlasRelationshipStore atlasRelationshipStore) { + this.typeRegistry = typeRegistry; + this.entityRetriever = entityRetriever; + this.entityGraphMapper = entityGraphMapper; + this.discovery = discovery; + this.atlasRelationshipStore = atlasRelationshipStore; + } + + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, + EntityMutations.EntityOperation operation) throws AtlasBaseException { + //Handle name & qualifiedName + if (LOG.isDebugEnabled()) { + LOG.debug("ModelPreProcessor.processAttributes: pre processing {}, {}", + entityStruct.getAttribute(QUALIFIED_NAME), operation); + } + + + AtlasEntity entity = (AtlasEntity) entityStruct; + AtlasVertex vertex = context.getVertex(entity.getGuid()); + + + switch (operation) { + case CREATE: + processCreateModel(entity, vertex, context); + // modelVersion --->modelName + // entity ---> modelVersion + // attribute ---> entityName + + break; + case UPDATE: + processUpdateModel(entity, vertex, context); + + break; + } + } + + private void processCreateModel(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) { + return; + } + + private void processUpdateModel(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateMOdel"); + /*** + * + * 1. if entity is updated + * create deep copy of entity , e1' set expiredAtBusinessDate and expiredAtSystemDate + * get relationshipEdges of e1: a1,a1 + * create relationship a1, a2 to e1' + * get modelVersion from modelVersionQualifiedName in entity + * deep copy model version and name to v2 + * create relationship v2--e1' + * + * + * + * + * + */ + switch (entity.getTypeName()) { + case ENTITY_TYPE: + updateDMEntity(entity, vertex, context); + break; + case ATTRIBUTE_TYPE: + break; + } + + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + String entityName = (String) entity.getAttribute(NAME); + + if (StringUtils.isEmpty(entityName) || isNameInvalid(entityName)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); + } + + long now = Instant.now().toEpochMilli(); + + AtlasEntity.AtlasEntityWithExtInfo existingEntity = entityRetriever.toAtlasEntityWithExtInfo(vertex, false); + AtlasRelatedObjectId existingModelVersion = (AtlasRelatedObjectId) existingEntity.getEntity().getRelationshipAttributes().get("dMVersion"); + List existingEntityAttributes = (List) existingEntity.getEntity().getRelationshipAttributes().get("dMAttributes"); + + + // create modelVersion v2 + AtlasEntity existingModelVersionEntity = entityRetriever.toAtlasEntity(existingModelVersion.getGuid()); + AtlasVertex existingModelVersionVertex = entityRetriever.getEntityVertex(existingModelVersion.getGuid()); + AtlasVertex copyModelVertex = entityGraphMapper.createVertex(entityRetriever.toAtlasEntity(existingModelVersion.getGuid())); + AtlasEntity copyModelVersion = entityRetriever.toAtlasEntity(copyModelVertex); + setModelDates(copyModelVersion, copyModelVertex, now); + String modelQualifiedName = (String) existingModelVersionEntity.getAttribute(QUALIFIED_NAME) + now; + setQualifiedName(copyModelVersion, copyModelVertex, modelQualifiedName); + setModelExpiredAtDates(existingModelVersionEntity, existingModelVersionVertex, now); + + // create entity e1' + AtlasVertex copyEntityVertex = entityGraphMapper.createVertex(entity); + AtlasEntity copyEntity = entityRetriever.toAtlasEntity(copyEntityVertex); + setModelDates(copyEntity, copyEntityVertex, now); + String entityQualifiedName = modelQualifiedName + "/" + entityName; + setQualifiedName(copyEntity, copyEntityVertex, entityQualifiedName); + setModelDates(copyEntity, copyEntityVertex, now); + setModelExpiredAtDates(entity, vertex, now); + + // create modelVersion-modelEntity relationship + AtlasRelationship modelVersionEntityRelation = new AtlasRelationship("d_m_version_d_m_entities"); + modelVersionEntityRelation.setEnd1(new AtlasObjectId(copyModelVersion.getGuid(), + copyModelVersion.getTypeName(), Collections.singletonMap(QUALIFIED_NAME, modelQualifiedName))); + modelVersionEntityRelation.setEnd2(new AtlasObjectId(copyEntity.getGuid(), + copyEntity.getTypeName(), Collections.singletonMap(QUALIFIED_NAME, entityQualifiedName))); + atlasRelationshipStore.create(modelVersionEntityRelation); + + + // create modelEntity-modelAttributeRelationship + List entityAttributes = new ArrayList<>(); + AtlasRelationship modelEntityAttributeRelation = new AtlasRelationship("d_m_entity_d_m_attributes"); + modelEntityAttributeRelation.setEnd1( + new AtlasObjectId(copyEntity.getGuid(), copyEntity.getTypeName())); + for (AtlasRelatedObjectId existingEntityAttribute: existingEntityAttributes){ + modelEntityAttributeRelation.setEnd2( + new AtlasObjectId(existingEntityAttribute.getGuid(), existingEntityAttribute.getTypeName())); + atlasRelationshipStore.create(modelEntityAttributeRelation); + entityAttributes.add(entityRetriever.toAtlasEntity(existingModelVersion.getGuid())); + } + + // create model-modelVersion relation + AtlasEntity.AtlasEntityWithExtInfo existingModelVersionExtInfo = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersionVertex, false); + AtlasRelatedObjectId existingModel = (AtlasRelatedObjectId) existingModelVersionExtInfo.getEntity().getRelationshipAttributes().get("dMDataModel"); + AtlasRelationship modelVersionModelRelation = new AtlasRelationship("d_m_data_model_d_m_versions"); + modelVersionModelRelation.setEnd1( + new AtlasObjectId(existingModel.getGuid(), existingModel.getTypeName())); + modelVersionModelRelation.setEnd2( + new AtlasObjectId(copyModelVersion.getGuid(), copyModelVersion.getTypeName())); + atlasRelationshipStore.create(modelVersionModelRelation); + + + /** + * update context + */ + // previousEntity and previousModelVersion + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); + AtlasEntityType modelVersionType = typeRegistry.getEntityTypeByName(existingModelVersionEntity.getTypeName()); + + context.addUpdated(entity.getGuid(), entity, + entityType, vertex); + + context.addUpdated(existingModelVersionEntity.getGuid(), existingModelVersionEntity, + modelVersionType, existingModelVersionVertex); + + // add createdEntity and createdModelVersion + context.addCreated(copyEntity.getGuid(), copyEntity, entityType, copyEntityVertex); + context.addCreated(copyModelVersion.getGuid(), copyModelVersion, modelVersionType, copyModelVertex); + + // resolve references + context.getDiscoveryContext(). + addResolvedGuid( + existingModel.getGuid(), + entityRetriever.getEntityVertex(existingModel.getGuid())); + + // add attributes as relationship is updated : check + } + + + private void setModelDates(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { + newEntity.setAttribute(MODEL_SYSTEM_DATE, value); + newEntity.setAttribute(MODEL_BUSINESS_DATE, value); + AtlasGraphUtilsV2.setEncodedProperty(newVertex, MODEL_SYSTEM_DATE, value); + AtlasGraphUtilsV2.setEncodedProperty(newVertex, MODEL_BUSINESS_DATE, value); + } + + private void setModelExpiredAtDates(AtlasEntity oldEntity, AtlasVertex oldVertex, Object value) { + oldEntity.setAttribute(MODEL_EXPIRED_AT_SYSTEM_DATE, value); + oldEntity.setAttribute(MODEL_EXPIRED_AT_BUSINESS_DATE, value); + AtlasGraphUtilsV2.setEncodedProperty(oldVertex, MODEL_EXPIRED_AT_SYSTEM_DATE, value); + AtlasGraphUtilsV2.setEncodedProperty(oldVertex, MODEL_EXPIRED_AT_BUSINESS_DATE, value); + } + + private void setQualifiedName(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { + newEntity.setAttribute(QUALIFIED_NAME, value); + AtlasGraphUtilsV2.setEncodedProperty(newVertex, QUALIFIED_NAME, value); + } + + private boolean dmEntityExists(String dmEntityName) { + return AtlasGraphUtilsV2.dmEntityExists(dmEntityName); + } +} From b1ba7203afda2d3bed4af7b78ca56757b152b25e Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 13 Sep 2024 17:28:52 +0530 Subject: [PATCH 03/55] Expose attribute and entity prepocessors --- .../repository/store/graph/v2/AtlasEntityStoreV2.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index c0bb42298c..cc8fe76240 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -66,6 +66,8 @@ import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.CategoryPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.GlossaryPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.TermPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.model.DMAttributePreprocessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.model.DMEntityPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.LinkPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.ReadmePreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.sql.QueryCollectionPreProcessor; @@ -1925,6 +1927,12 @@ public PreProcessor getPreProcessor(String typeName) { case STAKEHOLDER_TITLE_ENTITY_TYPE: preProcessor = new StakeholderTitlePreProcessor(graph, typeRegistry, entityRetriever); break; + case ATLAS_DM_ENTITY_TYPE: + preProcessor = new DMEntityPreProcessor(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore); + break; + case ATLAS_DM_ATTRIBUTE_TYPE: + preProcessor = new DMAttributePreprocessor(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore); + break; } return preProcessor; From 70ee325d14b1fa2210b00a7b5bca789f922143fd Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 13 Sep 2024 17:29:29 +0530 Subject: [PATCH 04/55] Add DMEntity Preprocessor --- .../model/AbstractModelPreProcessor.java | 149 +++++++++++++++++- .../model/DMEntityPreProcessor.java | 148 ++++------------- 2 files changed, 177 insertions(+), 120 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index 5f986ede98..2238794062 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -1,2 +1,149 @@ -package org.apache.atlas.repository.store.graph.v2.preprocessor.model;public class AbstractModelPreProcessor { +package org.apache.atlas.repository.store.graph.v2.preprocessor.model; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.model.instance.AtlasRelatedObjectId; +import org.apache.atlas.model.instance.AtlasRelationship; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; +import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; + +public abstract class AbstractModelPreProcessor implements PreProcessor { + + private static final Logger LOG = LoggerFactory.getLogger(AbstractModelPreProcessor.class); + private static final String ATTRIBUTE_TYPE = "DMAttribute"; + + protected final AtlasTypeRegistry typeRegistry; + + protected final EntityGraphRetriever entityRetriever; + + protected EntityGraphMapper entityGraphMapper; + protected AtlasRelationshipStore atlasRelationshipStore; + + public AbstractModelPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, EntityGraphMapper entityGraphMapper, AtlasRelationshipStore atlasRelationshipStore) { + this.typeRegistry = typeRegistry; + this.entityRetriever = entityRetriever; + this.entityGraphMapper = entityGraphMapper; + this.atlasRelationshipStore = atlasRelationshipStore; + } + + protected static class ModelResponse { + + private AtlasEntity existingEntity; + private AtlasEntity copyEntity; + private AtlasVertex existingVertex; + private AtlasVertex copyVertex; + + protected ModelResponse(AtlasEntity existingEntity, AtlasEntity copyEntity, + AtlasVertex existingVertex, AtlasVertex copyVertex) { + this.existingEntity = existingEntity; + this.copyEntity = copyEntity; + this.existingVertex = existingVertex; + this.copyVertex = copyVertex; + } + + public AtlasEntity getExistingEntity() { + return existingEntity; + } + + public AtlasEntity getCopyEntity() { + return copyEntity; + } + + public AtlasVertex getExistingVertex() { + return existingVertex; + } + + public AtlasVertex getCopyVertex() { + return copyVertex; + } + } + + protected void setModelDates(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { + newEntity.setAttribute(MODEL_SYSTEM_DATE, value); + newEntity.setAttribute(MODEL_BUSINESS_DATE, value); + AtlasGraphUtilsV2.setEncodedProperty(newVertex, MODEL_SYSTEM_DATE, value); + AtlasGraphUtilsV2.setEncodedProperty(newVertex, MODEL_BUSINESS_DATE, value); + } + + protected void setModelExpiredAtDates(AtlasEntity oldEntity, AtlasVertex oldVertex, Object value) { + oldEntity.setAttribute(MODEL_EXPIRED_AT_SYSTEM_DATE, value); + oldEntity.setAttribute(MODEL_EXPIRED_AT_BUSINESS_DATE, value); + AtlasGraphUtilsV2.setEncodedProperty(oldVertex, MODEL_EXPIRED_AT_SYSTEM_DATE, value); + AtlasGraphUtilsV2.setEncodedProperty(oldVertex, MODEL_EXPIRED_AT_BUSINESS_DATE, value); + } + + protected void setQualifiedName(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { + newEntity.setAttribute(QUALIFIED_NAME, value); + AtlasGraphUtilsV2.setEncodedProperty(newVertex, QUALIFIED_NAME, value); + } + + protected ModelResponse replicateModelVersion(AtlasRelatedObjectId existingModelVersion, long epoch) throws AtlasBaseException { + AtlasEntity existingModelVersionEntity = entityRetriever.toAtlasEntity(existingModelVersion.getGuid()); + AtlasVertex existingModelVersionVertex = entityRetriever.getEntityVertex(existingModelVersion.getGuid()); + AtlasVertex copyModelVertex = entityGraphMapper.createVertex(entityRetriever.toAtlasEntity(existingModelVersion.getGuid())); + AtlasEntity copyModelVersion = entityRetriever.toAtlasEntity(copyModelVertex); + setModelDates(copyModelVersion, copyModelVertex, epoch); + String modelQualifiedName = (String) existingModelVersionEntity.getAttribute(QUALIFIED_NAME) + epoch; + setQualifiedName(copyModelVersion, copyModelVertex, modelQualifiedName); + setModelExpiredAtDates(existingModelVersionEntity, existingModelVersionVertex, epoch); + return new ModelResponse(existingModelVersionEntity, copyModelVersion, existingModelVersionVertex, copyModelVertex); + } + + protected ModelResponse replicateModelEntity(AtlasEntity entity, AtlasVertex vertex, String modelQualifiedName, long epoch) throws AtlasBaseException { + AtlasVertex copyEntityVertex = entityGraphMapper.createVertex(entity); + AtlasEntity copyEntity = entityRetriever.toAtlasEntity(copyEntityVertex); + setModelDates(copyEntity, copyEntityVertex, epoch); + String entityQualifiedName = modelQualifiedName + "/" + entity.getAttribute(NAME); + setQualifiedName(copyEntity, copyEntityVertex, entityQualifiedName); + setModelDates(copyEntity, copyEntityVertex, epoch); + setModelExpiredAtDates(entity, vertex, epoch); + return new ModelResponse(entity, copyEntity, vertex, copyEntityVertex); + } + + protected void createModelVersionModelEntityRelationship(AtlasEntity modelVersion, AtlasEntity modelEntity) throws AtlasBaseException { + AtlasRelationship modelVersionEntityRelation = new AtlasRelationship("d_m_version_d_m_entities"); + modelVersionEntityRelation.setEnd1(new AtlasObjectId( + modelVersion.getGuid(), + modelEntity.getTypeName())); + modelVersionEntityRelation.setEnd2(new AtlasObjectId( + modelEntity.getGuid(), + modelEntity.getTypeName())); + atlasRelationshipStore.create(modelVersionEntityRelation); + } + + protected void createModelEntityModelAttributeRelation(AtlasEntity entity, List existingEntityAttributes) throws AtlasBaseException { + AtlasRelationship modelEntityAttributeRelation = new AtlasRelationship("d_m_entity_d_m_attributes"); + modelEntityAttributeRelation.setEnd1( + new AtlasObjectId(entity.getGuid(), entity.getTypeName())); + for (AtlasRelatedObjectId existingEntityAttribute : existingEntityAttributes) { + modelEntityAttributeRelation.setEnd2( + new AtlasObjectId(existingEntityAttribute.getGuid(), existingEntityAttribute.getTypeName())); + atlasRelationshipStore.create(modelEntityAttributeRelation); + } + } + protected AtlasRelatedObjectId createModelModelVersionRelation(AtlasVertex existingModelVersionVertex, AtlasEntity latestModelVersionEntity) throws AtlasBaseException { + AtlasEntity.AtlasEntityWithExtInfo existingModelVersionExtInfo = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersionVertex, false); + AtlasRelatedObjectId existingModel = (AtlasRelatedObjectId) existingModelVersionExtInfo.getEntity().getRelationshipAttributes().get("dMDataModel"); + AtlasRelationship modelVersionModelRelation = new AtlasRelationship("d_m_data_model_d_m_versions"); + modelVersionModelRelation.setEnd1( + new AtlasObjectId(existingModel.getGuid(), existingModel.getTypeName())); + modelVersionModelRelation.setEnd2( + new AtlasObjectId(latestModelVersionEntity.getGuid(), latestModelVersionEntity.getTypeName())); + atlasRelationshipStore.create(modelVersionModelRelation); + return existingModel; + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java index f6905cccd7..956533dab2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -17,35 +17,23 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.lang.StringUtils; -import org.apache.poi.ss.formula.functions.Na; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.time.Instant; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; -public class ModelPreProcessor implements PreProcessor { - private static final Logger LOG = LoggerFactory.getLogger(CategoryPreProcessor.class); - private static final String ENTITY_TYPE = "DMEntity"; - private static final String ATTRIBUTE_TYPE = "DMAttribute"; - protected final AtlasTypeRegistry typeRegistry; - protected final EntityGraphRetriever entityRetriever; - protected EntityGraphMapper entityGraphMapper; - protected EntityDiscoveryService discovery; - protected AtlasRelationshipStore atlasRelationshipStore; - - public ModelPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, EntityGraphMapper entityGraphMapper, EntityDiscoveryService discovery, AtlasRelationshipStore atlasRelationshipStore) { - this.typeRegistry = typeRegistry; - this.entityRetriever = entityRetriever; - this.entityGraphMapper = entityGraphMapper; - this.discovery = discovery; - this.atlasRelationshipStore = atlasRelationshipStore; +public class DMEntityPreProcessor extends AbstractModelPreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(AbstractModelPreProcessor.class); + + + public DMEntityPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, EntityGraphMapper entityGraphMapper, AtlasRelationshipStore atlasRelationshipStore) { + super(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore); } @@ -65,50 +53,22 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co switch (operation) { case CREATE: - processCreateModel(entity, vertex, context); + createDMEntity(entity, vertex, context); // modelVersion --->modelName // entity ---> modelVersion // attribute ---> entityName break; case UPDATE: - processUpdateModel(entity, vertex, context); - + updateDMEntity(entity, vertex, context); break; } } - private void processCreateModel(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) { + private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) { return; } - private void processUpdateModel(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateMOdel"); - /*** - * - * 1. if entity is updated - * create deep copy of entity , e1' set expiredAtBusinessDate and expiredAtSystemDate - * get relationshipEdges of e1: a1,a1 - * create relationship a1, a2 to e1' - * get modelVersion from modelVersionQualifiedName in entity - * deep copy model version and name to v2 - * create relationship v2--e1' - * - * - * - * - * - */ - switch (entity.getTypeName()) { - case ENTITY_TYPE: - updateDMEntity(entity, vertex, context); - break; - case ATTRIBUTE_TYPE: - break; - } - - RequestContext.get().endMetricRecord(metricRecorder); - } private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { String entityName = (String) entity.getAttribute(NAME); @@ -118,61 +78,33 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati } long now = Instant.now().toEpochMilli(); - AtlasEntity.AtlasEntityWithExtInfo existingEntity = entityRetriever.toAtlasEntityWithExtInfo(vertex, false); AtlasRelatedObjectId existingModelVersion = (AtlasRelatedObjectId) existingEntity.getEntity().getRelationshipAttributes().get("dMVersion"); List existingEntityAttributes = (List) existingEntity.getEntity().getRelationshipAttributes().get("dMAttributes"); // create modelVersion v2 - AtlasEntity existingModelVersionEntity = entityRetriever.toAtlasEntity(existingModelVersion.getGuid()); - AtlasVertex existingModelVersionVertex = entityRetriever.getEntityVertex(existingModelVersion.getGuid()); - AtlasVertex copyModelVertex = entityGraphMapper.createVertex(entityRetriever.toAtlasEntity(existingModelVersion.getGuid())); - AtlasEntity copyModelVersion = entityRetriever.toAtlasEntity(copyModelVertex); - setModelDates(copyModelVersion, copyModelVertex, now); - String modelQualifiedName = (String) existingModelVersionEntity.getAttribute(QUALIFIED_NAME) + now; - setQualifiedName(copyModelVersion, copyModelVertex, modelQualifiedName); - setModelExpiredAtDates(existingModelVersionEntity, existingModelVersionVertex, now); + ModelResponse modelVersionResponse = replicateModelVersion(existingModelVersion, now); + AtlasEntity existingModelVersionEntity = modelVersionResponse.getExistingEntity(); + AtlasEntity copyModelVersion = modelVersionResponse.getCopyEntity(); + AtlasVertex existingModelVersionVertex = modelVersionResponse.getExistingVertex(); + AtlasVertex copyModelVersionVertex = modelVersionResponse.getCopyVertex(); + String modelQualifiedName = (String) copyModelVersion.getAttribute(QUALIFIED_NAME); // create entity e1' - AtlasVertex copyEntityVertex = entityGraphMapper.createVertex(entity); - AtlasEntity copyEntity = entityRetriever.toAtlasEntity(copyEntityVertex); - setModelDates(copyEntity, copyEntityVertex, now); - String entityQualifiedName = modelQualifiedName + "/" + entityName; - setQualifiedName(copyEntity, copyEntityVertex, entityQualifiedName); - setModelDates(copyEntity, copyEntityVertex, now); - setModelExpiredAtDates(entity, vertex, now); + ModelResponse modelEntityResponse = replicateModelEntity(entity, vertex, modelQualifiedName, now); + AtlasVertex copyEntityVertex = modelEntityResponse.getCopyVertex(); + AtlasEntity copyEntity = modelEntityResponse.getCopyEntity(); + // create modelVersion-modelEntity relationship - AtlasRelationship modelVersionEntityRelation = new AtlasRelationship("d_m_version_d_m_entities"); - modelVersionEntityRelation.setEnd1(new AtlasObjectId(copyModelVersion.getGuid(), - copyModelVersion.getTypeName(), Collections.singletonMap(QUALIFIED_NAME, modelQualifiedName))); - modelVersionEntityRelation.setEnd2(new AtlasObjectId(copyEntity.getGuid(), - copyEntity.getTypeName(), Collections.singletonMap(QUALIFIED_NAME, entityQualifiedName))); - atlasRelationshipStore.create(modelVersionEntityRelation); - - - // create modelEntity-modelAttributeRelationship - List entityAttributes = new ArrayList<>(); - AtlasRelationship modelEntityAttributeRelation = new AtlasRelationship("d_m_entity_d_m_attributes"); - modelEntityAttributeRelation.setEnd1( - new AtlasObjectId(copyEntity.getGuid(), copyEntity.getTypeName())); - for (AtlasRelatedObjectId existingEntityAttribute: existingEntityAttributes){ - modelEntityAttributeRelation.setEnd2( - new AtlasObjectId(existingEntityAttribute.getGuid(), existingEntityAttribute.getTypeName())); - atlasRelationshipStore.create(modelEntityAttributeRelation); - entityAttributes.add(entityRetriever.toAtlasEntity(existingModelVersion.getGuid())); - } + createModelVersionModelEntityRelationship(copyModelVersion, copyEntity); + + // create modelEntity-modelAttributeRelationship + createModelEntityModelAttributeRelation(copyEntity, existingEntityAttributes); // create model-modelVersion relation - AtlasEntity.AtlasEntityWithExtInfo existingModelVersionExtInfo = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersionVertex, false); - AtlasRelatedObjectId existingModel = (AtlasRelatedObjectId) existingModelVersionExtInfo.getEntity().getRelationshipAttributes().get("dMDataModel"); - AtlasRelationship modelVersionModelRelation = new AtlasRelationship("d_m_data_model_d_m_versions"); - modelVersionModelRelation.setEnd1( - new AtlasObjectId(existingModel.getGuid(), existingModel.getTypeName())); - modelVersionModelRelation.setEnd2( - new AtlasObjectId(copyModelVersion.getGuid(), copyModelVersion.getTypeName())); - atlasRelationshipStore.create(modelVersionModelRelation); + AtlasRelatedObjectId modelObject = createModelModelVersionRelation(existingModelVersionVertex, copyModelVersion); /** @@ -186,42 +118,20 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati entityType, vertex); context.addUpdated(existingModelVersionEntity.getGuid(), existingModelVersionEntity, - modelVersionType, existingModelVersionVertex); + modelVersionType, existingModelVersionVertex); // add createdEntity and createdModelVersion context.addCreated(copyEntity.getGuid(), copyEntity, entityType, copyEntityVertex); - context.addCreated(copyModelVersion.getGuid(), copyModelVersion, modelVersionType, copyModelVertex); + context.addCreated(copyModelVersion.getGuid(), copyModelVersion, modelVersionType, copyModelVersionVertex); // resolve references context.getDiscoveryContext(). addResolvedGuid( - existingModel.getGuid(), - entityRetriever.getEntityVertex(existingModel.getGuid())); + modelObject.getGuid(), + entityRetriever.getEntityVertex(modelObject.getGuid())); // add attributes as relationship is updated : check } +} - private void setModelDates(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { - newEntity.setAttribute(MODEL_SYSTEM_DATE, value); - newEntity.setAttribute(MODEL_BUSINESS_DATE, value); - AtlasGraphUtilsV2.setEncodedProperty(newVertex, MODEL_SYSTEM_DATE, value); - AtlasGraphUtilsV2.setEncodedProperty(newVertex, MODEL_BUSINESS_DATE, value); - } - - private void setModelExpiredAtDates(AtlasEntity oldEntity, AtlasVertex oldVertex, Object value) { - oldEntity.setAttribute(MODEL_EXPIRED_AT_SYSTEM_DATE, value); - oldEntity.setAttribute(MODEL_EXPIRED_AT_BUSINESS_DATE, value); - AtlasGraphUtilsV2.setEncodedProperty(oldVertex, MODEL_EXPIRED_AT_SYSTEM_DATE, value); - AtlasGraphUtilsV2.setEncodedProperty(oldVertex, MODEL_EXPIRED_AT_BUSINESS_DATE, value); - } - - private void setQualifiedName(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { - newEntity.setAttribute(QUALIFIED_NAME, value); - AtlasGraphUtilsV2.setEncodedProperty(newVertex, QUALIFIED_NAME, value); - } - - private boolean dmEntityExists(String dmEntityName) { - return AtlasGraphUtilsV2.dmEntityExists(dmEntityName); - } -} From b048c7b44c5a0a31e74f3540de1e55d5a81846bf Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 13 Sep 2024 17:34:00 +0530 Subject: [PATCH 05/55] remove updated entities from context --- .../store/graph/v2/EntityMutationContext.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java index 72c3200809..9a78c9690e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java @@ -84,6 +84,20 @@ public void addUpdated(String internalGuid, AtlasEntity entity, AtlasEntityType } } + public void removeUpdated(String internalGuid, AtlasEntity entity, AtlasEntityType type, AtlasVertex atlasVertex) { + if (!entityVsVertex.containsKey(internalGuid)) { // if the entity was already created/updated + entitiesUpdated.remove(entity); + entityVsType.remove(entity.getGuid(), type); + entityVsVertex.remove(entity.getGuid(), atlasVertex); + +// if (!StringUtils.equals(internalGuid, entity.getGuid())) { +// guidAssignments.put(internalGuid, entity.getGuid()); +// entityVsVertex.put(internalGuid, atlasVertex); +// } + } + } + + public void addEntityToRestore(AtlasVertex vertex) { if (entitiesToRestore == null) { entitiesToRestore = new ArrayList<>(); From 6d988f2eb6ed10069abbdee04729bb3fde6a9789 Mon Sep 17 00:00:00 2001 From: aarshi Date: Sun, 15 Sep 2024 12:40:54 +0530 Subject: [PATCH 06/55] Update removeContext util --- .../atlas/repository/store/graph/v2/EntityMutationContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java index 9a78c9690e..76f4ed4880 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java @@ -85,7 +85,7 @@ public void addUpdated(String internalGuid, AtlasEntity entity, AtlasEntityType } public void removeUpdated(String internalGuid, AtlasEntity entity, AtlasEntityType type, AtlasVertex atlasVertex) { - if (!entityVsVertex.containsKey(internalGuid)) { // if the entity was already created/updated + if (entityVsVertex.containsKey(internalGuid)) { // if the entity was already created/updated entitiesUpdated.remove(entity); entityVsType.remove(entity.getGuid(), type); entityVsVertex.remove(entity.getGuid(), atlasVertex); From 251411627ebc9a453db0e7ad13e79d637cd9eaff Mon Sep 17 00:00:00 2001 From: aarshi Date: Sun, 15 Sep 2024 12:41:28 +0530 Subject: [PATCH 07/55] Base class for ModelPreprocessors --- .../preprocessor/model/AbstractModelPreProcessor.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index 2238794062..e0c70f011c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -10,8 +10,11 @@ import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.type.AtlasTypeUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,7 +97,7 @@ protected void setQualifiedName(AtlasEntity newEntity, AtlasVertex newVertex, Ob protected ModelResponse replicateModelVersion(AtlasRelatedObjectId existingModelVersion, long epoch) throws AtlasBaseException { AtlasEntity existingModelVersionEntity = entityRetriever.toAtlasEntity(existingModelVersion.getGuid()); AtlasVertex existingModelVersionVertex = entityRetriever.getEntityVertex(existingModelVersion.getGuid()); - AtlasVertex copyModelVertex = entityGraphMapper.createVertex(entityRetriever.toAtlasEntity(existingModelVersion.getGuid())); + AtlasVertex copyModelVertex = entityGraphMapper.createVertex(existingModelVersionEntity); AtlasEntity copyModelVersion = entityRetriever.toAtlasEntity(copyModelVertex); setModelDates(copyModelVersion, copyModelVertex, epoch); String modelQualifiedName = (String) existingModelVersionEntity.getAttribute(QUALIFIED_NAME) + epoch; @@ -106,6 +109,8 @@ protected ModelResponse replicateModelVersion(AtlasRelatedObjectId existingModel protected ModelResponse replicateModelEntity(AtlasEntity entity, AtlasVertex vertex, String modelQualifiedName, long epoch) throws AtlasBaseException { AtlasVertex copyEntityVertex = entityGraphMapper.createVertex(entity); AtlasEntity copyEntity = entityRetriever.toAtlasEntity(copyEntityVertex); + copyEntity.setAttributes(entity.getAttributes()); + copyEntity.setRemoveRelationshipAttributes(entity.getRelationshipAttributes()); setModelDates(copyEntity, copyEntityVertex, epoch); String entityQualifiedName = modelQualifiedName + "/" + entity.getAttribute(NAME); setQualifiedName(copyEntity, copyEntityVertex, entityQualifiedName); @@ -114,6 +119,10 @@ protected ModelResponse replicateModelEntity(AtlasEntity entity, AtlasVertex ver return new ModelResponse(entity, copyEntity, vertex, copyEntityVertex); } +// protected ModelResponse replicateModelAttribute(){ +// +// } + protected void createModelVersionModelEntityRelationship(AtlasEntity modelVersion, AtlasEntity modelEntity) throws AtlasBaseException { AtlasRelationship modelVersionEntityRelation = new AtlasRelationship("d_m_version_d_m_entities"); modelVersionEntityRelation.setEnd1(new AtlasObjectId( From 608b508140c6f87118c4e329029e8a1f9e93afcd Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 16 Sep 2024 12:14:19 +0530 Subject: [PATCH 08/55] Error code declaration --- intg/src/main/java/org/apache/atlas/AtlasErrorCode.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index 1ad37b2564..4aab7e8742 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -291,7 +291,10 @@ public enum AtlasErrorCode { TASK_TYPE_NOT_SUPPORTED(400, "ATLAS-400-00-112", "Task type {0} is not supported"), PERSONA_POLICY_ASSETS_LIMIT_EXCEEDED(400, "ATLAS-400-00-113", "Exceeded limit of maximum allowed assets across policies for a Persona: Limit: {0}, assets: {1}"), - ADMIN_LIST_SHOULD_NOT_BE_EMPTY(400, "ATLAS-400-00-114", "Admin list should not be empty for type {0}"); + ADMIN_LIST_SHOULD_NOT_BE_EMPTY(400, "ATLAS-400-00-114", "Admin list should not be empty for type {0}"), + MODEL_VERSION_NOT_EXIST(400, "ATLAS-400-00-115", "Model version {0} does not exist"), + DATA_ENTITY_NOT_EXIST(400, "ATLAS-400-00-115", "DataEntity {0} does not exist"); + private String errorCode; private String errorMessage; From 6168274e874a065c357e095d4ac7db22f2033b77 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 16 Sep 2024 12:14:58 +0530 Subject: [PATCH 09/55] Expose helpers in abstarct class --- .../model/AbstractModelPreProcessor.java | 91 ++++++++++++++----- .../model/DMAttributePreprocessor.java | 83 ++++++++++++++++- .../model/DMEntityPreProcessor.java | 59 ++++++++++-- 3 files changed, 201 insertions(+), 32 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index e0c70f011c..44c5728683 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -5,6 +5,7 @@ import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasRelatedObjectId; import org.apache.atlas.model.instance.AtlasRelationship; +import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; @@ -15,11 +16,12 @@ import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeUtil; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.List; +import java.util.*; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; @@ -95,22 +97,23 @@ protected void setQualifiedName(AtlasEntity newEntity, AtlasVertex newVertex, Ob } protected ModelResponse replicateModelVersion(AtlasRelatedObjectId existingModelVersion, long epoch) throws AtlasBaseException { - AtlasEntity existingModelVersionEntity = entityRetriever.toAtlasEntity(existingModelVersion.getGuid()); + AtlasEntity.AtlasEntityWithExtInfo existingModelVersionEntityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersion.getGuid()); AtlasVertex existingModelVersionVertex = entityRetriever.getEntityVertex(existingModelVersion.getGuid()); - AtlasVertex copyModelVertex = entityGraphMapper.createVertex(existingModelVersionEntity); + AtlasVertex copyModelVertex = entityGraphMapper.createVertex(existingModelVersionEntityWithExtInfo.getEntity()); AtlasEntity copyModelVersion = entityRetriever.toAtlasEntity(copyModelVertex); + copyAllAttributes(existingModelVersionEntityWithExtInfo.getEntity(), copyModelVersion,epoch); setModelDates(copyModelVersion, copyModelVertex, epoch); - String modelQualifiedName = (String) existingModelVersionEntity.getAttribute(QUALIFIED_NAME) + epoch; + String modelQualifiedName = (String) existingModelVersionEntityWithExtInfo.getEntity().getAttribute(QUALIFIED_NAME) + epoch; setQualifiedName(copyModelVersion, copyModelVertex, modelQualifiedName); - setModelExpiredAtDates(existingModelVersionEntity, existingModelVersionVertex, epoch); - return new ModelResponse(existingModelVersionEntity, copyModelVersion, existingModelVersionVertex, copyModelVertex); + setModelExpiredAtDates(existingModelVersionEntityWithExtInfo.getEntity(), existingModelVersionVertex, epoch); + return new ModelResponse(existingModelVersionEntityWithExtInfo.getEntity(), copyModelVersion, existingModelVersionVertex, copyModelVertex); } protected ModelResponse replicateModelEntity(AtlasEntity entity, AtlasVertex vertex, String modelQualifiedName, long epoch) throws AtlasBaseException { AtlasVertex copyEntityVertex = entityGraphMapper.createVertex(entity); AtlasEntity copyEntity = entityRetriever.toAtlasEntity(copyEntityVertex); - copyEntity.setAttributes(entity.getAttributes()); - copyEntity.setRemoveRelationshipAttributes(entity.getRelationshipAttributes()); + copyAllAttributes(entity, copyEntity, epoch); + // copyEntity.setRelationshipAttributes(entity.getRelationshipAttributes()); setModelDates(copyEntity, copyEntityVertex, epoch); String entityQualifiedName = modelQualifiedName + "/" + entity.getAttribute(NAME); setQualifiedName(copyEntity, copyEntityVertex, entityQualifiedName); @@ -123,36 +126,82 @@ protected ModelResponse replicateModelEntity(AtlasEntity entity, AtlasVertex ver // // } - protected void createModelVersionModelEntityRelationship(AtlasEntity modelVersion, AtlasEntity modelEntity) throws AtlasBaseException { + protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersionVertex, AtlasVertex modelEntityVertex) throws AtlasBaseException { AtlasRelationship modelVersionEntityRelation = new AtlasRelationship("d_m_version_d_m_entities"); modelVersionEntityRelation.setEnd1(new AtlasObjectId( - modelVersion.getGuid(), - modelEntity.getTypeName())); + GraphHelper.getGuid(modelVersionVertex), + GraphHelper.getTypeName(modelVersionVertex))); modelVersionEntityRelation.setEnd2(new AtlasObjectId( - modelEntity.getGuid(), - modelEntity.getTypeName())); - atlasRelationshipStore.create(modelVersionEntityRelation); + GraphHelper.getGuid(modelEntityVertex), + GraphHelper.getTypeName(modelEntityVertex))); + atlasRelationshipStore.create(modelVersionEntityRelation); } - protected void createModelEntityModelAttributeRelation(AtlasEntity entity, List existingEntityAttributes) throws AtlasBaseException { + protected void createModelEntityModelAttributeRelation(AtlasVertex entity, List existingEntityAttributes) throws AtlasBaseException { AtlasRelationship modelEntityAttributeRelation = new AtlasRelationship("d_m_entity_d_m_attributes"); modelEntityAttributeRelation.setEnd1( - new AtlasObjectId(entity.getGuid(), entity.getTypeName())); + new AtlasObjectId( + GraphHelper.getGuid(entity), + GraphHelper.getTypeName(entity))); for (AtlasRelatedObjectId existingEntityAttribute : existingEntityAttributes) { modelEntityAttributeRelation.setEnd2( - new AtlasObjectId(existingEntityAttribute.getGuid(), existingEntityAttribute.getTypeName())); + new AtlasObjectId( + existingEntityAttribute.getGuid(), + existingEntityAttribute.getTypeName())); atlasRelationshipStore.create(modelEntityAttributeRelation); } } - protected AtlasRelatedObjectId createModelModelVersionRelation(AtlasVertex existingModelVersionVertex, AtlasEntity latestModelVersionEntity) throws AtlasBaseException { + protected AtlasRelatedObjectId createModelModelVersionRelation(AtlasVertex existingModelVersionVertex, AtlasVertex latestModelVersionVertex) throws AtlasBaseException { AtlasEntity.AtlasEntityWithExtInfo existingModelVersionExtInfo = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersionVertex, false); AtlasRelatedObjectId existingModel = (AtlasRelatedObjectId) existingModelVersionExtInfo.getEntity().getRelationshipAttributes().get("dMDataModel"); AtlasRelationship modelVersionModelRelation = new AtlasRelationship("d_m_data_model_d_m_versions"); modelVersionModelRelation.setEnd1( - new AtlasObjectId(existingModel.getGuid(), existingModel.getTypeName())); + new AtlasObjectId( + existingModel.getGuid(), + existingModel.getTypeName())); modelVersionModelRelation.setEnd2( - new AtlasObjectId(latestModelVersionEntity.getGuid(), latestModelVersionEntity.getTypeName())); + new AtlasObjectId( + GraphHelper.getGuid(latestModelVersionVertex), + GraphHelper.getTypeName(latestModelVersionVertex))); atlasRelationshipStore.create(modelVersionModelRelation); return existingModel; } + + protected void copyAllAttributes(AtlasEntity source , AtlasEntity destination, long epochNow){ + destination.setAttributes(source.getAttributes()); + destination.setMeanings(source.getMeanings()); + destination.setCreateTime(new Date(epochNow)); + destination.setUpdateTime(new Date(epochNow)); + // destination.setRelationshipAttributes(source.getRelationshipAttributes()); + destination.setCustomAttributes(source.getCustomAttributes()); + destination.setClassifications(source.getClassifications()); + destination.setAppendRelationshipAttributes(source.getAppendRelationshipAttributes()); + destination.setRemoveRelationshipAttributes(source.getRemoveRelationshipAttributes()); + } + + public static void replaceAttributes(Map existingAttributes, Map diffAttributes) { + if (MapUtils.isEmpty(diffAttributes)){ + return; + } + // Temporary map to hold new key-value pairs during replacement + Map tempMap = new HashMap<>(); + + // Iterate over the original map + for (Map.Entry entry : existingAttributes.entrySet()) { + String originalKey = entry.getKey(); + Object value = entry.getValue(); + + // Check if the second map contains a key for replacement + if (diffAttributes.containsKey(originalKey)) { + Object newValue = diffAttributes.get(originalKey); // Get the new key from second map + tempMap.put(originalKey, newValue); // Put the new key in the temp map + } else { + tempMap.put(originalKey, value); // No replacement, keep the original key + } + } + + // Clear the original map and put all the updated entries + existingAttributes.clear(); + existingAttributes.putAll(tempMap); + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java index a0d96f296f..36c9f10b4d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java @@ -1,2 +1,81 @@ -package org.apache.atlas.repository.store.graph.v2.preprocessor.model;public class DMAttributePreprocessor { -} +package org.apache.atlas.repository.store.graph.v2.preprocessor.model; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasRelatedObjectId; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; +import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.util.List; + +import static org.apache.atlas.repository.Constants.NAME; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; + +public class DMAttributePreprocessor extends AbstractModelPreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(AbstractModelPreProcessor.class); + + + public DMAttributePreprocessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, EntityGraphMapper entityGraphMapper, AtlasRelationshipStore atlasRelationshipStore) { + super(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore); + } + + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, + EntityMutations.EntityOperation operation) throws AtlasBaseException { + //Handle name & qualifiedName + if (LOG.isDebugEnabled()) { + LOG.debug("ModelPreProcessor.processAttributes: pre processing {}, {}", + entityStruct.getAttribute(QUALIFIED_NAME), operation); + } + + AtlasEntity entity = (AtlasEntity) entityStruct; + AtlasVertex vertex = context.getVertex(entity.getGuid()); + + switch (operation){ + case CREATE: + break; + case UPDATE: + } + } + + private void updateDMAttribute(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + String entityName = (String) entity.getAttribute(NAME); + + if (StringUtils.isEmpty(entityName) || isNameInvalid(entityName)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); + } + + long now = Instant.now().toEpochMilli(); + + AtlasEntity.AtlasEntityWithExtInfo existingEntityAttributeWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(vertex, false); + AtlasRelatedObjectId existingEntityObject = (AtlasRelatedObjectId) existingEntityAttributeWithExtInfo.getEntity().getRelationshipAttributes().get("dMEntity"); + AtlasEntity.AtlasEntityWithExtInfo existingEntityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(existingEntityObject.getGuid()); + AtlasRelatedObjectId existingModelVersionObject = (AtlasRelatedObjectId) existingEntityWithExtInfo.getEntity().getRelationshipAttributes().get("dMVersion"); + AtlasEntity.AtlasEntityWithExtInfo existingEntityVersionWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersionObject.getGuid()); + AtlasRelatedObjectId existingModelObject= (AtlasRelatedObjectId) existingEntityVersionWithExtInfo.getEntity().getRelationshipAttributes().get("dMModel"); + + + // create Model version + ModelResponse modelVersionResponse = replicateModelVersion(existingModelVersionObject, now); + AtlasEntity existingModelVersionEntity = modelVersionResponse.getExistingEntity(); + AtlasEntity copyModelVersion = modelVersionResponse.getCopyEntity(); + AtlasVertex existingModelVersionVertex = modelVersionResponse.getExistingVertex(); + AtlasVertex copyModelVersionVertex = modelVersionResponse.getCopyVertex(); + String modelQualifiedName = (String) copyModelVersion.getAttribute(QUALIFIED_NAME); + } + +} \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java index 956533dab2..2e53394894 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -16,6 +16,7 @@ import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,6 +48,8 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co } + + AtlasEntity entity = (AtlasEntity) entityStruct; AtlasVertex vertex = context.getVertex(entity.getGuid()); @@ -71,6 +74,10 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + if (CollectionUtils.isEmpty(context.getUpdatedEntities())) { + return; + } + String entityName = (String) entity.getAttribute(NAME); if (StringUtils.isEmpty(entityName) || isNameInvalid(entityName)) { @@ -82,6 +89,10 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati AtlasRelatedObjectId existingModelVersion = (AtlasRelatedObjectId) existingEntity.getEntity().getRelationshipAttributes().get("dMVersion"); List existingEntityAttributes = (List) existingEntity.getEntity().getRelationshipAttributes().get("dMAttributes"); + if (existingModelVersion == null){ + throw new AtlasBaseException(AtlasErrorCode.MODEL_VERSION_NOT_EXIST); + } + // create modelVersion v2 ModelResponse modelVersionResponse = replicateModelVersion(existingModelVersion, now); @@ -90,21 +101,26 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati AtlasVertex existingModelVersionVertex = modelVersionResponse.getExistingVertex(); AtlasVertex copyModelVersionVertex = modelVersionResponse.getCopyVertex(); String modelQualifiedName = (String) copyModelVersion.getAttribute(QUALIFIED_NAME); +// System.out.println("copyModelVersion: " +copyModelVersion.getGuid() + " existingModelVersion: " +existingModelVersionEntity.getGuid()); +// System.out.println("copyModelVersionVertex: " +copyModelVersionVertex.getId() + " existingModelVersionVertex: " +existingModelVersionVertex.getId()); // create entity e1' - ModelResponse modelEntityResponse = replicateModelEntity(entity, vertex, modelQualifiedName, now); + ModelResponse modelEntityResponse = replicateModelEntity(existingEntity.getEntity(), vertex, modelQualifiedName, now); AtlasVertex copyEntityVertex = modelEntityResponse.getCopyVertex(); AtlasEntity copyEntity = modelEntityResponse.getCopyEntity(); + applyDiffs(entity, copyEntity); +// System.out.println("copyEntity: " +copyEntity.getGuid() + " existingEntity: " +entity.getGuid()); +// System.out.println("copyModelVersionVertex: " +copyModelVersionVertex.getId() + " existingModelVersionVertex: " +existingModelVersionVertex.getId()); // create modelVersion-modelEntity relationship - createModelVersionModelEntityRelationship(copyModelVersion, copyEntity); + createModelVersionModelEntityRelationship(copyModelVersionVertex, copyEntityVertex); // create modelEntity-modelAttributeRelationship - createModelEntityModelAttributeRelation(copyEntity, existingEntityAttributes); + createModelEntityModelAttributeRelation(copyEntityVertex, existingEntityAttributes); // create model-modelVersion relation - AtlasRelatedObjectId modelObject = createModelModelVersionRelation(existingModelVersionVertex, copyModelVersion); + AtlasRelatedObjectId modelObject = createModelModelVersionRelation(existingModelVersionVertex, copyModelVersionVertex); /** @@ -114,24 +130,49 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); AtlasEntityType modelVersionType = typeRegistry.getEntityTypeByName(existingModelVersionEntity.getTypeName()); - context.addUpdated(entity.getGuid(), entity, - entityType, vertex); - - context.addUpdated(existingModelVersionEntity.getGuid(), existingModelVersionEntity, - modelVersionType, existingModelVersionVertex); +// context.addUpdated(entity.getGuid(), entity, +// entityType, vertex); +// +// context.addUpdated(existingModelVersionEntity.getGuid(), existingModelVersionEntity, +// modelVersionType, existingModelVersionVertex); // add createdEntity and createdModelVersion + context.addCreated(copyEntity.getGuid(), copyEntity, entityType, copyEntityVertex); context.addCreated(copyModelVersion.getGuid(), copyModelVersion, modelVersionType, copyModelVersionVertex); + // remove existing entity from context so it is not updated + context.removeUpdated(entity.getGuid(), entity, + entityType, vertex); + // resolve references context.getDiscoveryContext(). addResolvedGuid( modelObject.getGuid(), entityRetriever.getEntityVertex(modelObject.getGuid())); +// for (AtlasRelatedObjectId attributeObject: existingEntityAttributes){ +// context.getDiscoveryContext(). +// addResolvedGuid( +// attributeObject.getGuid(), +// entityRetriever.getEntityVertex(attributeObject.getGuid())); +// } // add attributes as relationship is updated : check } + + + private void applyDiffs(AtlasEntity sourceEntity, AtlasEntity destinationEntity) { + RequestContext reqContext = RequestContext.get(); + AtlasEntity diffEntity = reqContext.getDifferentialEntity(sourceEntity.getGuid()); + if (!diffEntity.getTypeName().equals(ATLAS_DM_ENTITY_TYPE)){ + return; + } + replaceAttributes(destinationEntity.getAttributes(), diffEntity.getAttributes()); + replaceAttributes(destinationEntity.getRelationshipAttributes(), diffEntity.getRelationshipAttributes()); + replaceAttributes(destinationEntity.getAppendRelationshipAttributes(), diffEntity.getAppendRelationshipAttributes()); + replaceAttributes(destinationEntity.getRemoveRelationshipAttributes(), diffEntity.getRemoveRelationshipAttributes()); + } + } From 3a80492336dd436c58856647ead4d6a1966b4f7c Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 16 Sep 2024 17:57:18 +0530 Subject: [PATCH 10/55] Data model error codes --- intg/src/main/java/org/apache/atlas/AtlasErrorCode.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index 4aab7e8742..b7fa72a7aa 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -292,8 +292,9 @@ public enum AtlasErrorCode { PERSONA_POLICY_ASSETS_LIMIT_EXCEEDED(400, "ATLAS-400-00-113", "Exceeded limit of maximum allowed assets across policies for a Persona: Limit: {0}, assets: {1}"), ADMIN_LIST_SHOULD_NOT_BE_EMPTY(400, "ATLAS-400-00-114", "Admin list should not be empty for type {0}"), - MODEL_VERSION_NOT_EXIST(400, "ATLAS-400-00-115", "Model version {0} does not exist"), - DATA_ENTITY_NOT_EXIST(400, "ATLAS-400-00-115", "DataEntity {0} does not exist"); + DATA_MODEL_VERSION_NOT_EXIST(400, "ATLAS-400-00-115", "Model version {0} does not exist"), + DATA_ENTITY_NOT_EXIST(400, "ATLAS-400-00-116", "DataEntity {0} does not exist"), + DATA_MODEL_NOT_EXIST(400, "ATLAS-400-00-117", "DataModel {0} does not exist"); private String errorCode; From 5cdd7517013593722c4eed04015d75a6e7e71d03 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 16 Sep 2024 17:58:00 +0530 Subject: [PATCH 11/55] Updates for Entity and attribute --- .../model/AbstractModelPreProcessor.java | 112 ++++++++++++---- .../model/DMAttributePreprocessor.java | 124 ++++++++++++++++-- .../model/DMEntityPreProcessor.java | 67 +++------- 3 files changed, 220 insertions(+), 83 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index 44c5728683..15d6418f27 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -1,5 +1,7 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.model; +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasObjectId; @@ -96,12 +98,12 @@ protected void setQualifiedName(AtlasEntity newEntity, AtlasVertex newVertex, Ob AtlasGraphUtilsV2.setEncodedProperty(newVertex, QUALIFIED_NAME, value); } - protected ModelResponse replicateModelVersion(AtlasRelatedObjectId existingModelVersion, long epoch) throws AtlasBaseException { - AtlasEntity.AtlasEntityWithExtInfo existingModelVersionEntityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersion.getGuid()); - AtlasVertex existingModelVersionVertex = entityRetriever.getEntityVertex(existingModelVersion.getGuid()); + protected ModelResponse replicateModelVersion(AtlasRelatedObjectId relatedObjectId, long epoch) throws AtlasBaseException { + AtlasEntity.AtlasEntityWithExtInfo existingModelVersionEntityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(relatedObjectId.getGuid()); + AtlasVertex existingModelVersionVertex = entityRetriever.getEntityVertex(relatedObjectId.getGuid()); AtlasVertex copyModelVertex = entityGraphMapper.createVertex(existingModelVersionEntityWithExtInfo.getEntity()); AtlasEntity copyModelVersion = entityRetriever.toAtlasEntity(copyModelVertex); - copyAllAttributes(existingModelVersionEntityWithExtInfo.getEntity(), copyModelVersion,epoch); + copyAllAttributes(existingModelVersionEntityWithExtInfo.getEntity(), copyModelVersion, epoch); setModelDates(copyModelVersion, copyModelVertex, epoch); String modelQualifiedName = (String) existingModelVersionEntityWithExtInfo.getEntity().getAttribute(QUALIFIED_NAME) + epoch; setQualifiedName(copyModelVersion, copyModelVertex, modelQualifiedName); @@ -109,41 +111,71 @@ protected ModelResponse replicateModelVersion(AtlasRelatedObjectId existingModel return new ModelResponse(existingModelVersionEntityWithExtInfo.getEntity(), copyModelVersion, existingModelVersionVertex, copyModelVertex); } - protected ModelResponse replicateModelEntity(AtlasEntity entity, AtlasVertex vertex, String modelQualifiedName, long epoch) throws AtlasBaseException { - AtlasVertex copyEntityVertex = entityGraphMapper.createVertex(entity); + protected ModelResponse replicateModelEntity(AtlasEntity existingEntity, AtlasVertex existingEntityVertex, String modelVersionQualifiedName, long epoch) throws AtlasBaseException { + AtlasVertex copyEntityVertex = entityGraphMapper.createVertex(existingEntity); AtlasEntity copyEntity = entityRetriever.toAtlasEntity(copyEntityVertex); - copyAllAttributes(entity, copyEntity, epoch); - // copyEntity.setRelationshipAttributes(entity.getRelationshipAttributes()); + copyAllAttributes(existingEntity, copyEntity, epoch); + // copyEntity.setRelationshipAttributes(entity.getRelationshipAttributes()); setModelDates(copyEntity, copyEntityVertex, epoch); - String entityQualifiedName = modelQualifiedName + "/" + entity.getAttribute(NAME); + String entityQualifiedName = modelVersionQualifiedName + "/" + existingEntity.getAttribute(NAME); setQualifiedName(copyEntity, copyEntityVertex, entityQualifiedName); setModelDates(copyEntity, copyEntityVertex, epoch); - setModelExpiredAtDates(entity, vertex, epoch); - return new ModelResponse(entity, copyEntity, vertex, copyEntityVertex); + setModelExpiredAtDates(existingEntity, existingEntityVertex, epoch); + return new ModelResponse(existingEntity, copyEntity, existingEntityVertex, copyEntityVertex); } -// protected ModelResponse replicateModelAttribute(){ -// -// } + protected ModelResponse replicateModelAttribute(AtlasEntity existingAttribute, AtlasVertex existingAttributeVertex, String entityQualifiedName, long epoch) throws AtlasBaseException { + AtlasVertex copyAttributeVertex = entityGraphMapper.createVertex(existingAttribute); + AtlasEntity copyAttributeEntity = entityRetriever.toAtlasEntity(copyAttributeVertex); + copyAllAttributes(existingAttribute, copyAttributeEntity, epoch); + // copyEntity.setRelationshipAttributes(entity.getRelationshipAttributes()); + setModelDates(copyAttributeEntity, copyAttributeVertex, epoch); + String attributeQualifiedName = entityQualifiedName + "/" + existingAttribute.getAttribute(NAME); + setQualifiedName(copyAttributeEntity, copyAttributeVertex, attributeQualifiedName); + setModelDates(copyAttributeEntity, copyAttributeVertex, epoch); + setModelExpiredAtDates(existingAttribute, existingAttributeVertex, epoch); + return new ModelResponse(existingAttribute, copyAttributeEntity, existingAttributeVertex, copyAttributeVertex); + } - protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersionVertex, AtlasVertex modelEntityVertex) throws AtlasBaseException { + protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersionVertex, + AtlasVertex modelEntityVertex) throws AtlasBaseException { AtlasRelationship modelVersionEntityRelation = new AtlasRelationship("d_m_version_d_m_entities"); modelVersionEntityRelation.setEnd1(new AtlasObjectId( GraphHelper.getGuid(modelVersionVertex), GraphHelper.getTypeName(modelVersionVertex))); modelVersionEntityRelation.setEnd2(new AtlasObjectId( - GraphHelper.getGuid(modelEntityVertex), - GraphHelper.getTypeName(modelEntityVertex))); - atlasRelationshipStore.create(modelVersionEntityRelation); + GraphHelper.getGuid(modelEntityVertex), + GraphHelper.getTypeName(modelEntityVertex))); + atlasRelationshipStore.create(modelVersionEntityRelation); + } + + protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersionVertex, String removeEntityGuid, + List existingEntities) { + AtlasRelationship modelVersionEntityRelation = new AtlasRelationship("d_m_version_d_m_entities"); + modelVersionEntityRelation.setEnd1(new AtlasObjectId( + GraphHelper.getGuid(modelVersionVertex), + GraphHelper.getTypeName(modelVersionVertex))); + for (AtlasRelatedObjectId entity : existingEntities) { + if (removeEntityGuid.equals(entity.getGuid())){ + continue; + } + modelVersionEntityRelation.setEnd2(new AtlasObjectId( + entity.getGuid(), + entity.getTypeName() + )); + } } - protected void createModelEntityModelAttributeRelation(AtlasVertex entity, List existingEntityAttributes) throws AtlasBaseException { + protected void createModelEntityModelAttributeRelation(AtlasVertex entity, String removeAttributeGuid, List existingEntityAttributes) throws AtlasBaseException { AtlasRelationship modelEntityAttributeRelation = new AtlasRelationship("d_m_entity_d_m_attributes"); modelEntityAttributeRelation.setEnd1( new AtlasObjectId( GraphHelper.getGuid(entity), GraphHelper.getTypeName(entity))); for (AtlasRelatedObjectId existingEntityAttribute : existingEntityAttributes) { + if (removeAttributeGuid.equals(existingEntityAttribute.getGuid())){ + continue; + } modelEntityAttributeRelation.setEnd2( new AtlasObjectId( existingEntityAttribute.getGuid(), @@ -151,14 +183,31 @@ protected void createModelEntityModelAttributeRelation(AtlasVertex entity, List< atlasRelationshipStore.create(modelEntityAttributeRelation); } } + + protected void createModelEntityModelAttributeRelation(AtlasVertex entity, AtlasVertex attribute) throws AtlasBaseException { + AtlasRelationship modelEntityAttributeRelation = new AtlasRelationship("d_m_entity_d_m_attributes"); + modelEntityAttributeRelation.setEnd1( + new AtlasObjectId( + GraphHelper.getGuid(entity), + GraphHelper.getTypeName(entity))); + modelEntityAttributeRelation.setEnd2( + new AtlasObjectId( + GraphHelper.getGuid(attribute), + GraphHelper.getTypeName(attribute))); + atlasRelationshipStore.create(modelEntityAttributeRelation); + } + protected AtlasRelatedObjectId createModelModelVersionRelation(AtlasVertex existingModelVersionVertex, AtlasVertex latestModelVersionVertex) throws AtlasBaseException { AtlasEntity.AtlasEntityWithExtInfo existingModelVersionExtInfo = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersionVertex, false); AtlasRelatedObjectId existingModel = (AtlasRelatedObjectId) existingModelVersionExtInfo.getEntity().getRelationshipAttributes().get("dMDataModel"); + if (existingModel == null) { + throw new AtlasBaseException(AtlasErrorCode.DATA_MODEL_NOT_EXIST); + } AtlasRelationship modelVersionModelRelation = new AtlasRelationship("d_m_data_model_d_m_versions"); modelVersionModelRelation.setEnd1( new AtlasObjectId( existingModel.getGuid(), - existingModel.getTypeName())); + existingModel.getTypeName())); modelVersionModelRelation.setEnd2( new AtlasObjectId( GraphHelper.getGuid(latestModelVersionVertex), @@ -167,12 +216,12 @@ protected AtlasRelatedObjectId createModelModelVersionRelation(AtlasVertex exist return existingModel; } - protected void copyAllAttributes(AtlasEntity source , AtlasEntity destination, long epochNow){ + protected void copyAllAttributes(AtlasEntity source, AtlasEntity destination, long epochNow) { destination.setAttributes(source.getAttributes()); destination.setMeanings(source.getMeanings()); destination.setCreateTime(new Date(epochNow)); destination.setUpdateTime(new Date(epochNow)); - // destination.setRelationshipAttributes(source.getRelationshipAttributes()); + // destination.setRelationshipAttributes(source.getRelationshipAttributes()); destination.setCustomAttributes(source.getCustomAttributes()); destination.setClassifications(source.getClassifications()); destination.setAppendRelationshipAttributes(source.getAppendRelationshipAttributes()); @@ -180,9 +229,9 @@ protected void copyAllAttributes(AtlasEntity source , AtlasEntity destination, l } public static void replaceAttributes(Map existingAttributes, Map diffAttributes) { - if (MapUtils.isEmpty(diffAttributes)){ - return; - } + if (MapUtils.isEmpty(diffAttributes)) { + return; + } // Temporary map to hold new key-value pairs during replacement Map tempMap = new HashMap<>(); @@ -204,4 +253,17 @@ public static void replaceAttributes(Map existingAttributes, Map existingAttributes.clear(); existingAttributes.putAll(tempMap); } + + protected void applyDiffs(AtlasEntity sourceEntity, AtlasEntity destinationEntity, String typeName) { + RequestContext reqContext = RequestContext.get(); + AtlasEntity diffEntity = reqContext.getDifferentialEntity(sourceEntity.getGuid()); + boolean diffExistsForSameType = diffEntity.getTypeName().equals(typeName); + if (!diffExistsForSameType) { + return; + } + replaceAttributes(destinationEntity.getAttributes(), diffEntity.getAttributes()); + replaceAttributes(destinationEntity.getRelationshipAttributes(), diffEntity.getRelationshipAttributes()); + replaceAttributes(destinationEntity.getAppendRelationshipAttributes(), diffEntity.getAppendRelationshipAttributes()); + replaceAttributes(destinationEntity.getRemoveRelationshipAttributes(), diffEntity.getRemoveRelationshipAttributes()); + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java index 36c9f10b4d..2217529fc0 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java @@ -6,6 +6,7 @@ import org.apache.atlas.model.instance.AtlasRelatedObjectId; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; @@ -13,15 +14,16 @@ import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.time.Instant; +import java.util.ArrayList; import java.util.List; -import static org.apache.atlas.repository.Constants.NAME; -import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; public class DMAttributePreprocessor extends AbstractModelPreProcessor { @@ -45,15 +47,24 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co AtlasEntity entity = (AtlasEntity) entityStruct; AtlasVertex vertex = context.getVertex(entity.getGuid()); - switch (operation){ + switch (operation) { case CREATE: break; case UPDATE: + updateDMAttribute(entity, vertex, context); } } - private void updateDMAttribute(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { - String entityName = (String) entity.getAttribute(NAME); + + private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAttribute, EntityMutationContext context) throws AtlasBaseException { + if (!entityAttribute.getTypeName().equals(ATLAS_DM_ATTRIBUTE_TYPE)) { + return; + } + if (CollectionUtils.isEmpty(context.getUpdatedEntities())) { + return; + } + + String entityName = (String) entityAttribute.getAttribute(NAME); if (StringUtils.isEmpty(entityName) || isNameInvalid(entityName)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); @@ -61,21 +72,114 @@ private void updateDMAttribute(AtlasEntity entity, AtlasVertex vertex, EntityMut long now = Instant.now().toEpochMilli(); - AtlasEntity.AtlasEntityWithExtInfo existingEntityAttributeWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(vertex, false); - AtlasRelatedObjectId existingEntityObject = (AtlasRelatedObjectId) existingEntityAttributeWithExtInfo.getEntity().getRelationshipAttributes().get("dMEntity"); + AtlasEntity.AtlasEntityWithExtInfo existingEntityAttributeWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(vertexAttribute, false); + List existingEntityObjects = (List) existingEntityAttributeWithExtInfo.getEntity().getRelationshipAttributes().get("dMEntities"); + + if (CollectionUtils.isEmpty(existingEntityObjects)) { + throw new AtlasBaseException(AtlasErrorCode.DATA_ENTITY_NOT_EXIST); + } + + AtlasRelatedObjectId existingEntityObject = null; + // retrieve entity where expiredAtBusinessDate and expiredAtSystemDate is not set + for (AtlasRelatedObjectId _existingEntityObject : existingEntityObjects) { + long expiredBusinessDate = (long) entityRetriever.toAtlasEntity(_existingEntityObject.getGuid()).getAttribute(MODEL_EXPIRED_AT_BUSINESS_DATE); + long expiredSystemDate = (long) entityRetriever.toAtlasEntity(_existingEntityObject.getGuid()).getAttribute(MODEL_EXPIRED_AT_SYSTEM_DATE); + if (expiredBusinessDate > 0 && expiredSystemDate > 0) { + existingEntityObject = _existingEntityObject; + } + } + + if (existingEntityObject == null) { + throw new AtlasBaseException(AtlasErrorCode.DATA_ENTITY_NOT_EXIST); + } + AtlasEntity.AtlasEntityWithExtInfo existingEntityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(existingEntityObject.getGuid()); AtlasRelatedObjectId existingModelVersionObject = (AtlasRelatedObjectId) existingEntityWithExtInfo.getEntity().getRelationshipAttributes().get("dMVersion"); + + if (existingModelVersionObject == null) { + throw new AtlasBaseException(AtlasErrorCode.DATA_MODEL_VERSION_NOT_EXIST); + } AtlasEntity.AtlasEntityWithExtInfo existingEntityVersionWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersionObject.getGuid()); - AtlasRelatedObjectId existingModelObject= (AtlasRelatedObjectId) existingEntityVersionWithExtInfo.getEntity().getRelationshipAttributes().get("dMModel"); + AtlasRelatedObjectId existingModelObject = (AtlasRelatedObjectId) existingEntityVersionWithExtInfo.getEntity().getRelationshipAttributes().get("dMModel"); + if (existingModelObject == null) { + throw new AtlasBaseException(AtlasErrorCode.DATA_MODEL_NOT_EXIST); + } - // create Model version + // create modelVersion v1 ---> v2 ModelResponse modelVersionResponse = replicateModelVersion(existingModelVersionObject, now); AtlasEntity existingModelVersionEntity = modelVersionResponse.getExistingEntity(); AtlasEntity copyModelVersion = modelVersionResponse.getCopyEntity(); AtlasVertex existingModelVersionVertex = modelVersionResponse.getExistingVertex(); AtlasVertex copyModelVersionVertex = modelVersionResponse.getCopyVertex(); - String modelQualifiedName = (String) copyModelVersion.getAttribute(QUALIFIED_NAME); + String modelVersionQualifiedName = (String) copyModelVersion.getAttribute(QUALIFIED_NAME); + + // retrieve entities linked to previous version + existingEntityObjects = (List) existingModelVersionEntity.getRelationshipAttributes().get("dMEntities"); + List activeEntities = new ArrayList<>(); + for (AtlasRelatedObjectId _existingEntityObject : existingEntityObjects) { + AtlasEntity entity = entityRetriever.toAtlasEntity(_existingEntityObject.getGuid()); + if ((long) entity.getAttributes().get(MODEL_EXPIRED_AT_BUSINESS_DATE) > 0 && (long) entity.getAttributes().get(MODEL_EXPIRED_AT_BUSINESS_DATE) > 0) { + activeEntities.add(entity); + } + } + + // create entity e1 ---> e1' + ModelResponse modelEntityResponse = replicateModelEntity( + existingEntityWithExtInfo.getEntity(), + entityRetriever.getEntityVertex(existingModelVersionEntity.getGuid()), + modelVersionQualifiedName, + now); + AtlasVertex copyEntityVertex = modelEntityResponse.getCopyVertex(); + AtlasEntity copyEntity = modelEntityResponse.getCopyEntity(); + String entityQualifiedName = (String) copyModelVersion.getAttribute(QUALIFIED_NAME); + List existingEntityAttributes = (List) modelEntityResponse.getExistingEntity().getRelationshipAttributes().get("dMAttributes"); + + + // create attribute a1 ---> a1' + ModelResponse modelAttributeResponse = replicateModelAttribute( + existingEntityAttributeWithExtInfo.getEntity(), + entityRetriever.getEntityVertex(entityAttribute.getGuid()), + entityQualifiedName, + now); + AtlasVertex copyAttributeVertex = modelEntityResponse.getCopyVertex(); + AtlasEntity copyAttribute = modelEntityResponse.getCopyEntity(); + applyDiffs(entityAttribute, copyAttribute, ATLAS_DM_ATTRIBUTE_TYPE); + + // create model-modelVersion relationship + AtlasRelatedObjectId modelObject = createModelModelVersionRelation(existingModelVersionVertex, copyModelVersionVertex); + + // create modelVersion-entity relationship [with new entity] + createModelVersionModelEntityRelationship(copyModelVersionVertex, copyEntityVertex); + + // create modelVersion-entity relationship [with existing entities] + for (AtlasEntity entity : activeEntities) { + createModelVersionModelEntityRelationship(copyModelVersionVertex, entityRetriever.getEntityVertex(entity.getGuid())); + } + + // create entity - attribute relation [with new attribute] + createModelEntityModelAttributeRelation(copyEntityVertex, copyAttributeVertex); + + // create entity- attribute relation [with existing attributes] + createModelEntityModelAttributeRelation(copyEntityVertex, entityAttribute.getGuid(), existingEntityAttributes); + + AtlasEntityType attributeType = typeRegistry.getEntityTypeByName(entityAttribute.getTypeName()); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(copyEntity.getTypeName()); + AtlasEntityType modelVersionType = typeRegistry.getEntityTypeByName(existingModelVersionEntity.getTypeName()); + + context.addCreated(copyAttribute.getGuid(), copyAttribute, attributeType, copyAttributeVertex); + context.addCreated(copyEntity.getGuid(), copyEntity, entityType, copyEntityVertex); + context.addCreated(copyModelVersion.getGuid(), copyModelVersion, modelVersionType, copyModelVersionVertex); + + // remove existing entity from context so it is not updated + context.removeUpdated(entityAttribute.getGuid(), entityAttribute, + entityType, vertexAttribute); + + // resolve references + context.getDiscoveryContext(). + addResolvedGuid( + modelObject.getGuid(), + entityRetriever.getEntityVertex(modelObject.getGuid())); } } \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java index 2e53394894..2ad87873eb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -48,8 +48,6 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co } - - AtlasEntity entity = (AtlasEntity) entityStruct; AtlasVertex vertex = context.getVertex(entity.getGuid()); @@ -74,6 +72,9 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + if (!entity.getTypeName().equals(ATLAS_DM_ENTITY_TYPE)) { + return; + } if (CollectionUtils.isEmpty(context.getUpdatedEntities())) { return; } @@ -89,38 +90,37 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati AtlasRelatedObjectId existingModelVersion = (AtlasRelatedObjectId) existingEntity.getEntity().getRelationshipAttributes().get("dMVersion"); List existingEntityAttributes = (List) existingEntity.getEntity().getRelationshipAttributes().get("dMAttributes"); - if (existingModelVersion == null){ - throw new AtlasBaseException(AtlasErrorCode.MODEL_VERSION_NOT_EXIST); + if (existingModelVersion == null) { + throw new AtlasBaseException(AtlasErrorCode.DATA_MODEL_VERSION_NOT_EXIST, existingModelVersion.getGuid()); } - - // create modelVersion v2 + // create modelVersion v1 ---> v2 ModelResponse modelVersionResponse = replicateModelVersion(existingModelVersion, now); AtlasEntity existingModelVersionEntity = modelVersionResponse.getExistingEntity(); AtlasEntity copyModelVersion = modelVersionResponse.getCopyEntity(); AtlasVertex existingModelVersionVertex = modelVersionResponse.getExistingVertex(); AtlasVertex copyModelVersionVertex = modelVersionResponse.getCopyVertex(); - String modelQualifiedName = (String) copyModelVersion.getAttribute(QUALIFIED_NAME); -// System.out.println("copyModelVersion: " +copyModelVersion.getGuid() + " existingModelVersion: " +existingModelVersionEntity.getGuid()); -// System.out.println("copyModelVersionVertex: " +copyModelVersionVertex.getId() + " existingModelVersionVertex: " +existingModelVersionVertex.getId()); + String modelVersionAttributeQualifiedName = (String) copyModelVersion.getAttribute(QUALIFIED_NAME); + List existingEntities = (List) existingModelVersionEntity.getRelationshipAttributes().get("dMEntities"); - // create entity e1' - ModelResponse modelEntityResponse = replicateModelEntity(existingEntity.getEntity(), vertex, modelQualifiedName, now); + + // create entity e1 ---> e1' + ModelResponse modelEntityResponse = replicateModelEntity(existingEntity.getEntity(), vertex, modelVersionAttributeQualifiedName, now); AtlasVertex copyEntityVertex = modelEntityResponse.getCopyVertex(); AtlasEntity copyEntity = modelEntityResponse.getCopyEntity(); - applyDiffs(entity, copyEntity); -// System.out.println("copyEntity: " +copyEntity.getGuid() + " existingEntity: " +entity.getGuid()); -// System.out.println("copyModelVersionVertex: " +copyModelVersionVertex.getId() + " existingModelVersionVertex: " +existingModelVersionVertex.getId()); + applyDiffs(entity, copyEntity, ATLAS_DM_ENTITY_TYPE); + // create model-modelVersion relation + AtlasRelatedObjectId modelObject = createModelModelVersionRelation(existingModelVersionVertex, copyModelVersionVertex); - // create modelVersion-modelEntity relationship + // create modelVersion-modelEntity relationship with new entity createModelVersionModelEntityRelationship(copyModelVersionVertex, copyEntityVertex); - // create modelEntity-modelAttributeRelationship - createModelEntityModelAttributeRelation(copyEntityVertex, existingEntityAttributes); + // create modelVersion-modelEntity relation with old entities + createModelVersionModelEntityRelationship(copyModelVersionVertex, entity.getGuid(), existingEntities); - // create model-modelVersion relation - AtlasRelatedObjectId modelObject = createModelModelVersionRelation(existingModelVersionVertex, copyModelVersionVertex); + // create modelEntity-modelAttributeRelationship + createModelEntityModelAttributeRelation(copyEntityVertex, "", existingEntityAttributes); /** @@ -130,14 +130,6 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); AtlasEntityType modelVersionType = typeRegistry.getEntityTypeByName(existingModelVersionEntity.getTypeName()); -// context.addUpdated(entity.getGuid(), entity, -// entityType, vertex); -// -// context.addUpdated(existingModelVersionEntity.getGuid(), existingModelVersionEntity, -// modelVersionType, existingModelVersionVertex); - - // add createdEntity and createdModelVersion - context.addCreated(copyEntity.getGuid(), copyEntity, entityType, copyEntityVertex); context.addCreated(copyModelVersion.getGuid(), copyModelVersion, modelVersionType, copyModelVersionVertex); @@ -151,28 +143,7 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati modelObject.getGuid(), entityRetriever.getEntityVertex(modelObject.getGuid())); -// for (AtlasRelatedObjectId attributeObject: existingEntityAttributes){ -// context.getDiscoveryContext(). -// addResolvedGuid( -// attributeObject.getGuid(), -// entityRetriever.getEntityVertex(attributeObject.getGuid())); -// } - // add attributes as relationship is updated : check - } - - - private void applyDiffs(AtlasEntity sourceEntity, AtlasEntity destinationEntity) { - RequestContext reqContext = RequestContext.get(); - AtlasEntity diffEntity = reqContext.getDifferentialEntity(sourceEntity.getGuid()); - if (!diffEntity.getTypeName().equals(ATLAS_DM_ENTITY_TYPE)){ - return; - } - replaceAttributes(destinationEntity.getAttributes(), diffEntity.getAttributes()); - replaceAttributes(destinationEntity.getRelationshipAttributes(), diffEntity.getRelationshipAttributes()); - replaceAttributes(destinationEntity.getAppendRelationshipAttributes(), diffEntity.getAppendRelationshipAttributes()); - replaceAttributes(destinationEntity.getRemoveRelationshipAttributes(), diffEntity.getRemoveRelationshipAttributes()); } - } From 91a405ab5e6e24594a0252d8251aaceb3cea09e9 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 16 Sep 2024 19:18:43 +0530 Subject: [PATCH 12/55] expose preprocessors --- .../atlas/repository/store/graph/v2/AtlasEntityStoreV2.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 1caa6bbc84..441ecd342e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1928,10 +1928,10 @@ public List getPreProcessor(String typeName) { preProcessors.add(new StakeholderTitlePreProcessor(graph, typeRegistry, entityRetriever)); break; case ATLAS_DM_ENTITY_TYPE: - preProcessor = new DMEntityPreProcessor(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore); + preProcessors.add(new DMEntityPreProcessor(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore)); break; case ATLAS_DM_ATTRIBUTE_TYPE: - preProcessor = new DMAttributePreprocessor(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore); + preProcessors.add(new DMAttributePreprocessor(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore)); break; } From 46c51d394e7bd17ff2c7dd05f2a931992cd36936 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 18 Sep 2024 17:06:45 +0530 Subject: [PATCH 13/55] Add constants for data model atttributes --- .../src/main/java/org/apache/atlas/repository/Constants.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index e84a3b2ca3..3f3e7df14c 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -208,6 +208,10 @@ public final class Constants { public static final String ATLAS_DM_ENTITY_TYPE = "DMEntity"; public static final String ATLAS_DM_ATTRIBUTE_TYPE = "DMAttribute"; + public static final String ATLAS_DM_DATA_MODEL = "DMDataModel"; + + public static final String ATLAS_DM_QUALIFIED_NAME_PREFIX = "dMQualifiedNamePrefix"; + public static String[] PROCESS_EDGE_LABELS = {PROCESS_OUTPUTS, PROCESS_INPUTS}; /** From 746fb12e2ccf2922c52ff980d53e97b959e79515 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 18 Sep 2024 17:07:16 +0530 Subject: [PATCH 14/55] add data model error codes --- intg/src/main/java/org/apache/atlas/AtlasErrorCode.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index b7fa72a7aa..7760676f5e 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -294,7 +294,10 @@ public enum AtlasErrorCode { ADMIN_LIST_SHOULD_NOT_BE_EMPTY(400, "ATLAS-400-00-114", "Admin list should not be empty for type {0}"), DATA_MODEL_VERSION_NOT_EXIST(400, "ATLAS-400-00-115", "Model version {0} does not exist"), DATA_ENTITY_NOT_EXIST(400, "ATLAS-400-00-116", "DataEntity {0} does not exist"), - DATA_MODEL_NOT_EXIST(400, "ATLAS-400-00-117", "DataModel {0} does not exist"); + DATA_MODEL_NOT_EXIST(400, "ATLAS-400-00-117", "DataModel {0} does not exist"), + QUALIFIED_NAME_PREFIX (400, "ATLAS-400-00-118", "dMQualifiedNamePrefix is mandatory for DMEntity/DMAttribute"), + + NO_TYPE_EXISTS_FOR_QUALIFIED_NAME_PREFIX (400,"ATLAS-400-00-119", "No DMEntity/DMAttribute exists for dMQualifiedNamePrefix : {0}"); private String errorCode; From 6fefc0209b9847a958d8b00f9a212b779c61593b Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 18 Sep 2024 17:08:19 +0530 Subject: [PATCH 15/55] add util to fetch latet entity/attribute from prefix --- .../graph/v2/AtlasEntityGraphDiscoveryV2.java | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java index 040822dccc..cb221ba66f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java @@ -24,7 +24,9 @@ import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.EntityGraphDiscovery; import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext; import org.apache.atlas.repository.store.graph.EntityResolver; @@ -103,8 +105,30 @@ public void validateAndNormalize(AtlasEntity entity) throws AtlasBaseException { validateLabels(entity.getLabels()); + // entity is user input type.validateValue(entity, entity.getTypeName(), messages); + // set guid here + + // DMEntity and DMAttributeType are requested for update + // by dMQualifiedNamePrefix which is not a unique attribute + // This can return multiple entity/attribute that match this prefix vale + // we have to return latest entity/attribute + if (entity.getTypeName().equals(Constants.ATLAS_DM_ENTITY_TYPE) || + entity.getTypeName().equals(Constants.ATLAS_DM_ATTRIBUTE_TYPE)){ + + AtlasVertex vertex = AtlasGraphUtilsV2.findLatestEntityAttributeVerticesByType(entity.getTypeName()); + String guidFromVertex= AtlasGraphUtilsV2.getIdFromVertex(vertex); + + if (guidFromVertex.isEmpty()){ + throw new AtlasBaseException(AtlasErrorCode.NO_TYPE_EXISTS_FOR_QUALIFIED_NAME_PREFIX, (String) entity.getAttributes().get(Constants.ATLAS_DM_QUALIFIED_NAME_PREFIX)); + } + + entity.setGuid(AtlasGraphUtilsV2.getIdFromVertex(vertex)); + type.getNormalizedValue(entity); + return; + } + if (!messages.isEmpty()) { throw new AtlasBaseException(AtlasErrorCode.INSTANCE_CRUD_INVALID_PARAMS, messages); } @@ -161,6 +185,8 @@ protected void discover() throws AtlasBaseException { throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "found null entity"); } + validateAttributesForDataModel(entity); + processDynamicAttributes(entity); walkEntityGraph(entity); @@ -485,4 +511,15 @@ private void processDynamicAttributes(AtlasEntity entity) throws AtlasBaseExcept } } } + + private void validateAttributesForDataModel(AtlasEntity entity) throws AtlasBaseException { + if (entity.getTypeName().equals(Constants.ATLAS_DM_ENTITY_TYPE) || + entity.getTypeName().equals(Constants.ATLAS_DM_ATTRIBUTE_TYPE)) { + if (entity.getAttributes().get(Constants.ATLAS_DM_QUALIFIED_NAME_PREFIX) == null || + entity.getAttributes().get(Constants.ATLAS_DM_QUALIFIED_NAME_PREFIX) == "") { + throw new AtlasBaseException(AtlasErrorCode.QUALIFIED_NAME_PREFIX); + } + } + + } } From aee983a02d58a6e9446bb27c7723fb472c2b2a5a Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 18 Sep 2024 17:09:33 +0530 Subject: [PATCH 16/55] graph util to fetch latest entity/attribute --- .../store/graph/v2/AtlasGraphUtilsV2.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java index 42d30d39ca..d21a159bcd 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java @@ -51,19 +51,7 @@ import java.text.SimpleDateFormat; import java.util.*; -import static org.apache.atlas.repository.Constants.ATLAS_GLOSSARY_ENTITY_TYPE; -import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY; -import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY; -import static org.apache.atlas.repository.Constants.GLOSSARY_TERMS_EDGE_LABEL; -import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_DEFAULT; -import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_PROPERTY; -import static org.apache.atlas.repository.Constants.NAME; -import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY; -import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; -import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; -import static org.apache.atlas.repository.Constants.SUPER_TYPES_PROPERTY_KEY; -import static org.apache.atlas.repository.Constants.TYPENAME_PROPERTY_KEY; -import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graph.AtlasGraphProvider.getGraphInstance; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.DESC; @@ -708,6 +696,18 @@ public static Iterator findActiveEntityVerticesByType(AtlasGraph gr return query.vertices().iterator(); } + public static AtlasVertex findLatestEntityAttributeVerticesByType(String typename) { + AtlasGraph graph= getGraphInstance(); + AtlasGraphQuery query = graph.query() + .has(ENTITY_TYPE_PROPERTY_KEY, typename) + .has(MODEL_EXPIRED_AT_SYSTEM_DATE, 0) + .has(MODEL_EXPIRED_AT_BUSINESS_DATE, 0); + + Iterator results = query.vertices().iterator(); + AtlasVertex vertex = results.hasNext() ? results.next() : null; + return vertex; + } + public static boolean relationshipTypeHasInstanceEdges(String typeName) throws AtlasBaseException { return relationshipTypeHasInstanceEdges(getGraphInstance(), typeName); } From 08b52155182ac3c4b4709e635e3772bb23042da3 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 18 Sep 2024 17:10:13 +0530 Subject: [PATCH 17/55] unset expired dates --- .../preprocessor/model/AbstractModelPreProcessor.java | 9 +++++++-- .../v2/preprocessor/model/DMAttributePreprocessor.java | 1 + .../v2/preprocessor/model/DMEntityPreProcessor.java | 10 ++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index 15d6418f27..f8c4fa1c9c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -156,7 +156,7 @@ protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersio GraphHelper.getGuid(modelVersionVertex), GraphHelper.getTypeName(modelVersionVertex))); for (AtlasRelatedObjectId entity : existingEntities) { - if (removeEntityGuid.equals(entity.getGuid())){ + if (removeEntityGuid.equals(entity.getGuid())) { continue; } modelVersionEntityRelation.setEnd2(new AtlasObjectId( @@ -173,7 +173,7 @@ protected void createModelEntityModelAttributeRelation(AtlasVertex entity, Strin GraphHelper.getGuid(entity), GraphHelper.getTypeName(entity))); for (AtlasRelatedObjectId existingEntityAttribute : existingEntityAttributes) { - if (removeAttributeGuid.equals(existingEntityAttribute.getGuid())){ + if (removeAttributeGuid.equals(existingEntityAttribute.getGuid())) { continue; } modelEntityAttributeRelation.setEnd2( @@ -266,4 +266,9 @@ protected void applyDiffs(AtlasEntity sourceEntity, AtlasEntity destinationEntit replaceAttributes(destinationEntity.getAppendRelationshipAttributes(), diffEntity.getAppendRelationshipAttributes()); replaceAttributes(destinationEntity.getRemoveRelationshipAttributes(), diffEntity.getRemoveRelationshipAttributes()); } + + protected void unsetExpiredDates(AtlasEntity entity) { + entity.setAttribute(MODEL_EXPIRED_AT_SYSTEM_DATE, 0); + entity.setAttribute(MODEL_EXPIRED_AT_BUSINESS_DATE, 0); + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java index 2217529fc0..27df0e5627 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java @@ -145,6 +145,7 @@ private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt AtlasVertex copyAttributeVertex = modelEntityResponse.getCopyVertex(); AtlasEntity copyAttribute = modelEntityResponse.getCopyEntity(); applyDiffs(entityAttribute, copyAttribute, ATLAS_DM_ATTRIBUTE_TYPE); + unsetExpiredDates(copyAttribute); // create model-modelVersion relationship AtlasRelatedObjectId modelObject = createModelModelVersionRelation(existingModelVersionVertex, copyModelVersionVertex); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java index 2ad87873eb..8ef6406963 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -5,6 +5,7 @@ import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.*; +import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; @@ -17,6 +18,7 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,6 +26,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import static org.apache.atlas.repository.Constants.*; @@ -72,6 +75,10 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + // lets say entity has attribute : versionQualifiedName + // default/dm/1725359500/com.jpmc.ct.fri/RegulatoryReporting/entity1/epoch + // query with qualifiedName : default/dm/1725359500/com.jpmc.ct.fri/RegulatoryReporting/entity1 + if (!entity.getTypeName().equals(ATLAS_DM_ENTITY_TYPE)) { return; } @@ -85,6 +92,8 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } + + long now = Instant.now().toEpochMilli(); AtlasEntity.AtlasEntityWithExtInfo existingEntity = entityRetriever.toAtlasEntityWithExtInfo(vertex, false); AtlasRelatedObjectId existingModelVersion = (AtlasRelatedObjectId) existingEntity.getEntity().getRelationshipAttributes().get("dMVersion"); @@ -109,6 +118,7 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati AtlasVertex copyEntityVertex = modelEntityResponse.getCopyVertex(); AtlasEntity copyEntity = modelEntityResponse.getCopyEntity(); applyDiffs(entity, copyEntity, ATLAS_DM_ENTITY_TYPE); + unsetExpiredDates(copyEntity); // create model-modelVersion relation AtlasRelatedObjectId modelObject = createModelModelVersionRelation(existingModelVersionVertex, copyModelVersionVertex); From c32dc1f59cddbb148698f61ba76a11461dabdd79 Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 19 Sep 2024 13:29:40 +0530 Subject: [PATCH 18/55] Expose constats and errors for data model --- .../main/java/org/apache/atlas/repository/Constants.java | 3 +++ intg/src/main/java/org/apache/atlas/AtlasErrorCode.java | 8 +++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 3f3e7df14c..8e7e804e65 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -21,6 +21,7 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; +import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.service.FeatureFlagStore; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang3.StringUtils; @@ -212,6 +213,8 @@ public final class Constants { public static final String ATLAS_DM_QUALIFIED_NAME_PREFIX = "dMQualifiedNamePrefix"; + public static final String ATLAS_DM_NAMESPACE = "dMDataModelNamespace"; + public static String[] PROCESS_EDGE_LABELS = {PROCESS_OUTPUTS, PROCESS_INPUTS}; /** diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index 7760676f5e..55ee1ff1a0 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -295,11 +295,9 @@ public enum AtlasErrorCode { DATA_MODEL_VERSION_NOT_EXIST(400, "ATLAS-400-00-115", "Model version {0} does not exist"), DATA_ENTITY_NOT_EXIST(400, "ATLAS-400-00-116", "DataEntity {0} does not exist"), DATA_MODEL_NOT_EXIST(400, "ATLAS-400-00-117", "DataModel {0} does not exist"), - QUALIFIED_NAME_PREFIX (400, "ATLAS-400-00-118", "dMQualifiedNamePrefix is mandatory for DMEntity/DMAttribute"), - - NO_TYPE_EXISTS_FOR_QUALIFIED_NAME_PREFIX (400,"ATLAS-400-00-119", "No DMEntity/DMAttribute exists for dMQualifiedNamePrefix : {0}"); - - + QUALIFIED_NAME_PREFIX_NOT_EXIST(400, "ATLAS-400-00-118", "dMQualifiedNamePrefix is mandatory for DMEntity/DMAttribute"), + NO_TYPE_EXISTS_FOR_QUALIFIED_NAME_PREFIX (400,"ATLAS-400-00-119", "No DMEntity/DMAttribute exists for dMQualifiedNamePrefix : {0}"), + NAME_NAMESPACE_NOT_EXIST (400, "ATLAS-400-00-120", "name/namespace are mandatory for DMEntity/DMAttribute"); private String errorCode; private String errorMessage; private Response.Status httpCode; From 9834366474f7b0e499af27b99baff367dd0c95f6 Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 19 Sep 2024 13:30:47 +0530 Subject: [PATCH 19/55] update discovery flow for data models --- .../graph/v2/AtlasEntityGraphDiscoveryV2.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java index cb221ba66f..36f9deaaf0 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java @@ -42,9 +42,11 @@ import org.apache.atlas.type.TemplateToken; import org.apache.atlas.utils.AtlasEntityUtil; import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder; +import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; @@ -117,13 +119,22 @@ public void validateAndNormalize(AtlasEntity entity) throws AtlasBaseException { if (entity.getTypeName().equals(Constants.ATLAS_DM_ENTITY_TYPE) || entity.getTypeName().equals(Constants.ATLAS_DM_ATTRIBUTE_TYPE)){ - AtlasVertex vertex = AtlasGraphUtilsV2.findLatestEntityAttributeVerticesByType(entity.getTypeName()); - String guidFromVertex= AtlasGraphUtilsV2.getIdFromVertex(vertex); + String qualifiedNamePrefix = (String) entity.getAttributes().get(Constants.ATLAS_DM_QUALIFIED_NAME_PREFIX); + if (qualifiedNamePrefix.isEmpty()){ + throw new AtlasBaseException(AtlasErrorCode.QUALIFIED_NAME_PREFIX_NOT_EXIST); + } + AtlasVertex vertex = AtlasGraphUtilsV2.findLatestEntityAttributeVerticesByType(entity.getTypeName(), qualifiedNamePrefix); + + String guidFromVertex = AtlasGraphUtilsV2.getIdFromVertex(vertex); + if (guidFromVertex.isEmpty()){ - throw new AtlasBaseException(AtlasErrorCode.NO_TYPE_EXISTS_FOR_QUALIFIED_NAME_PREFIX, (String) entity.getAttributes().get(Constants.ATLAS_DM_QUALIFIED_NAME_PREFIX)); + // no entity exists with this qualifiedName, set qualifiedName and let entity be created + entity.setAttribute(Constants.QUALIFIED_NAME, qualifiedNamePrefix + Instant.now().toEpochMilli()); + return; } + // if guidFromVertex is found let entity be updated entity.setGuid(AtlasGraphUtilsV2.getIdFromVertex(vertex)); type.getNormalizedValue(entity); return; @@ -517,7 +528,7 @@ private void validateAttributesForDataModel(AtlasEntity entity) throws AtlasBase entity.getTypeName().equals(Constants.ATLAS_DM_ATTRIBUTE_TYPE)) { if (entity.getAttributes().get(Constants.ATLAS_DM_QUALIFIED_NAME_PREFIX) == null || entity.getAttributes().get(Constants.ATLAS_DM_QUALIFIED_NAME_PREFIX) == "") { - throw new AtlasBaseException(AtlasErrorCode.QUALIFIED_NAME_PREFIX); + throw new AtlasBaseException(AtlasErrorCode.QUALIFIED_NAME_PREFIX_NOT_EXIST); } } From 59f7c44cd3089783089870d8b2004bd5994dcab6 Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 19 Sep 2024 13:31:52 +0530 Subject: [PATCH 20/55] Update entity/attribute discover method --- .../atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java index d21a159bcd..56c95c39a1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java @@ -696,10 +696,11 @@ public static Iterator findActiveEntityVerticesByType(AtlasGraph gr return query.vertices().iterator(); } - public static AtlasVertex findLatestEntityAttributeVerticesByType(String typename) { + public static AtlasVertex findLatestEntityAttributeVerticesByType(String typename, String dMQualifiedNamePrefix) { AtlasGraph graph= getGraphInstance(); AtlasGraphQuery query = graph.query() .has(ENTITY_TYPE_PROPERTY_KEY, typename) + .has(ATLAS_DM_QUALIFIED_NAME_PREFIX, dMQualifiedNamePrefix) .has(MODEL_EXPIRED_AT_SYSTEM_DATE, 0) .has(MODEL_EXPIRED_AT_BUSINESS_DATE, 0); From 9365f32e5b6942c3202677b56350ac16a3d61be4 Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 19 Sep 2024 16:08:20 +0530 Subject: [PATCH 21/55] Format dm constants --- .../org/apache/atlas/repository/Constants.java | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 8e7e804e65..cd61a0fc37 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -21,7 +21,6 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; -import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.service.FeatureFlagStore; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang3.StringUtils; @@ -202,6 +201,8 @@ public final class Constants { public static final String PROCESS_OUTPUTS = "__Process.outputs"; public static final String PROCESS_INPUTS = "__Process.inputs"; + public static String[] PROCESS_EDGE_LABELS = {PROCESS_OUTPUTS, PROCESS_INPUTS}; + /*** * DataModel */ @@ -214,8 +215,11 @@ public final class Constants { public static final String ATLAS_DM_QUALIFIED_NAME_PREFIX = "dMQualifiedNamePrefix"; public static final String ATLAS_DM_NAMESPACE = "dMDataModelNamespace"; + public static final String ATLAS_DM_EXPIRED_AT_SYSTEM_DATE = "dMDataModelExpiredAtSystemDate"; + public static final String ATLAS_DM_EXPIRED_AT_BUSINESS_DATE = "dMDataModelExpiredAtBusinessDate"; + public static final String ATLAS_DM_SYSTEM_DATE = "dMDataModelSystemDate"; + public static final String ATLAS_DM_BUSINESS_DATE = "dMDataModelBusinessDate"; - public static String[] PROCESS_EDGE_LABELS = {PROCESS_OUTPUTS, PROCESS_INPUTS}; /** * The homeId field is used when saving into Atlas a copy of an object that is being imported from another @@ -388,15 +392,6 @@ public final class Constants { public static final String ASSET_POLICY_GUIDS = "assetPolicyGUIDs"; public static final String ASSET_POLICIES_COUNT = "assetPoliciesCount"; - /*** - * Model related constants - * - */ - public static final String MODEL_EXPIRED_AT_SYSTEM_DATE= "dMDataModelExpiredAtSystemDate"; - public static final String MODEL_EXPIRED_AT_BUSINESS_DATE= "dMDataModelExpiredAtBusinessDate"; - public static final String MODEL_SYSTEM_DATE= "dMDataModelSystemDate"; - public static final String MODEL_BUSINESS_DATE= "dMDataModelBusinessDate"; - /* From bf54b02d0c05a7ac2db0add125d7166629188142 Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 19 Sep 2024 16:08:52 +0530 Subject: [PATCH 22/55] fix names in utils --- .../atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java index 56c95c39a1..4dbe3629c7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java @@ -701,8 +701,8 @@ public static AtlasVertex findLatestEntityAttributeVerticesByType(String typenam AtlasGraphQuery query = graph.query() .has(ENTITY_TYPE_PROPERTY_KEY, typename) .has(ATLAS_DM_QUALIFIED_NAME_PREFIX, dMQualifiedNamePrefix) - .has(MODEL_EXPIRED_AT_SYSTEM_DATE, 0) - .has(MODEL_EXPIRED_AT_BUSINESS_DATE, 0); + .has(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE, 0) + .has(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE, 0); Iterator results = query.vertices().iterator(); AtlasVertex vertex = results.hasNext() ? results.next() : null; From d7891e1caf28fb7661622802cd8fdf75988680ef Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 19 Sep 2024 16:16:42 +0530 Subject: [PATCH 23/55] update preprocessors for update entity/attribute flow --- .../model/AbstractModelPreProcessor.java | 83 +++++++++++++------ .../model/DMAttributePreprocessor.java | 59 ++++++++----- .../model/DMEntityPreProcessor.java | 19 +++-- 3 files changed, 106 insertions(+), 55 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index f8c4fa1c9c..c8bba00aa2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -13,12 +13,8 @@ import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; -import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; -import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.type.AtlasTypeUtil; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,17 +76,17 @@ public AtlasVertex getCopyVertex() { } protected void setModelDates(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { - newEntity.setAttribute(MODEL_SYSTEM_DATE, value); - newEntity.setAttribute(MODEL_BUSINESS_DATE, value); - AtlasGraphUtilsV2.setEncodedProperty(newVertex, MODEL_SYSTEM_DATE, value); - AtlasGraphUtilsV2.setEncodedProperty(newVertex, MODEL_BUSINESS_DATE, value); + newEntity.setAttribute(ATLAS_DM_SYSTEM_DATE, value); + newEntity.setAttribute(ATLAS_DM_BUSINESS_DATE, value); + AtlasGraphUtilsV2.setEncodedProperty(newVertex, ATLAS_DM_SYSTEM_DATE, value); + AtlasGraphUtilsV2.setEncodedProperty(newVertex, ATLAS_DM_BUSINESS_DATE, value); } protected void setModelExpiredAtDates(AtlasEntity oldEntity, AtlasVertex oldVertex, Object value) { - oldEntity.setAttribute(MODEL_EXPIRED_AT_SYSTEM_DATE, value); - oldEntity.setAttribute(MODEL_EXPIRED_AT_BUSINESS_DATE, value); - AtlasGraphUtilsV2.setEncodedProperty(oldVertex, MODEL_EXPIRED_AT_SYSTEM_DATE, value); - AtlasGraphUtilsV2.setEncodedProperty(oldVertex, MODEL_EXPIRED_AT_BUSINESS_DATE, value); + oldEntity.setAttribute(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE, value); + oldEntity.setAttribute(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE, value); + AtlasGraphUtilsV2.setEncodedProperty(oldVertex, ATLAS_DM_EXPIRED_AT_SYSTEM_DATE, value); + AtlasGraphUtilsV2.setEncodedProperty(oldVertex, ATLAS_DM_EXPIRED_AT_BUSINESS_DATE, value); } protected void setQualifiedName(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { @@ -98,6 +94,11 @@ protected void setQualifiedName(AtlasEntity newEntity, AtlasVertex newVertex, Ob AtlasGraphUtilsV2.setEncodedProperty(newVertex, QUALIFIED_NAME, value); } + protected void setName(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { + newEntity.setAttribute(NAME, value); + AtlasGraphUtilsV2.setEncodedProperty(newVertex, NAME, value); + } + protected ModelResponse replicateModelVersion(AtlasRelatedObjectId relatedObjectId, long epoch) throws AtlasBaseException { AtlasEntity.AtlasEntityWithExtInfo existingModelVersionEntityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(relatedObjectId.getGuid()); AtlasVertex existingModelVersionVertex = entityRetriever.getEntityVertex(relatedObjectId.getGuid()); @@ -105,19 +106,34 @@ protected ModelResponse replicateModelVersion(AtlasRelatedObjectId relatedObject AtlasEntity copyModelVersion = entityRetriever.toAtlasEntity(copyModelVertex); copyAllAttributes(existingModelVersionEntityWithExtInfo.getEntity(), copyModelVersion, epoch); setModelDates(copyModelVersion, copyModelVertex, epoch); - String modelQualifiedName = (String) existingModelVersionEntityWithExtInfo.getEntity().getAttribute(QUALIFIED_NAME) + epoch; - setQualifiedName(copyModelVersion, copyModelVertex, modelQualifiedName); + // ex : "default/dm/1234/modelName/v1"; + String existingModelVersionQualifiedName = ((String) existingModelVersionEntityWithExtInfo.getEntity().getAttribute(QUALIFIED_NAME)); + int lastIndex = existingModelVersionQualifiedName.lastIndexOf("/"); + + // v1 + String existingVersion = existingModelVersionQualifiedName.substring(lastIndex + 1); + int existingVersionNumber = Integer.parseInt(existingVersion.substring(1)); + + // default/dm/1234/modelName + String modelName = existingModelVersionQualifiedName.substring(0, lastIndex); + + String modelVersion = "v" + (++existingVersionNumber); + + String modelVersionQualifiedName = modelName + "/" + modelVersion; + + setName(copyModelVersion, copyModelVertex, modelVersion); + setQualifiedName(copyModelVersion, copyModelVertex, modelVersionQualifiedName); setModelExpiredAtDates(existingModelVersionEntityWithExtInfo.getEntity(), existingModelVersionVertex, epoch); return new ModelResponse(existingModelVersionEntityWithExtInfo.getEntity(), copyModelVersion, existingModelVersionVertex, copyModelVertex); } - protected ModelResponse replicateModelEntity(AtlasEntity existingEntity, AtlasVertex existingEntityVertex, String modelVersionQualifiedName, long epoch) throws AtlasBaseException { + protected ModelResponse replicateModelEntity(AtlasEntity existingEntity, AtlasVertex existingEntityVertex, String entityQualifiedNamePrefix, long epoch) throws AtlasBaseException { AtlasVertex copyEntityVertex = entityGraphMapper.createVertex(existingEntity); AtlasEntity copyEntity = entityRetriever.toAtlasEntity(copyEntityVertex); copyAllAttributes(existingEntity, copyEntity, epoch); // copyEntity.setRelationshipAttributes(entity.getRelationshipAttributes()); setModelDates(copyEntity, copyEntityVertex, epoch); - String entityQualifiedName = modelVersionQualifiedName + "/" + existingEntity.getAttribute(NAME); + String entityQualifiedName = entityQualifiedNamePrefix + "/" + epoch; setQualifiedName(copyEntity, copyEntityVertex, entityQualifiedName); setModelDates(copyEntity, copyEntityVertex, epoch); setModelExpiredAtDates(existingEntity, existingEntityVertex, epoch); @@ -130,7 +146,7 @@ protected ModelResponse replicateModelAttribute(AtlasEntity existingAttribute, A copyAllAttributes(existingAttribute, copyAttributeEntity, epoch); // copyEntity.setRelationshipAttributes(entity.getRelationshipAttributes()); setModelDates(copyAttributeEntity, copyAttributeVertex, epoch); - String attributeQualifiedName = entityQualifiedName + "/" + existingAttribute.getAttribute(NAME); + String attributeQualifiedName = entityQualifiedName + "/" + existingAttribute.getAttribute(NAME) + "/" + epoch; setQualifiedName(copyAttributeEntity, copyAttributeVertex, attributeQualifiedName); setModelDates(copyAttributeEntity, copyAttributeVertex, epoch); setModelExpiredAtDates(existingAttribute, existingAttributeVertex, epoch); @@ -140,6 +156,7 @@ protected ModelResponse replicateModelAttribute(AtlasEntity existingAttribute, A protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersionVertex, AtlasVertex modelEntityVertex) throws AtlasBaseException { AtlasRelationship modelVersionEntityRelation = new AtlasRelationship("d_m_version_d_m_entities"); + modelVersionEntityRelation.setStatus(AtlasRelationship.Status.ACTIVE); modelVersionEntityRelation.setEnd1(new AtlasObjectId( GraphHelper.getGuid(modelVersionVertex), GraphHelper.getTypeName(modelVersionVertex))); @@ -149,31 +166,39 @@ protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersio atlasRelationshipStore.create(modelVersionEntityRelation); } - protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersionVertex, String removeEntityGuid, + protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersionVertex, List existingEntities) { AtlasRelationship modelVersionEntityRelation = new AtlasRelationship("d_m_version_d_m_entities"); + modelVersionEntityRelation.setStatus(AtlasRelationship.Status.ACTIVE); modelVersionEntityRelation.setEnd1(new AtlasObjectId( GraphHelper.getGuid(modelVersionVertex), GraphHelper.getTypeName(modelVersionVertex))); - for (AtlasRelatedObjectId entity : existingEntities) { - if (removeEntityGuid.equals(entity.getGuid())) { + for (AtlasRelatedObjectId existingEntity : existingEntities) { + if ( + ((int) (existingEntity.getAttributes().get(ATLAS_DM_BUSINESS_DATE)) > 0) || + ((int) (existingEntity.getAttributes().get(ATLAS_DM_SYSTEM_DATE)) > 0) + ) { continue; } modelVersionEntityRelation.setEnd2(new AtlasObjectId( - entity.getGuid(), - entity.getTypeName() + existingEntity.getGuid(), + existingEntity.getTypeName() )); } } - protected void createModelEntityModelAttributeRelation(AtlasVertex entity, String removeAttributeGuid, List existingEntityAttributes) throws AtlasBaseException { + protected void createModelEntityModelAttributeRelation(AtlasVertex entity, List existingEntityAttributes) throws AtlasBaseException { AtlasRelationship modelEntityAttributeRelation = new AtlasRelationship("d_m_entity_d_m_attributes"); + modelEntityAttributeRelation.setStatus(AtlasRelationship.Status.ACTIVE); modelEntityAttributeRelation.setEnd1( new AtlasObjectId( GraphHelper.getGuid(entity), GraphHelper.getTypeName(entity))); for (AtlasRelatedObjectId existingEntityAttribute : existingEntityAttributes) { - if (removeAttributeGuid.equals(existingEntityAttribute.getGuid())) { + if ( + ((int) (existingEntityAttribute.getAttributes().get(ATLAS_DM_BUSINESS_DATE)) > 0) || + ((int) (existingEntityAttribute.getAttributes().get(ATLAS_DM_SYSTEM_DATE)) > 0) + ) { continue; } modelEntityAttributeRelation.setEnd2( @@ -186,6 +211,7 @@ protected void createModelEntityModelAttributeRelation(AtlasVertex entity, Strin protected void createModelEntityModelAttributeRelation(AtlasVertex entity, AtlasVertex attribute) throws AtlasBaseException { AtlasRelationship modelEntityAttributeRelation = new AtlasRelationship("d_m_entity_d_m_attributes"); + modelEntityAttributeRelation.setStatus(AtlasRelationship.Status.ACTIVE); modelEntityAttributeRelation.setEnd1( new AtlasObjectId( GraphHelper.getGuid(entity), @@ -204,6 +230,7 @@ protected AtlasRelatedObjectId createModelModelVersionRelation(AtlasVertex exist throw new AtlasBaseException(AtlasErrorCode.DATA_MODEL_NOT_EXIST); } AtlasRelationship modelVersionModelRelation = new AtlasRelationship("d_m_data_model_d_m_versions"); + modelVersionModelRelation.setStatus(AtlasRelationship.Status.ACTIVE); modelVersionModelRelation.setEnd1( new AtlasObjectId( existingModel.getGuid(), @@ -267,8 +294,10 @@ protected void applyDiffs(AtlasEntity sourceEntity, AtlasEntity destinationEntit replaceAttributes(destinationEntity.getRemoveRelationshipAttributes(), diffEntity.getRemoveRelationshipAttributes()); } - protected void unsetExpiredDates(AtlasEntity entity) { - entity.setAttribute(MODEL_EXPIRED_AT_SYSTEM_DATE, 0); - entity.setAttribute(MODEL_EXPIRED_AT_BUSINESS_DATE, 0); + protected void unsetExpiredDates(AtlasEntity latestEntity, AtlasVertex latestVertex) { + latestEntity.setAttribute(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE, 0); + latestEntity.setAttribute(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE, 0); + AtlasGraphUtilsV2.setEncodedProperty(latestVertex, ATLAS_DM_EXPIRED_AT_SYSTEM_DATE, 0); + AtlasGraphUtilsV2.setEncodedProperty(latestVertex, ATLAS_DM_SYSTEM_DATE, 0); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java index 27df0e5627..c16bf74395 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java @@ -6,7 +6,6 @@ import org.apache.atlas.model.instance.AtlasRelatedObjectId; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.EntityMutations; -import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; @@ -57,16 +56,14 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAttribute, EntityMutationContext context) throws AtlasBaseException { - if (!entityAttribute.getTypeName().equals(ATLAS_DM_ATTRIBUTE_TYPE)) { - return; - } - if (CollectionUtils.isEmpty(context.getUpdatedEntities())) { + if (!entityAttribute.getTypeName().equals(ATLAS_DM_ATTRIBUTE_TYPE) || + CollectionUtils.isEmpty(context.getUpdatedEntities())) { return; } - String entityName = (String) entityAttribute.getAttribute(NAME); + String attributeName = (String) entityAttribute.getAttribute(NAME); - if (StringUtils.isEmpty(entityName) || isNameInvalid(entityName)) { + if (StringUtils.isEmpty(attributeName) || isNameInvalid(attributeName)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } @@ -81,10 +78,11 @@ private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt AtlasRelatedObjectId existingEntityObject = null; // retrieve entity where expiredAtBusinessDate and expiredAtSystemDate is not set + // there will always be only one active entity for (AtlasRelatedObjectId _existingEntityObject : existingEntityObjects) { - long expiredBusinessDate = (long) entityRetriever.toAtlasEntity(_existingEntityObject.getGuid()).getAttribute(MODEL_EXPIRED_AT_BUSINESS_DATE); - long expiredSystemDate = (long) entityRetriever.toAtlasEntity(_existingEntityObject.getGuid()).getAttribute(MODEL_EXPIRED_AT_SYSTEM_DATE); - if (expiredBusinessDate > 0 && expiredSystemDate > 0) { + long expiredBusinessDate = (long) entityRetriever.toAtlasEntity(_existingEntityObject.getGuid()).getAttribute(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE); + long expiredSystemDate = (long) entityRetriever.toAtlasEntity(_existingEntityObject.getGuid()).getAttribute(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE); + if (expiredBusinessDate <= 0 && expiredSystemDate <= 0) { existingEntityObject = _existingEntityObject; } } @@ -93,6 +91,7 @@ private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt throw new AtlasBaseException(AtlasErrorCode.DATA_ENTITY_NOT_EXIST); } + AtlasEntity.AtlasEntityWithExtInfo existingEntityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(existingEntityObject.getGuid()); AtlasRelatedObjectId existingModelVersionObject = (AtlasRelatedObjectId) existingEntityWithExtInfo.getEntity().getRelationshipAttributes().get("dMVersion"); @@ -112,27 +111,39 @@ private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt AtlasEntity copyModelVersion = modelVersionResponse.getCopyEntity(); AtlasVertex existingModelVersionVertex = modelVersionResponse.getExistingVertex(); AtlasVertex copyModelVersionVertex = modelVersionResponse.getCopyVertex(); - String modelVersionQualifiedName = (String) copyModelVersion.getAttribute(QUALIFIED_NAME); + // String modelVersionQualifiedName = (String) copyModelVersion.getAttribute(QUALIFIED_NAME); - // retrieve entities linked to previous version + // retrieve active entities linked to previous version existingEntityObjects = (List) existingModelVersionEntity.getRelationshipAttributes().get("dMEntities"); + List activeEntities = new ArrayList<>(); for (AtlasRelatedObjectId _existingEntityObject : existingEntityObjects) { AtlasEntity entity = entityRetriever.toAtlasEntity(_existingEntityObject.getGuid()); - if ((long) entity.getAttributes().get(MODEL_EXPIRED_AT_BUSINESS_DATE) > 0 && (long) entity.getAttributes().get(MODEL_EXPIRED_AT_BUSINESS_DATE) > 0) { - activeEntities.add(entity); + if ((long) entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE) > 0 && (long) entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE) > 0) { + continue; } + activeEntities.add(entity); } + + // ex: default/dm/1234/modelName/entityName/attributeName + String qualifiedAttributeNamePrefix = (String) entityAttribute.getAttributes().get(ATLAS_DM_QUALIFIED_NAME_PREFIX); + int lastIndex = qualifiedAttributeNamePrefix.lastIndexOf("/"); + + // ex: default/dm/1234/modelName/entityName + String qualifiedEntityNamePrefix = qualifiedAttributeNamePrefix.substring(0, lastIndex); + + // create entity e1 ---> e1' ModelResponse modelEntityResponse = replicateModelEntity( existingEntityWithExtInfo.getEntity(), entityRetriever.getEntityVertex(existingModelVersionEntity.getGuid()), - modelVersionQualifiedName, + qualifiedEntityNamePrefix, now); + AtlasEntity existingEntity= modelEntityResponse.getExistingEntity(); + AtlasVertex existingEntityVertex = modelEntityResponse.getExistingVertex(); AtlasVertex copyEntityVertex = modelEntityResponse.getCopyVertex(); AtlasEntity copyEntity = modelEntityResponse.getCopyEntity(); - String entityQualifiedName = (String) copyModelVersion.getAttribute(QUALIFIED_NAME); List existingEntityAttributes = (List) modelEntityResponse.getExistingEntity().getRelationshipAttributes().get("dMAttributes"); @@ -140,12 +151,13 @@ private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt ModelResponse modelAttributeResponse = replicateModelAttribute( existingEntityAttributeWithExtInfo.getEntity(), entityRetriever.getEntityVertex(entityAttribute.getGuid()), - entityQualifiedName, + qualifiedAttributeNamePrefix, now); - AtlasVertex copyAttributeVertex = modelEntityResponse.getCopyVertex(); - AtlasEntity copyAttribute = modelEntityResponse.getCopyEntity(); + + AtlasVertex copyAttributeVertex = modelAttributeResponse.getCopyVertex(); + AtlasEntity copyAttribute = modelAttributeResponse.getCopyEntity(); applyDiffs(entityAttribute, copyAttribute, ATLAS_DM_ATTRIBUTE_TYPE); - unsetExpiredDates(copyAttribute); + unsetExpiredDates(copyAttribute, copyAttributeVertex); // create model-modelVersion relationship AtlasRelatedObjectId modelObject = createModelModelVersionRelation(existingModelVersionVertex, copyModelVersionVertex); @@ -162,7 +174,7 @@ private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt createModelEntityModelAttributeRelation(copyEntityVertex, copyAttributeVertex); // create entity- attribute relation [with existing attributes] - createModelEntityModelAttributeRelation(copyEntityVertex, entityAttribute.getGuid(), existingEntityAttributes); + createModelEntityModelAttributeRelation(copyEntityVertex, existingEntityAttributes); AtlasEntityType attributeType = typeRegistry.getEntityTypeByName(entityAttribute.getTypeName()); AtlasEntityType entityType = typeRegistry.getEntityTypeByName(copyEntity.getTypeName()); @@ -172,6 +184,11 @@ private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt context.addCreated(copyEntity.getGuid(), copyEntity, entityType, copyEntityVertex); context.addCreated(copyModelVersion.getGuid(), copyModelVersion, modelVersionType, copyModelVersionVertex); + + context.addUpdated(existingModelVersionEntity.getGuid(), existingModelVersionEntity, + modelVersionType, existingModelVersionVertex); + context.addUpdated(existingEntity.getGuid(), existingEntity, + entityType, existingEntityVertex); // remove existing entity from context so it is not updated context.removeUpdated(entityAttribute.getGuid(), entityAttribute, entityType, vertexAttribute); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java index 8ef6406963..dfbc891c9b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -5,6 +5,7 @@ import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.*; +import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; @@ -93,7 +94,6 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati } - long now = Instant.now().toEpochMilli(); AtlasEntity.AtlasEntityWithExtInfo existingEntity = entityRetriever.toAtlasEntityWithExtInfo(vertex, false); AtlasRelatedObjectId existingModelVersion = (AtlasRelatedObjectId) existingEntity.getEntity().getRelationshipAttributes().get("dMVersion"); @@ -109,16 +109,17 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati AtlasEntity copyModelVersion = modelVersionResponse.getCopyEntity(); AtlasVertex existingModelVersionVertex = modelVersionResponse.getExistingVertex(); AtlasVertex copyModelVersionVertex = modelVersionResponse.getCopyVertex(); - String modelVersionAttributeQualifiedName = (String) copyModelVersion.getAttribute(QUALIFIED_NAME); + // String modelVersionAttributeQualifiedName = (String) copyModelVersion.getAttribute(QUALIFIED_NAME); List existingEntities = (List) existingModelVersionEntity.getRelationshipAttributes().get("dMEntities"); + String entityQualifiedNamePrefix = (String) entity.getAttributes().get(ATLAS_DM_QUALIFIED_NAME_PREFIX); // create entity e1 ---> e1' - ModelResponse modelEntityResponse = replicateModelEntity(existingEntity.getEntity(), vertex, modelVersionAttributeQualifiedName, now); + ModelResponse modelEntityResponse = replicateModelEntity(existingEntity.getEntity(), vertex, entityQualifiedNamePrefix, now); AtlasVertex copyEntityVertex = modelEntityResponse.getCopyVertex(); AtlasEntity copyEntity = modelEntityResponse.getCopyEntity(); applyDiffs(entity, copyEntity, ATLAS_DM_ENTITY_TYPE); - unsetExpiredDates(copyEntity); + unsetExpiredDates(copyEntity, copyEntityVertex); // create model-modelVersion relation AtlasRelatedObjectId modelObject = createModelModelVersionRelation(existingModelVersionVertex, copyModelVersionVertex); @@ -126,11 +127,11 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati // create modelVersion-modelEntity relationship with new entity createModelVersionModelEntityRelationship(copyModelVersionVertex, copyEntityVertex); - // create modelVersion-modelEntity relation with old entities - createModelVersionModelEntityRelationship(copyModelVersionVertex, entity.getGuid(), existingEntities); + // create modelVersion-modelEntity relation with old entities which are not expired + createModelVersionModelEntityRelationship(copyModelVersionVertex, existingEntities); // create modelEntity-modelAttributeRelationship - createModelEntityModelAttributeRelation(copyEntityVertex, "", existingEntityAttributes); + createModelEntityModelAttributeRelation(copyEntityVertex, existingEntityAttributes); /** @@ -143,6 +144,10 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati context.addCreated(copyEntity.getGuid(), copyEntity, entityType, copyEntityVertex); context.addCreated(copyModelVersion.getGuid(), copyModelVersion, modelVersionType, copyModelVersionVertex); + context.addUpdated(entity.getGuid(), entity, entityType, vertex); + context.addUpdated(existingModelVersionEntity.getGuid(), existingModelVersionEntity, + modelVersionType, existingModelVersionVertex ); + // remove existing entity from context so it is not updated context.removeUpdated(entity.getGuid(), entity, entityType, vertex); From d5e696c863e51b096ab0cc8dd0f60c03af20d7b4 Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 19 Sep 2024 16:27:27 +0530 Subject: [PATCH 24/55] fix qualifiedName nomenclature --- .../v2/preprocessor/model/AbstractModelPreProcessor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index c8bba00aa2..0f156022c8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -133,20 +133,20 @@ protected ModelResponse replicateModelEntity(AtlasEntity existingEntity, AtlasVe copyAllAttributes(existingEntity, copyEntity, epoch); // copyEntity.setRelationshipAttributes(entity.getRelationshipAttributes()); setModelDates(copyEntity, copyEntityVertex, epoch); - String entityQualifiedName = entityQualifiedNamePrefix + "/" + epoch; + String entityQualifiedName = entityQualifiedNamePrefix + "_" + epoch; setQualifiedName(copyEntity, copyEntityVertex, entityQualifiedName); setModelDates(copyEntity, copyEntityVertex, epoch); setModelExpiredAtDates(existingEntity, existingEntityVertex, epoch); return new ModelResponse(existingEntity, copyEntity, existingEntityVertex, copyEntityVertex); } - protected ModelResponse replicateModelAttribute(AtlasEntity existingAttribute, AtlasVertex existingAttributeVertex, String entityQualifiedName, long epoch) throws AtlasBaseException { + protected ModelResponse replicateModelAttribute(AtlasEntity existingAttribute, AtlasVertex existingAttributeVertex, String attributeQualifiedNamePrefix, long epoch) throws AtlasBaseException { AtlasVertex copyAttributeVertex = entityGraphMapper.createVertex(existingAttribute); AtlasEntity copyAttributeEntity = entityRetriever.toAtlasEntity(copyAttributeVertex); copyAllAttributes(existingAttribute, copyAttributeEntity, epoch); // copyEntity.setRelationshipAttributes(entity.getRelationshipAttributes()); setModelDates(copyAttributeEntity, copyAttributeVertex, epoch); - String attributeQualifiedName = entityQualifiedName + "/" + existingAttribute.getAttribute(NAME) + "/" + epoch; + String attributeQualifiedName = attributeQualifiedNamePrefix + "_" + epoch; setQualifiedName(copyAttributeEntity, copyAttributeVertex, attributeQualifiedName); setModelDates(copyAttributeEntity, copyAttributeVertex, epoch); setModelExpiredAtDates(existingAttribute, existingAttributeVertex, epoch); From 9a1153e1aec7b2810088c7a6076b2447464a94a8 Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 19 Sep 2024 21:28:14 +0530 Subject: [PATCH 25/55] update constants --- common/src/main/java/org/apache/atlas/repository/Constants.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index cd61a0fc37..47a01bf08f 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -212,6 +212,8 @@ public final class Constants { public static final String ATLAS_DM_ATTRIBUTE_TYPE = "DMAttribute"; public static final String ATLAS_DM_DATA_MODEL = "DMDataModel"; + public static final String ATLAS_DM_VERSION_TYPE = "DMVersion"; + public static final String ATLAS_DM_QUALIFIED_NAME_PREFIX = "dMQualifiedNamePrefix"; public static final String ATLAS_DM_NAMESPACE = "dMDataModelNamespace"; From 6f76993eb7e997831395f00d311cef36ec2c099f Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 19 Sep 2024 21:29:40 +0530 Subject: [PATCH 26/55] update DM entity flow --- .../model/AbstractModelPreProcessor.java | 41 +++++++++ .../model/DMAttributePreprocessor.java | 3 +- .../model/DMEntityPreProcessor.java | 85 +++++++++++++++++-- 3 files changed, 120 insertions(+), 9 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index 0f156022c8..e8a8fd5981 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -7,14 +7,18 @@ import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasRelatedObjectId; import org.apache.atlas.model.instance.AtlasRelationship; +import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,6 +62,11 @@ protected ModelResponse(AtlasEntity existingEntity, AtlasEntity copyEntity, this.copyVertex = copyVertex; } + protected ModelResponse(AtlasEntity copyEntity, AtlasVertex copyVertex) { + this.copyEntity = copyEntity; + this.copyVertex = copyVertex; + } + public AtlasEntity getExistingEntity() { return existingEntity; } @@ -99,6 +108,21 @@ protected void setName(AtlasEntity newEntity, AtlasVertex newVertex, Object valu AtlasGraphUtilsV2.setEncodedProperty(newVertex, NAME, value); } + + protected ModelResponse createModelVersion(String modelQualifiedName, String modelVersion, String namespace, EntityMutationContext context) throws AtlasBaseException { + String guid = UUID.randomUUID().toString(); + AtlasEntity modelVersionEntity = new AtlasEntity(ATLAS_DM_VERSION_TYPE); + modelVersionEntity.setAttribute(VERSION_PROPERTY_KEY, 0); + modelVersionEntity.setAttribute(QUALIFIED_NAME, modelQualifiedName + "/" + modelVersion); + modelVersionEntity.setAttribute(ATLAS_DM_NAMESPACE, namespace); + modelVersionEntity.setAttribute(ATLAS_DM_BUSINESS_DATE, RequestContext.get().getRequestTime()); + modelVersionEntity.setAttribute(ATLAS_DM_SYSTEM_DATE, RequestContext.get().getRequestTime()); + AtlasVertex versionVertex = entityGraphMapper.createVertexWithGuid(modelVersionEntity, guid); + context.getDiscoveryContext().addResolvedGuid(guid, versionVertex); + modelVersionEntity.setGuid(guid); + context.addCreated(guid, modelVersionEntity, typeRegistry.getEntityTypeByName(ATLAS_DM_VERSION_TYPE), versionVertex); + return new ModelResponse(modelVersionEntity, versionVertex); + } protected ModelResponse replicateModelVersion(AtlasRelatedObjectId relatedObjectId, long epoch) throws AtlasBaseException { AtlasEntity.AtlasEntityWithExtInfo existingModelVersionEntityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(relatedObjectId.getGuid()); AtlasVertex existingModelVersionVertex = entityRetriever.getEntityVertex(relatedObjectId.getGuid()); @@ -168,6 +192,9 @@ protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersio protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersionVertex, List existingEntities) { + if (CollectionUtils.isEmpty(existingEntities)){ + return; + } AtlasRelationship modelVersionEntityRelation = new AtlasRelationship("d_m_version_d_m_entities"); modelVersionEntityRelation.setStatus(AtlasRelationship.Status.ACTIVE); modelVersionEntityRelation.setEnd1(new AtlasObjectId( @@ -188,6 +215,9 @@ protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersio } protected void createModelEntityModelAttributeRelation(AtlasVertex entity, List existingEntityAttributes) throws AtlasBaseException { + if (CollectionUtils.isEmpty(existingEntityAttributes)){ + return; + } AtlasRelationship modelEntityAttributeRelation = new AtlasRelationship("d_m_entity_d_m_attributes"); modelEntityAttributeRelation.setStatus(AtlasRelationship.Status.ACTIVE); modelEntityAttributeRelation.setEnd1( @@ -223,6 +253,17 @@ protected void createModelEntityModelAttributeRelation(AtlasVertex entity, Atlas atlasRelationshipStore.create(modelEntityAttributeRelation); } + + protected void createModelModelVersionRelation(String modelGuid, String latestModelVersionGuid) throws AtlasBaseException { + AtlasRelationship modelVersionModelRelation = new AtlasRelationship("d_m_data_model_d_m_versions"); + modelVersionModelRelation.setStatus(AtlasRelationship.Status.ACTIVE); + modelVersionModelRelation.setEnd1( + new AtlasObjectId( + modelGuid, ATLAS_DM_DATA_MODEL)); + modelVersionModelRelation.setEnd2( + new AtlasObjectId( latestModelVersionGuid, ATLAS_DM_VERSION_TYPE)); + atlasRelationshipStore.create(modelVersionModelRelation); + } protected AtlasRelatedObjectId createModelModelVersionRelation(AtlasVertex existingModelVersionVertex, AtlasVertex latestModelVersionVertex) throws AtlasBaseException { AtlasEntity.AtlasEntityWithExtInfo existingModelVersionExtInfo = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersionVertex, false); AtlasRelatedObjectId existingModel = (AtlasRelatedObjectId) existingModelVersionExtInfo.getEntity().getRelationshipAttributes().get("dMDataModel"); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java index c16bf74395..231db38337 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java @@ -1,6 +1,7 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.model; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasRelatedObjectId; @@ -67,7 +68,7 @@ private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } - long now = Instant.now().toEpochMilli(); + long now = RequestContext.get().getRequestTime(); AtlasEntity.AtlasEntityWithExtInfo existingEntityAttributeWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(vertexAttribute, false); List existingEntityObjects = (List) existingEntityAttributeWithExtInfo.getEntity().getRelationshipAttributes().get("dMEntities"); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java index dfbc891c9b..ea51820d1a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -25,10 +25,7 @@ import org.slf4j.LoggerFactory; import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; +import java.util.*; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; @@ -70,8 +67,80 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co } } - private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) { - return; + private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + + if (!entity.getTypeName().equals(ATLAS_DM_ENTITY_TYPE)) { + return; + } + if (CollectionUtils.isEmpty(context.getCreatedEntities())) { + return; + } + + String entityName = (String) entity.getAttribute(NAME); + + if (StringUtils.isEmpty(entityName) || isNameInvalid(entityName)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); + } + long now = RequestContext.get().getRequestTime(); + + // get model qualifiedName with qualifiedNamePrefix + String qualifiedNamePrefix = (String) entity.getAttributes().get(ATLAS_DM_QUALIFIED_NAME_PREFIX); + int lastIndex = qualifiedNamePrefix.lastIndexOf("/"); + String modelQualifiedName = qualifiedNamePrefix.substring(0, lastIndex); + + String modelGuid = AtlasGraphUtilsV2.getGuidByUniqueAttributes( + typeRegistry.getEntityTypeByName(ATLAS_DM_DATA_MODEL), + (Map) new HashMap().put(QUALIFIED_NAME, modelQualifiedName)); + + AtlasEntity.AtlasEntityWithExtInfo dataModel = entityRetriever.toAtlasEntityWithExtInfo(modelGuid, false); + List existingModelVersions = (List) dataModel.getEntity().getAttributes().get("dMVersions"); + + String modelVersion = "v1"; + AtlasRelatedObjectId existingModelVersionObj = null; + + if (CollectionUtils.isNotEmpty(existingModelVersions)) { + int existingVersionNumber = existingModelVersions.size(); + modelVersion = "v" + (++existingVersionNumber); + + for (AtlasRelatedObjectId modelVersionObj : existingModelVersions) { + if (((int) modelVersionObj.getAttributes().get(ATLAS_DM_BUSINESS_DATE) > 0) || + ((int) modelVersionObj.getAttributes().get(ATLAS_DM_SYSTEM_DATE) > 0)) { + continue; + } + existingModelVersionObj = modelVersionObj; + } + } + + AtlasEntity existingModelVersionEntity = null; + AtlasVertex existingModelVersionVertex = null; + List existingEntities = null; + + if (existingModelVersionObj != null) { + existingModelVersionEntity = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersionObj.getGuid()).getEntity(); + existingModelVersionVertex = entityRetriever.getEntityVertex(existingModelVersionObj.getGuid()); + setModelExpiredAtDates(existingModelVersionEntity, existingModelVersionVertex, now); + existingEntities= (List) existingModelVersionEntity.getRelationshipAttributes().get("dMEntities"); + } + + String namespace = (String) entity.getAttributes().get(ATLAS_DM_NAMESPACE); + ModelResponse modelVersionResponse = createModelVersion(modelQualifiedName, modelVersion, namespace, context); + AtlasEntity latestModelVersionEntity = modelVersionResponse.getCopyEntity(); + AtlasVertex latestModelVersionVertex = modelVersionResponse.getCopyVertex(); + + // model --- modelVersion relation + createModelModelVersionRelation(modelGuid,latestModelVersionEntity.getGuid()); + + // modelVersion --- entity relation + createModelVersionModelEntityRelationship(latestModelVersionVertex, vertex); + + // modelVersion --- entitiesOfExistingModelVersion + createModelVersionModelEntityRelationship(latestModelVersionVertex, existingEntities); + + // resolve references + context.getDiscoveryContext(). + addResolvedGuid( + modelGuid, + entityRetriever.getEntityVertex(modelGuid)); } @@ -94,9 +163,9 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati } - long now = Instant.now().toEpochMilli(); + long now = RequestContext.get().getRequestTime(); AtlasEntity.AtlasEntityWithExtInfo existingEntity = entityRetriever.toAtlasEntityWithExtInfo(vertex, false); - AtlasRelatedObjectId existingModelVersion = (AtlasRelatedObjectId) existingEntity.getEntity().getRelationshipAttributes().get("dMVersion"); + AtlasRelatedObjectId existingModelVersion = (AtlasRelatedObjectId) existingEntity.getEntity().getRelationshipAttributes().get("dMVersions"); List existingEntityAttributes = (List) existingEntity.getEntity().getRelationshipAttributes().get("dMAttributes"); if (existingModelVersion == null) { From f24168ed7c0f61c90aacb38fa0d303704046ad86 Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 20 Sep 2024 01:14:46 +0530 Subject: [PATCH 27/55] Update create logic for dmEntity and dmAttribute --- .../model/AbstractModelPreProcessor.java | 23 +-- .../model/DMAttributePreprocessor.java | 132 +++++++++++++++++- .../model/DMEntityPreProcessor.java | 17 ++- 3 files changed, 154 insertions(+), 18 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index e8a8fd5981..2c4f31e80e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -109,19 +109,20 @@ protected void setName(AtlasEntity newEntity, AtlasVertex newVertex, Object valu } - protected ModelResponse createModelVersion(String modelQualifiedName, String modelVersion, String namespace, EntityMutationContext context) throws AtlasBaseException { + + protected ModelResponse createEntity(String qualifiedName,String entityType, String namespace, EntityMutationContext context) throws AtlasBaseException { String guid = UUID.randomUUID().toString(); - AtlasEntity modelVersionEntity = new AtlasEntity(ATLAS_DM_VERSION_TYPE); - modelVersionEntity.setAttribute(VERSION_PROPERTY_KEY, 0); - modelVersionEntity.setAttribute(QUALIFIED_NAME, modelQualifiedName + "/" + modelVersion); - modelVersionEntity.setAttribute(ATLAS_DM_NAMESPACE, namespace); - modelVersionEntity.setAttribute(ATLAS_DM_BUSINESS_DATE, RequestContext.get().getRequestTime()); - modelVersionEntity.setAttribute(ATLAS_DM_SYSTEM_DATE, RequestContext.get().getRequestTime()); - AtlasVertex versionVertex = entityGraphMapper.createVertexWithGuid(modelVersionEntity, guid); + AtlasEntity entity = new AtlasEntity(entityType); + entity.setAttribute(VERSION_PROPERTY_KEY, 0); + entity.setAttribute(QUALIFIED_NAME, qualifiedName); + entity.setAttribute(ATLAS_DM_NAMESPACE, namespace); + entity.setAttribute(ATLAS_DM_BUSINESS_DATE, RequestContext.get().getRequestTime()); + entity.setAttribute(ATLAS_DM_SYSTEM_DATE, RequestContext.get().getRequestTime()); + AtlasVertex versionVertex = entityGraphMapper.createVertexWithGuid(entity, guid); context.getDiscoveryContext().addResolvedGuid(guid, versionVertex); - modelVersionEntity.setGuid(guid); - context.addCreated(guid, modelVersionEntity, typeRegistry.getEntityTypeByName(ATLAS_DM_VERSION_TYPE), versionVertex); - return new ModelResponse(modelVersionEntity, versionVertex); + entity.setGuid(guid); + context.addCreated(guid, entity, typeRegistry.getEntityTypeByName(entityType), versionVertex); + return new ModelResponse(entity, versionVertex); } protected ModelResponse replicateModelVersion(AtlasRelatedObjectId relatedObjectId, long epoch) throws AtlasBaseException { AtlasEntity.AtlasEntityWithExtInfo existingModelVersionEntityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(relatedObjectId.getGuid()); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java index 231db38337..a5e4a69951 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java @@ -9,6 +9,7 @@ import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; +import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; @@ -21,7 +22,9 @@ import java.time.Instant; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; @@ -55,6 +58,127 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co } } + private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAttribute, EntityMutationContext context) throws AtlasBaseException { + if (!entityAttribute.getTypeName().equals(ATLAS_DM_ATTRIBUTE_TYPE)) { + return; + } + if (CollectionUtils.isEmpty(context.getCreatedEntities())) { + return; + } + + String entityName = (String) entityAttribute.getAttribute(NAME); + + if (StringUtils.isEmpty(entityName) || isNameInvalid(entityName)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); + } + long now = RequestContext.get().getRequestTime(); + + + // get entity qualifiedName with qualifiedNamePrefix + String attributeQualifiedNamePrefix = (String) entityAttribute.getAttributes().get(ATLAS_DM_QUALIFIED_NAME_PREFIX); + int lastIndex = attributeQualifiedNamePrefix.lastIndexOf("/"); + String entityQualifiedNamePrefix = attributeQualifiedNamePrefix.substring(0, lastIndex); + String namespace = (String) entityAttribute.getAttributes().get(ATLAS_DM_NAMESPACE); + ModelResponse modelENtityResponse = null; + AtlasVertex latestEntityVertex = AtlasGraphUtilsV2.findLatestEntityAttributeVerticesByType(ATLAS_DM_ENTITY_TYPE, entityQualifiedNamePrefix); + + // get model qualifiedName with qualifiedNamePrefix + + lastIndex = entityQualifiedNamePrefix.lastIndexOf("/"); + String modelQualifiedName = entityQualifiedNamePrefix.substring(0, lastIndex); + + String modelGuid = AtlasGraphUtilsV2.getGuidByUniqueAttributes( + typeRegistry.getEntityTypeByName(ATLAS_DM_DATA_MODEL), + (Map) new HashMap().put(QUALIFIED_NAME, modelQualifiedName)); + + List existingAttributes = null; + + if (latestEntityVertex != null) { + modelENtityResponse = replicateModelEntity( + entityRetriever.toAtlasEntity(latestEntityVertex), + latestEntityVertex, + entityQualifiedNamePrefix, + now + ); + existingAttributes = (List) modelENtityResponse.getExistingEntity().getAttributes().get("dMAttributes"); + } else { + modelENtityResponse = createEntity( + attributeQualifiedNamePrefix + "_" + now, + ATLAS_DM_ATTRIBUTE_TYPE, + namespace, + context + ); + } + + + AtlasEntity.AtlasEntityWithExtInfo dataModel = entityRetriever.toAtlasEntityWithExtInfo(modelGuid, false); + List existingModelVersions = (List) dataModel.getEntity().getAttributes().get("dMVersions"); + + String modelVersion = "v2"; + AtlasRelatedObjectId existingModelVersionObj = null; + + + if (CollectionUtils.isNotEmpty(existingModelVersions)) { + int existingVersionNumber = existingModelVersions.size(); + modelVersion = "v" + (++existingVersionNumber); + + for (AtlasRelatedObjectId modelVersionObj : existingModelVersions) { + if (((int) modelVersionObj.getAttributes().get(ATLAS_DM_BUSINESS_DATE) > 0) || + ((int) modelVersionObj.getAttributes().get(ATLAS_DM_SYSTEM_DATE) > 0)) { + continue; + } + existingModelVersionObj = modelVersionObj; + } + } + + AtlasEntity existingModelVersionEntity = null; + AtlasVertex existingModelVersionVertex = null; + List existingEntities = null; + ModelResponse modelVersionResponse = null; + + if (existingModelVersionObj != null) { + existingModelVersionEntity = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersionObj.getGuid()).getEntity(); + existingModelVersionVertex = entityRetriever.getEntityVertex(existingModelVersionObj.getGuid()); + setModelExpiredAtDates(existingModelVersionEntity, existingModelVersionVertex, now); + existingEntities = (List) existingModelVersionEntity.getRelationshipAttributes().get("dMEntities"); + modelVersionResponse = replicateModelVersion(existingModelVersionObj, now); + } else { + modelVersionResponse = createEntity( + (modelQualifiedName + "/" + modelVersion), + ATLAS_DM_VERSION_TYPE, + namespace, + context); + } + + AtlasEntity latestModelVersionEntity = modelVersionResponse.getCopyEntity(); + AtlasVertex latestModelVersionVertex = modelVersionResponse.getCopyVertex(); + + + // model --- modelVersion relation + createModelModelVersionRelation(modelGuid, latestModelVersionEntity.getGuid()); + + // modelVersion --- entity relation + createModelVersionModelEntityRelationship(latestModelVersionVertex, modelENtityResponse.getCopyVertex()); + + // modelVersion --- entitiesOfExistingModelVersion + createModelVersionModelEntityRelationship(latestModelVersionVertex, existingEntities); + + // entity --- attributes of existingEntity relation + createModelEntityModelAttributeRelation(modelENtityResponse.getCopyVertex(), existingAttributes); + + + context.addCreated(latestModelVersionEntity.getGuid(), latestModelVersionEntity, + typeRegistry.getEntityTypeByName(ATLAS_DM_VERSION_TYPE), latestModelVersionVertex); + + context.addCreated(modelENtityResponse.getCopyEntity().getGuid(), modelENtityResponse.getCopyEntity(), + typeRegistry.getEntityTypeByName(ATLAS_DM_ENTITY_TYPE), modelENtityResponse.getCopyVertex()); + + // resolve references + context.getDiscoveryContext(). + addResolvedGuid( + modelGuid, + entityRetriever.getEntityVertex(modelGuid)); + } private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAttribute, EntityMutationContext context) throws AtlasBaseException { if (!entityAttribute.getTypeName().equals(ATLAS_DM_ATTRIBUTE_TYPE) || @@ -112,7 +236,7 @@ private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt AtlasEntity copyModelVersion = modelVersionResponse.getCopyEntity(); AtlasVertex existingModelVersionVertex = modelVersionResponse.getExistingVertex(); AtlasVertex copyModelVersionVertex = modelVersionResponse.getCopyVertex(); - // String modelVersionQualifiedName = (String) copyModelVersion.getAttribute(QUALIFIED_NAME); + // String modelVersionQualifiedName = (String) copyModelVersion.getAttribute(QUALIFIED_NAME); // retrieve active entities linked to previous version existingEntityObjects = (List) existingModelVersionEntity.getRelationshipAttributes().get("dMEntities"); @@ -121,7 +245,7 @@ private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt for (AtlasRelatedObjectId _existingEntityObject : existingEntityObjects) { AtlasEntity entity = entityRetriever.toAtlasEntity(_existingEntityObject.getGuid()); if ((long) entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE) > 0 && (long) entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE) > 0) { - continue; + continue; } activeEntities.add(entity); } @@ -141,7 +265,7 @@ private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt entityRetriever.getEntityVertex(existingModelVersionEntity.getGuid()), qualifiedEntityNamePrefix, now); - AtlasEntity existingEntity= modelEntityResponse.getExistingEntity(); + AtlasEntity existingEntity = modelEntityResponse.getExistingEntity(); AtlasVertex existingEntityVertex = modelEntityResponse.getExistingVertex(); AtlasVertex copyEntityVertex = modelEntityResponse.getCopyVertex(); AtlasEntity copyEntity = modelEntityResponse.getCopyEntity(); @@ -174,7 +298,7 @@ private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt // create entity - attribute relation [with new attribute] createModelEntityModelAttributeRelation(copyEntityVertex, copyAttributeVertex); - // create entity- attribute relation [with existing attributes] + // create entity - attribute relation [with existing attributes] createModelEntityModelAttributeRelation(copyEntityVertex, existingEntityAttributes); AtlasEntityType attributeType = typeRegistry.getEntityTypeByName(entityAttribute.getTypeName()); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java index ea51820d1a..322a824d0e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -68,7 +68,6 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co } private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { - if (!entity.getTypeName().equals(ATLAS_DM_ENTITY_TYPE)) { return; } @@ -114,19 +113,28 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati AtlasEntity existingModelVersionEntity = null; AtlasVertex existingModelVersionVertex = null; List existingEntities = null; + ModelResponse modelVersionResponse = null; + if (existingModelVersionObj != null) { existingModelVersionEntity = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersionObj.getGuid()).getEntity(); existingModelVersionVertex = entityRetriever.getEntityVertex(existingModelVersionObj.getGuid()); setModelExpiredAtDates(existingModelVersionEntity, existingModelVersionVertex, now); existingEntities= (List) existingModelVersionEntity.getRelationshipAttributes().get("dMEntities"); + modelVersionResponse = replicateModelVersion(existingModelVersionObj, now); + } else { + String namespace = (String) entity.getAttributes().get(ATLAS_DM_NAMESPACE); + modelVersionResponse = createEntity( + (modelQualifiedName + "/" + modelVersion), + ATLAS_DM_VERSION_TYPE, + namespace, + context); } - String namespace = (String) entity.getAttributes().get(ATLAS_DM_NAMESPACE); - ModelResponse modelVersionResponse = createModelVersion(modelQualifiedName, modelVersion, namespace, context); AtlasEntity latestModelVersionEntity = modelVersionResponse.getCopyEntity(); AtlasVertex latestModelVersionVertex = modelVersionResponse.getCopyVertex(); + // model --- modelVersion relation createModelModelVersionRelation(modelGuid,latestModelVersionEntity.getGuid()); @@ -136,6 +144,9 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati // modelVersion --- entitiesOfExistingModelVersion createModelVersionModelEntityRelationship(latestModelVersionVertex, existingEntities); + + context.addCreated(latestModelVersionEntity.getGuid(), latestModelVersionEntity, + typeRegistry.getEntityTypeByName(ATLAS_DM_VERSION_TYPE), latestModelVersionVertex); // resolve references context.getDiscoveryContext(). addResolvedGuid( From 4b3b692e370378082beecc58e160d92ba1d0cdd5 Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 20 Sep 2024 13:53:59 +0530 Subject: [PATCH 28/55] Fix conditions --- .../model/AbstractModelPreProcessor.java | 17 ++++++++--------- .../model/DMAttributePreprocessor.java | 6 ++++-- .../model/DMEntityPreProcessor.java | 13 +++++++------ 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index 2c4f31e80e..9ab7ec0e0f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -192,7 +192,7 @@ protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersio } protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersionVertex, - List existingEntities) { + List existingEntities) throws AtlasBaseException { if (CollectionUtils.isEmpty(existingEntities)){ return; } @@ -202,9 +202,10 @@ protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersio GraphHelper.getGuid(modelVersionVertex), GraphHelper.getTypeName(modelVersionVertex))); for (AtlasRelatedObjectId existingEntity : existingEntities) { + AtlasEntity entity = entityRetriever.toAtlasEntity(existingEntity.getGuid()); if ( - ((int) (existingEntity.getAttributes().get(ATLAS_DM_BUSINESS_DATE)) > 0) || - ((int) (existingEntity.getAttributes().get(ATLAS_DM_SYSTEM_DATE)) > 0) + ((int) (entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE)) > 0) || + ((int) (entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE)) > 0) ) { continue; } @@ -226,9 +227,10 @@ protected void createModelEntityModelAttributeRelation(AtlasVertex entity, List< GraphHelper.getGuid(entity), GraphHelper.getTypeName(entity))); for (AtlasRelatedObjectId existingEntityAttribute : existingEntityAttributes) { + AtlasEntity entityAttribute = entityRetriever.toAtlasEntity(existingEntityAttribute.getGuid()); if ( - ((int) (existingEntityAttribute.getAttributes().get(ATLAS_DM_BUSINESS_DATE)) > 0) || - ((int) (existingEntityAttribute.getAttributes().get(ATLAS_DM_SYSTEM_DATE)) > 0) + ((int) (entityAttribute.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE)) > 0) || + ((int) (entityAttribute.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE)) > 0) ) { continue; } @@ -331,15 +333,12 @@ protected void applyDiffs(AtlasEntity sourceEntity, AtlasEntity destinationEntit return; } replaceAttributes(destinationEntity.getAttributes(), diffEntity.getAttributes()); - replaceAttributes(destinationEntity.getRelationshipAttributes(), diffEntity.getRelationshipAttributes()); - replaceAttributes(destinationEntity.getAppendRelationshipAttributes(), diffEntity.getAppendRelationshipAttributes()); - replaceAttributes(destinationEntity.getRemoveRelationshipAttributes(), diffEntity.getRemoveRelationshipAttributes()); } protected void unsetExpiredDates(AtlasEntity latestEntity, AtlasVertex latestVertex) { latestEntity.setAttribute(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE, 0); latestEntity.setAttribute(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE, 0); AtlasGraphUtilsV2.setEncodedProperty(latestVertex, ATLAS_DM_EXPIRED_AT_SYSTEM_DATE, 0); - AtlasGraphUtilsV2.setEncodedProperty(latestVertex, ATLAS_DM_SYSTEM_DATE, 0); + AtlasGraphUtilsV2.setEncodedProperty(latestVertex, ATLAS_DM_EXPIRED_AT_BUSINESS_DATE, 0); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java index a5e4a69951..91bb1845f7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java @@ -52,6 +52,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co switch (operation) { case CREATE: + createDMAttribute(entity, vertex, context); break; case UPDATE: updateDMAttribute(entity, vertex, context); @@ -166,6 +167,8 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt // entity --- attributes of existingEntity relation createModelEntityModelAttributeRelation(modelENtityResponse.getCopyVertex(), existingAttributes); + // latest entity ---- new attribute relation + createModelEntityModelAttributeRelation(modelENtityResponse.getCopyVertex(), vertexAttribute); context.addCreated(latestModelVersionEntity.getGuid(), latestModelVersionEntity, typeRegistry.getEntityTypeByName(ATLAS_DM_VERSION_TYPE), latestModelVersionVertex); @@ -181,8 +184,7 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt } private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAttribute, EntityMutationContext context) throws AtlasBaseException { - if (!entityAttribute.getTypeName().equals(ATLAS_DM_ATTRIBUTE_TYPE) || - CollectionUtils.isEmpty(context.getUpdatedEntities())) { + if (!entityAttribute.getTypeName().equals(ATLAS_DM_ATTRIBUTE_TYPE)) { return; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java index 322a824d0e..1d8bc10bd3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -102,8 +102,8 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati modelVersion = "v" + (++existingVersionNumber); for (AtlasRelatedObjectId modelVersionObj : existingModelVersions) { - if (((int) modelVersionObj.getAttributes().get(ATLAS_DM_BUSINESS_DATE) > 0) || - ((int) modelVersionObj.getAttributes().get(ATLAS_DM_SYSTEM_DATE) > 0)) { + if (((int) modelVersionObj.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE) > 0) || + ((int) modelVersionObj.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE) > 0)) { continue; } existingModelVersionObj = modelVersionObj; @@ -155,6 +155,10 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati } + private void updateDMEntities(){ + + } + private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { // lets say entity has attribute : versionQualifiedName // default/dm/1725359500/com.jpmc.ct.fri/RegulatoryReporting/entity1/epoch @@ -163,9 +167,6 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati if (!entity.getTypeName().equals(ATLAS_DM_ENTITY_TYPE)) { return; } - if (CollectionUtils.isEmpty(context.getUpdatedEntities())) { - return; - } String entityName = (String) entity.getAttribute(NAME); @@ -176,7 +177,7 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati long now = RequestContext.get().getRequestTime(); AtlasEntity.AtlasEntityWithExtInfo existingEntity = entityRetriever.toAtlasEntityWithExtInfo(vertex, false); - AtlasRelatedObjectId existingModelVersion = (AtlasRelatedObjectId) existingEntity.getEntity().getRelationshipAttributes().get("dMVersions"); + AtlasRelatedObjectId existingModelVersion = (AtlasRelatedObjectId) existingEntity.getEntity().getRelationshipAttributes().get("dMVersion"); List existingEntityAttributes = (List) existingEntity.getEntity().getRelationshipAttributes().get("dMAttributes"); if (existingModelVersion == null) { From 01ba178d55b4c7077e21c10b20b246f31275312c Mon Sep 17 00:00:00 2001 From: aarshi Date: Sun, 22 Sep 2024 19:24:06 +0530 Subject: [PATCH 29/55] CU flow for entity. attribute --- .../graph/v2/AtlasEntityGraphDiscoveryV2.java | 17 +- .../store/graph/v2/AtlasEntityStoreV2.java | 26 +++ .../store/graph/v2/EntityMutationContext.java | 4 + .../store/graph/v2/IDBasedEntityResolver.java | 29 ++- .../model/AbstractModelPreProcessor.java | 156 ++++++++----- .../model/DMAttributePreprocessor.java | 211 +++++++----------- .../model/DMEntityPreProcessor.java | 187 ++++++++++------ 7 files changed, 360 insertions(+), 270 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java index 36f9deaaf0..669d7819ed 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java @@ -107,15 +107,15 @@ public void validateAndNormalize(AtlasEntity entity) throws AtlasBaseException { validateLabels(entity.getLabels()); - // entity is user input + type.validateValue(entity, entity.getTypeName(), messages); - // set guid here + // DMEntity and DMAttributeType are requested for update // by dMQualifiedNamePrefix which is not a unique attribute // This can return multiple entity/attribute that match this prefix vale - // we have to return latest entity/attribute + // we have to return latest entity/attribute if (entity.getTypeName().equals(Constants.ATLAS_DM_ENTITY_TYPE) || entity.getTypeName().equals(Constants.ATLAS_DM_ATTRIBUTE_TYPE)){ @@ -123,14 +123,11 @@ public void validateAndNormalize(AtlasEntity entity) throws AtlasBaseException { if (qualifiedNamePrefix.isEmpty()){ throw new AtlasBaseException(AtlasErrorCode.QUALIFIED_NAME_PREFIX_NOT_EXIST); } - AtlasVertex vertex = AtlasGraphUtilsV2.findLatestEntityAttributeVerticesByType(entity.getTypeName(), qualifiedNamePrefix); - - String guidFromVertex = AtlasGraphUtilsV2.getIdFromVertex(vertex); - - - if (guidFromVertex.isEmpty()){ + // AtlasVertex vertex = AtlasGraphUtilsV2.findLatestEntityAttributeVerticesByType(entity.getTypeName(), qualifiedNamePrefix); + AtlasVertex vertex= discoveryContext.getResolvedEntityVertex(entity.getGuid()); + if (vertex == null) { // no entity exists with this qualifiedName, set qualifiedName and let entity be created - entity.setAttribute(Constants.QUALIFIED_NAME, qualifiedNamePrefix + Instant.now().toEpochMilli()); + entity.setAttribute(Constants.QUALIFIED_NAME, qualifiedNamePrefix + "_" + RequestContext.get().getRequestTime()); return; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 441ecd342e..71b810cc79 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1587,6 +1587,24 @@ private void executePreProcessor(EntityMutationContext context) throws AtlasBase processor.processAttributes(entity, context, UPDATE); } } + + List copyOfAppendRelationshipAttributes = new ArrayList<>(context.getUpdatedEntitiesForAppendRelationshipAttribute()); + for (AtlasEntity entity: copyOfAppendRelationshipAttributes) { + entityType = context.getType(entity.getGuid()); + preProcessors = getPreProcessor(entityType.getTypeName()); + for(PreProcessor processor : preProcessors){ + processor.processAttributes(entity, context, UPDATE); + } + } + + List copyOfRemoveRelationshipAttributes = new ArrayList<>(context.getEntitiesUpdatedWithRemoveRelationshipAttribute()); + for (AtlasEntity entity: copyOfRemoveRelationshipAttributes) { + entityType = context.getType(entity.getGuid()); + preProcessors = getPreProcessor(entityType.getTypeName()); + for(PreProcessor processor : preProcessors){ + processor.processAttributes(entity, context, UPDATE); + } + } } private EntityMutationContext preCreateOrUpdate(EntityStream entityStream, EntityGraphMapper entityGraphMapper, boolean isPartialUpdate) throws AtlasBaseException { @@ -1600,11 +1618,15 @@ private EntityMutationContext preCreateOrUpdate(EntityStream entityStream, Entit Map referencedGuids = discoveryContext.getReferencedGuids(); for (Map.Entry element : referencedGuids.entrySet()) { String guid = element.getKey(); + + // guid negative + // entity == userInput AtlasEntity entity = entityStream.getByGuid(guid); if (entity != null) { // entity would be null if guid is not in the stream but referenced by an entity in the stream AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); + // entity pass in user request with attributes if (entityType == null) { throw new AtlasBaseException(element.getValue(), AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName()); } @@ -1642,8 +1664,11 @@ private EntityMutationContext preCreateOrUpdate(EntityStream entityStream, Entit context.addUpdated(guid, entity, entityType, vertex); } else { + graphDiscoverer.validateAndNormalize(entity); + + // Handle create flow here //Create vertices which do not exist in the repository if (RequestContext.get().isImportInProgress() && AtlasTypeUtil.isAssignedGuid(entity.getGuid())) { vertex = entityGraphMapper.createVertexWithGuid(entity, entity.getGuid()); @@ -1945,6 +1970,7 @@ private AtlasVertex getResolvedEntityVertex(EntityGraphDiscoveryContext context, AtlasObjectId objectId = getAtlasObjectId(entity); AtlasVertex ret = context.getResolvedEntityVertex(entity.getGuid()); + if (ret != null) { context.addResolvedIdByUniqAttribs(objectId, ret); if (entity.getLabels() != null) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java index 76f4ed4880..0bf14635b3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java @@ -97,6 +97,10 @@ public void removeUpdated(String internalGuid, AtlasEntity entity, AtlasEntityTy } } + public void removeUpdatedWithRelationshipAttributes(AtlasEntity entity){ + entitiesUpdatedWithAppendRelationshipAttribute.remove(entity); + } + public void addEntityToRestore(AtlasVertex vertex) { if (entitiesToRestore == null) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IDBasedEntityResolver.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IDBasedEntityResolver.java index d6a2d717d9..788be55ce7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IDBasedEntityResolver.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IDBasedEntityResolver.java @@ -22,6 +22,7 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext; @@ -69,7 +70,33 @@ public EntityGraphDiscoveryContext resolveEntityReferences(EntityGraphDiscoveryC throw new AtlasBaseException(element.getValue(), AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName()); } - vertex = AtlasGraphUtilsV2.findByUniqueAttributes(this.graph, entityType, entity.getAttributes()); + // ------- + + if ( + ((entity.getAttributes().get(Constants.QUALIFIED_NAME) == null) && (entity.getAttributes().get(Constants.ATLAS_DM_QUALIFIED_NAME_PREFIX)!=null)) + && + ((entity.getTypeName().equals(Constants.ATLAS_DM_ENTITY_TYPE)) || (entity.getTypeName().equals(Constants.ATLAS_DM_ATTRIBUTE_TYPE)))) { + + String qualifiedNamePrefix = (String) entity.getAttributes().get(Constants.ATLAS_DM_QUALIFIED_NAME_PREFIX); + if (qualifiedNamePrefix.isEmpty()){ + throw new AtlasBaseException(AtlasErrorCode.QUALIFIED_NAME_PREFIX_NOT_EXIST); + } + vertex = AtlasGraphUtilsV2.findLatestEntityAttributeVerticesByType(entity.getTypeName(), qualifiedNamePrefix); + + if (vertex == null) { + // no entity exists with this qualifiedName, set qualifiedName and let entity be created + entity.setAttribute(Constants.QUALIFIED_NAME, qualifiedNamePrefix + "_" + RequestContext.get().getRequestTime()); + return context; + } + + // if guidFromVertex is found let entity be updated + // entity.setGuid(AtlasGraphUtilsV2.getIdFromVertex(vertex)); + // else find qualifiedName and set qualifiedName : as it is mandatory + context.addResolvedGuid(guid, vertex); + }else { + vertex = AtlasGraphUtilsV2.findByUniqueAttributes(this.graph, entityType, entity.getAttributes()); + } + } else if (!isAssignedGuid) { // for local-guids, entity must be in the stream throw new AtlasBaseException(element.getValue(), AtlasErrorCode.REFERENCED_ENTITY_NOT_FOUND, guid); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index 9ab7ec0e0f..5e330abae3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -19,7 +19,9 @@ import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.ListUtils; import org.apache.commons.collections.MapUtils; +import org.mockito.internal.util.collections.ListUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -100,7 +102,7 @@ protected void setModelExpiredAtDates(AtlasEntity oldEntity, AtlasVertex oldVert protected void setQualifiedName(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { newEntity.setAttribute(QUALIFIED_NAME, value); - AtlasGraphUtilsV2.setEncodedProperty(newVertex, QUALIFIED_NAME, value); + // AtlasGraphUtilsV2.setEncodedProperty(newVertex, QUALIFIED_NAME, value); } protected void setName(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { @@ -109,8 +111,7 @@ protected void setName(AtlasEntity newEntity, AtlasVertex newVertex, Object valu } - - protected ModelResponse createEntity(String qualifiedName,String entityType, String namespace, EntityMutationContext context) throws AtlasBaseException { + protected ModelResponse createEntity(String qualifiedName, String entityType, String namespace, EntityMutationContext context) throws AtlasBaseException { String guid = UUID.randomUUID().toString(); AtlasEntity entity = new AtlasEntity(entityType); entity.setAttribute(VERSION_PROPERTY_KEY, 0); @@ -124,32 +125,48 @@ protected ModelResponse createEntity(String qualifiedName,String entityType, Str context.addCreated(guid, entity, typeRegistry.getEntityTypeByName(entityType), versionVertex); return new ModelResponse(entity, versionVertex); } - protected ModelResponse replicateModelVersion(AtlasRelatedObjectId relatedObjectId, long epoch) throws AtlasBaseException { - AtlasEntity.AtlasEntityWithExtInfo existingModelVersionEntityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(relatedObjectId.getGuid()); - AtlasVertex existingModelVersionVertex = entityRetriever.getEntityVertex(relatedObjectId.getGuid()); - AtlasVertex copyModelVertex = entityGraphMapper.createVertex(existingModelVersionEntityWithExtInfo.getEntity()); - AtlasEntity copyModelVersion = entityRetriever.toAtlasEntity(copyModelVertex); - copyAllAttributes(existingModelVersionEntityWithExtInfo.getEntity(), copyModelVersion, epoch); - setModelDates(copyModelVersion, copyModelVertex, epoch); - // ex : "default/dm/1234/modelName/v1"; - String existingModelVersionQualifiedName = ((String) existingModelVersionEntityWithExtInfo.getEntity().getAttribute(QUALIFIED_NAME)); - int lastIndex = existingModelVersionQualifiedName.lastIndexOf("/"); - // v1 - String existingVersion = existingModelVersionQualifiedName.substring(lastIndex + 1); - int existingVersionNumber = Integer.parseInt(existingVersion.substring(1)); + protected ModelResponse replicateModelVersion(String modelGuid, String modelQualifiedName, long now) throws AtlasBaseException { + AtlasEntity.AtlasEntityWithExtInfo dataModel = entityRetriever.toAtlasEntityWithExtInfo(modelGuid, false); + List existingModelVersions = (List) dataModel.getEntity().getRelationshipAttributes().get("dMVersions"); + + String modelVersion = "v1"; + AtlasRelatedObjectId existingModelVersionObj = null; + + if (CollectionUtils.isEmpty(existingModelVersions)) { + return new ModelResponse(null, null); + } + + int existingVersionNumber = existingModelVersions.size(); + modelVersion = "v" + (++existingVersionNumber); - // default/dm/1234/modelName - String modelName = existingModelVersionQualifiedName.substring(0, lastIndex); + // get active model version + for (AtlasRelatedObjectId modelVersionObj : existingModelVersions) { + AtlasEntity modelVersionEntity = entityRetriever.toAtlasEntity(modelVersionObj.getGuid()); - String modelVersion = "v" + (++existingVersionNumber); + if (modelVersionEntity.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE) != null || + modelVersionEntity.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE) != null) { + continue; + } + existingModelVersionObj = modelVersionObj; + } + + if (existingModelVersionObj == null) { + throw new AtlasBaseException(AtlasErrorCode.DATA_MODEL_VERSION_NOT_EXIST); + } - String modelVersionQualifiedName = modelName + "/" + modelVersion; + AtlasEntity existingModelVersionEntity = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersionObj.getGuid()).getEntity(); + AtlasVertex existingModelVersionVertex = entityRetriever.getEntityVertex(existingModelVersionObj.getGuid()); + + AtlasVertex copyModelVertex = entityGraphMapper.createVertex(existingModelVersionEntity); + AtlasEntity copyModelVersion = entityRetriever.toAtlasEntity(copyModelVertex); + copyAllAttributes(existingModelVersionEntity, copyModelVersion, now); + setModelDates(copyModelVersion, copyModelVertex, now); setName(copyModelVersion, copyModelVertex, modelVersion); - setQualifiedName(copyModelVersion, copyModelVertex, modelVersionQualifiedName); - setModelExpiredAtDates(existingModelVersionEntityWithExtInfo.getEntity(), existingModelVersionVertex, epoch); - return new ModelResponse(existingModelVersionEntityWithExtInfo.getEntity(), copyModelVersion, existingModelVersionVertex, copyModelVertex); + setQualifiedName(copyModelVersion, copyModelVertex, modelQualifiedName + "/" + modelVersion); + setModelExpiredAtDates(existingModelVersionEntity, existingModelVersionVertex, now); + return new ModelResponse(existingModelVersionEntity, copyModelVersion, existingModelVersionVertex, copyModelVertex); } protected ModelResponse replicateModelEntity(AtlasEntity existingEntity, AtlasVertex existingEntityVertex, String entityQualifiedNamePrefix, long epoch) throws AtlasBaseException { @@ -171,7 +188,7 @@ protected ModelResponse replicateModelAttribute(AtlasEntity existingAttribute, A copyAllAttributes(existingAttribute, copyAttributeEntity, epoch); // copyEntity.setRelationshipAttributes(entity.getRelationshipAttributes()); setModelDates(copyAttributeEntity, copyAttributeVertex, epoch); - String attributeQualifiedName = attributeQualifiedNamePrefix + "_" + epoch; + String attributeQualifiedName = attributeQualifiedNamePrefix + "_" + epoch; setQualifiedName(copyAttributeEntity, copyAttributeVertex, attributeQualifiedName); setModelDates(copyAttributeEntity, copyAttributeVertex, epoch); setModelExpiredAtDates(existingAttribute, existingAttributeVertex, epoch); @@ -193,7 +210,7 @@ protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersio protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersionVertex, List existingEntities) throws AtlasBaseException { - if (CollectionUtils.isEmpty(existingEntities)){ + if (CollectionUtils.isEmpty(existingEntities)) { return; } AtlasRelationship modelVersionEntityRelation = new AtlasRelationship("d_m_version_d_m_entities"); @@ -204,8 +221,8 @@ protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersio for (AtlasRelatedObjectId existingEntity : existingEntities) { AtlasEntity entity = entityRetriever.toAtlasEntity(existingEntity.getGuid()); if ( - ((int) (entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE)) > 0) || - ((int) (entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE)) > 0) + (entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE) != null) || + (entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE) != null) ) { continue; } @@ -217,7 +234,7 @@ protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersio } protected void createModelEntityModelAttributeRelation(AtlasVertex entity, List existingEntityAttributes) throws AtlasBaseException { - if (CollectionUtils.isEmpty(existingEntityAttributes)){ + if (CollectionUtils.isEmpty(existingEntityAttributes)) { return; } AtlasRelationship modelEntityAttributeRelation = new AtlasRelationship("d_m_entity_d_m_attributes"); @@ -229,8 +246,8 @@ protected void createModelEntityModelAttributeRelation(AtlasVertex entity, List< for (AtlasRelatedObjectId existingEntityAttribute : existingEntityAttributes) { AtlasEntity entityAttribute = entityRetriever.toAtlasEntity(existingEntityAttribute.getGuid()); if ( - ((int) (entityAttribute.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE)) > 0) || - ((int) (entityAttribute.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE)) > 0) + (entityAttribute.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE) != null) || + (entityAttribute.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE) != null) ) { continue; } @@ -264,41 +281,59 @@ protected void createModelModelVersionRelation(String modelGuid, String latestMo new AtlasObjectId( modelGuid, ATLAS_DM_DATA_MODEL)); modelVersionModelRelation.setEnd2( - new AtlasObjectId( latestModelVersionGuid, ATLAS_DM_VERSION_TYPE)); + new AtlasObjectId(latestModelVersionGuid, ATLAS_DM_VERSION_TYPE)); atlasRelationshipStore.create(modelVersionModelRelation); } - protected AtlasRelatedObjectId createModelModelVersionRelation(AtlasVertex existingModelVersionVertex, AtlasVertex latestModelVersionVertex) throws AtlasBaseException { - AtlasEntity.AtlasEntityWithExtInfo existingModelVersionExtInfo = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersionVertex, false); - AtlasRelatedObjectId existingModel = (AtlasRelatedObjectId) existingModelVersionExtInfo.getEntity().getRelationshipAttributes().get("dMDataModel"); - if (existingModel == null) { - throw new AtlasBaseException(AtlasErrorCode.DATA_MODEL_NOT_EXIST); - } - AtlasRelationship modelVersionModelRelation = new AtlasRelationship("d_m_data_model_d_m_versions"); - modelVersionModelRelation.setStatus(AtlasRelationship.Status.ACTIVE); - modelVersionModelRelation.setEnd1( - new AtlasObjectId( - existingModel.getGuid(), - existingModel.getTypeName())); - modelVersionModelRelation.setEnd2( - new AtlasObjectId( - GraphHelper.getGuid(latestModelVersionVertex), - GraphHelper.getTypeName(latestModelVersionVertex))); - atlasRelationshipStore.create(modelVersionModelRelation); - return existingModel; - } protected void copyAllAttributes(AtlasEntity source, AtlasEntity destination, long epochNow) { - destination.setAttributes(source.getAttributes()); - destination.setMeanings(source.getMeanings()); - destination.setCreateTime(new Date(epochNow)); - destination.setUpdateTime(new Date(epochNow)); - // destination.setRelationshipAttributes(source.getRelationshipAttributes()); - destination.setCustomAttributes(source.getCustomAttributes()); - destination.setClassifications(source.getClassifications()); - destination.setAppendRelationshipAttributes(source.getAppendRelationshipAttributes()); - destination.setRemoveRelationshipAttributes(source.getRemoveRelationshipAttributes()); + if (source == null || destination == null) { + throw new IllegalArgumentException("Source and destination entities must not be null."); + } + + if (source.getAttributes() != null) { + destination.setAttributes(new HashMap<>(source.getAttributes())); + } else { + destination.setAttributes(new HashMap<>()); + } + + + if (CollectionUtils.isNotEmpty(source.getMeanings())) { + destination.setMeanings(new ArrayList<>(source.getMeanings())); + } else { + destination.setMeanings(new ArrayList<>()); + } + + long requestTime = RequestContext.get().getRequestTime(); + destination.setCreateTime(new Date(requestTime)); + destination.setUpdateTime(new Date(requestTime)); + + + if (source.getCustomAttributes() != null) { + destination.setCustomAttributes(new HashMap<>(source.getCustomAttributes())); + } else { + destination.setCustomAttributes(new HashMap<>()); // Set empty map if source custom attributes are null + } + + if (CollectionUtils.isNotEmpty(source.getClassifications())) { + destination.setClassifications(new ArrayList<>(source.getClassifications())); + } else { + destination.setClassifications(new ArrayList<>()); // Set empty list if source classifications are null or empty + } + + if (source.getAppendRelationshipAttributes() != null) { + destination.setAppendRelationshipAttributes(new HashMap<>(source.getAppendRelationshipAttributes())); + } else { + destination.setAppendRelationshipAttributes(new HashMap<>()); // Set empty map if source append relationship attributes are null + } + + if (source.getRemoveRelationshipAttributes() != null) { + destination.setRemoveRelationshipAttributes(new HashMap<>(source.getRemoveRelationshipAttributes())); + } else { + destination.setRemoveRelationshipAttributes(new HashMap<>()); + } } + public static void replaceAttributes(Map existingAttributes, Map diffAttributes) { if (MapUtils.isEmpty(diffAttributes)) { return; @@ -328,6 +363,9 @@ public static void replaceAttributes(Map existingAttributes, Map protected void applyDiffs(AtlasEntity sourceEntity, AtlasEntity destinationEntity, String typeName) { RequestContext reqContext = RequestContext.get(); AtlasEntity diffEntity = reqContext.getDifferentialEntity(sourceEntity.getGuid()); + if (diffEntity == null) { + return; + } boolean diffExistsForSameType = diffEntity.getTypeName().equals(typeName); if (!diffExistsForSameType) { return; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java index 91bb1845f7..3c9d8cb1f2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java @@ -21,10 +21,7 @@ import org.slf4j.LoggerFactory; import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; @@ -80,17 +77,19 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt int lastIndex = attributeQualifiedNamePrefix.lastIndexOf("/"); String entityQualifiedNamePrefix = attributeQualifiedNamePrefix.substring(0, lastIndex); String namespace = (String) entityAttribute.getAttributes().get(ATLAS_DM_NAMESPACE); + String modelVersion = "v2"; + ModelResponse modelENtityResponse = null; AtlasVertex latestEntityVertex = AtlasGraphUtilsV2.findLatestEntityAttributeVerticesByType(ATLAS_DM_ENTITY_TYPE, entityQualifiedNamePrefix); - - // get model qualifiedName with qualifiedNamePrefix - lastIndex = entityQualifiedNamePrefix.lastIndexOf("/"); + String modelQualifiedName = entityQualifiedNamePrefix.substring(0, lastIndex); + Map attrValues = new HashMap<>(); + attrValues.put(QUALIFIED_NAME, modelQualifiedName); String modelGuid = AtlasGraphUtilsV2.getGuidByUniqueAttributes( typeRegistry.getEntityTypeByName(ATLAS_DM_DATA_MODEL), - (Map) new HashMap().put(QUALIFIED_NAME, modelQualifiedName)); + attrValues); List existingAttributes = null; @@ -101,7 +100,10 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt entityQualifiedNamePrefix, now ); - existingAttributes = (List) modelENtityResponse.getExistingEntity().getAttributes().get("dMAttributes"); + + if (modelENtityResponse.getExistingEntity() != null && modelENtityResponse.getExistingEntity().getRelationshipAttributes() != null) { + existingAttributes = (List) modelENtityResponse.getExistingEntity().getRelationshipAttributes().get("dMAttributes"); + } } else { modelENtityResponse = createEntity( attributeQualifiedNamePrefix + "_" + now, @@ -111,46 +113,15 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt ); } - - AtlasEntity.AtlasEntityWithExtInfo dataModel = entityRetriever.toAtlasEntityWithExtInfo(modelGuid, false); - List existingModelVersions = (List) dataModel.getEntity().getAttributes().get("dMVersions"); - - String modelVersion = "v2"; - AtlasRelatedObjectId existingModelVersionObj = null; - - - if (CollectionUtils.isNotEmpty(existingModelVersions)) { - int existingVersionNumber = existingModelVersions.size(); - modelVersion = "v" + (++existingVersionNumber); - - for (AtlasRelatedObjectId modelVersionObj : existingModelVersions) { - if (((int) modelVersionObj.getAttributes().get(ATLAS_DM_BUSINESS_DATE) > 0) || - ((int) modelVersionObj.getAttributes().get(ATLAS_DM_SYSTEM_DATE) > 0)) { - continue; - } - existingModelVersionObj = modelVersionObj; - } - } - - AtlasEntity existingModelVersionEntity = null; - AtlasVertex existingModelVersionVertex = null; List existingEntities = null; - ModelResponse modelVersionResponse = null; - - if (existingModelVersionObj != null) { - existingModelVersionEntity = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersionObj.getGuid()).getEntity(); - existingModelVersionVertex = entityRetriever.getEntityVertex(existingModelVersionObj.getGuid()); - setModelExpiredAtDates(existingModelVersionEntity, existingModelVersionVertex, now); - existingEntities = (List) existingModelVersionEntity.getRelationshipAttributes().get("dMEntities"); - modelVersionResponse = replicateModelVersion(existingModelVersionObj, now); - } else { + ModelResponse modelVersionResponse = replicateModelVersion(modelGuid, modelQualifiedName, now); + if (modelVersionResponse.getCopyEntity() == null) { modelVersionResponse = createEntity( (modelQualifiedName + "/" + modelVersion), ATLAS_DM_VERSION_TYPE, namespace, context); } - AtlasEntity latestModelVersionEntity = modelVersionResponse.getCopyEntity(); AtlasVertex latestModelVersionVertex = modelVersionResponse.getCopyVertex(); @@ -162,7 +133,10 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt createModelVersionModelEntityRelationship(latestModelVersionVertex, modelENtityResponse.getCopyVertex()); // modelVersion --- entitiesOfExistingModelVersion - createModelVersionModelEntityRelationship(latestModelVersionVertex, existingEntities); + if (modelVersionResponse.getExistingEntity() != null && modelVersionResponse.getExistingEntity().getRelationshipAttributes() != null) { + existingEntities = (List) modelVersionResponse.getExistingEntity().getRelationshipAttributes().get("dMEntities"); + createModelVersionModelEntityRelationship(latestModelVersionVertex, existingEntities); + } // entity --- attributes of existingEntity relation createModelEntityModelAttributeRelation(modelENtityResponse.getCopyVertex(), existingAttributes); @@ -181,6 +155,7 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt addResolvedGuid( modelGuid, entityRetriever.getEntityVertex(modelGuid)); + } private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAttribute, EntityMutationContext context) throws AtlasBaseException { @@ -196,89 +171,73 @@ private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt long now = RequestContext.get().getRequestTime(); - AtlasEntity.AtlasEntityWithExtInfo existingEntityAttributeWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(vertexAttribute, false); - List existingEntityObjects = (List) existingEntityAttributeWithExtInfo.getEntity().getRelationshipAttributes().get("dMEntities"); - - if (CollectionUtils.isEmpty(existingEntityObjects)) { - throw new AtlasBaseException(AtlasErrorCode.DATA_ENTITY_NOT_EXIST); - } - - AtlasRelatedObjectId existingEntityObject = null; - // retrieve entity where expiredAtBusinessDate and expiredAtSystemDate is not set - // there will always be only one active entity - for (AtlasRelatedObjectId _existingEntityObject : existingEntityObjects) { - long expiredBusinessDate = (long) entityRetriever.toAtlasEntity(_existingEntityObject.getGuid()).getAttribute(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE); - long expiredSystemDate = (long) entityRetriever.toAtlasEntity(_existingEntityObject.getGuid()).getAttribute(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE); - if (expiredBusinessDate <= 0 && expiredSystemDate <= 0) { - existingEntityObject = _existingEntityObject; - } - } - if (existingEntityObject == null) { - throw new AtlasBaseException(AtlasErrorCode.DATA_ENTITY_NOT_EXIST); - } + // get entity qualifiedName with qualifiedNamePrefix + String attributeQualifiedNamePrefix = (String) entityAttribute.getAttributes().get(ATLAS_DM_QUALIFIED_NAME_PREFIX); + int lastIndex = attributeQualifiedNamePrefix.lastIndexOf("/"); + String entityQualifiedNamePrefix = attributeQualifiedNamePrefix.substring(0, lastIndex); + String namespace = (String) entityAttribute.getAttributes().get(ATLAS_DM_NAMESPACE); + String modelVersion = "v2"; + ModelResponse modelENtityResponse = null; + AtlasVertex latestEntityVertex = AtlasGraphUtilsV2.findLatestEntityAttributeVerticesByType(ATLAS_DM_ENTITY_TYPE, entityQualifiedNamePrefix); - AtlasEntity.AtlasEntityWithExtInfo existingEntityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(existingEntityObject.getGuid()); - AtlasRelatedObjectId existingModelVersionObject = (AtlasRelatedObjectId) existingEntityWithExtInfo.getEntity().getRelationshipAttributes().get("dMVersion"); + // get model qualifiedName with qualifiedNamePrefix + lastIndex = entityQualifiedNamePrefix.lastIndexOf("/"); + String modelQualifiedName = entityQualifiedNamePrefix.substring(0, lastIndex); + Map attrValues = new HashMap<>(); + attrValues.put(QUALIFIED_NAME, modelQualifiedName); - if (existingModelVersionObject == null) { - throw new AtlasBaseException(AtlasErrorCode.DATA_MODEL_VERSION_NOT_EXIST); - } - AtlasEntity.AtlasEntityWithExtInfo existingEntityVersionWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersionObject.getGuid()); - AtlasRelatedObjectId existingModelObject = (AtlasRelatedObjectId) existingEntityVersionWithExtInfo.getEntity().getRelationshipAttributes().get("dMModel"); + String modelGuid = AtlasGraphUtilsV2.getGuidByUniqueAttributes( + typeRegistry.getEntityTypeByName(ATLAS_DM_DATA_MODEL), + attrValues); - if (existingModelObject == null) { - throw new AtlasBaseException(AtlasErrorCode.DATA_MODEL_NOT_EXIST); - } + List existingAttributes = null; - // create modelVersion v1 ---> v2 - ModelResponse modelVersionResponse = replicateModelVersion(existingModelVersionObject, now); - AtlasEntity existingModelVersionEntity = modelVersionResponse.getExistingEntity(); - AtlasEntity copyModelVersion = modelVersionResponse.getCopyEntity(); - AtlasVertex existingModelVersionVertex = modelVersionResponse.getExistingVertex(); - AtlasVertex copyModelVersionVertex = modelVersionResponse.getCopyVertex(); - // String modelVersionQualifiedName = (String) copyModelVersion.getAttribute(QUALIFIED_NAME); - - // retrieve active entities linked to previous version - existingEntityObjects = (List) existingModelVersionEntity.getRelationshipAttributes().get("dMEntities"); - - List activeEntities = new ArrayList<>(); - for (AtlasRelatedObjectId _existingEntityObject : existingEntityObjects) { - AtlasEntity entity = entityRetriever.toAtlasEntity(_existingEntityObject.getGuid()); - if ((long) entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE) > 0 && (long) entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE) > 0) { - continue; + if (latestEntityVertex != null) { + modelENtityResponse = replicateModelEntity( + entityRetriever.toAtlasEntity(latestEntityVertex), + latestEntityVertex, + entityQualifiedNamePrefix, + now + ); + if (modelENtityResponse.getExistingEntity()!=null && modelENtityResponse.getExistingEntity().getRelationshipAttributes()!=null){ + existingAttributes = (List) modelENtityResponse.getExistingEntity().getAttributes().get("dMAttributes"); } - activeEntities.add(entity); + } else { + modelENtityResponse = createEntity( + attributeQualifiedNamePrefix + "_" + now, + ATLAS_DM_ATTRIBUTE_TYPE, + namespace, + context + ); } + ModelResponse modelVersionResponse = replicateModelVersion(modelGuid, modelQualifiedName, now); - // ex: default/dm/1234/modelName/entityName/attributeName - String qualifiedAttributeNamePrefix = (String) entityAttribute.getAttributes().get(ATLAS_DM_QUALIFIED_NAME_PREFIX); - int lastIndex = qualifiedAttributeNamePrefix.lastIndexOf("/"); - - // ex: default/dm/1234/modelName/entityName - String qualifiedEntityNamePrefix = qualifiedAttributeNamePrefix.substring(0, lastIndex); + if (modelVersionResponse.getCopyEntity() == null) { + modelVersionResponse = createEntity( + (modelQualifiedName + "/" + modelVersion), + ATLAS_DM_VERSION_TYPE, + namespace, + context); + } + AtlasEntity latestModelVersionEntity = modelVersionResponse.getCopyEntity(); + AtlasVertex latestModelVersionVertex = modelVersionResponse.getCopyVertex(); + List existingEntities = null; - // create entity e1 ---> e1' - ModelResponse modelEntityResponse = replicateModelEntity( - existingEntityWithExtInfo.getEntity(), - entityRetriever.getEntityVertex(existingModelVersionEntity.getGuid()), - qualifiedEntityNamePrefix, - now); - AtlasEntity existingEntity = modelEntityResponse.getExistingEntity(); - AtlasVertex existingEntityVertex = modelEntityResponse.getExistingVertex(); - AtlasVertex copyEntityVertex = modelEntityResponse.getCopyVertex(); - AtlasEntity copyEntity = modelEntityResponse.getCopyEntity(); - List existingEntityAttributes = (List) modelEntityResponse.getExistingEntity().getRelationshipAttributes().get("dMAttributes"); + if (modelVersionResponse.getExistingEntity() != null && modelVersionResponse.getExistingEntity().getRelationshipAttributes() != null) { + existingEntities = (List) modelVersionResponse.getExistingEntity().getRelationshipAttributes().get("dMEntities"); + } + AtlasEntity existingEntityAttributeWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(entityAttribute.getGuid(), false).getEntity(); // create attribute a1 ---> a1' ModelResponse modelAttributeResponse = replicateModelAttribute( - existingEntityAttributeWithExtInfo.getEntity(), + existingEntityAttributeWithExtInfo, entityRetriever.getEntityVertex(entityAttribute.getGuid()), - qualifiedAttributeNamePrefix, + attributeQualifiedNamePrefix, now); AtlasVertex copyAttributeVertex = modelAttributeResponse.getCopyVertex(); @@ -287,44 +246,38 @@ private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt unsetExpiredDates(copyAttribute, copyAttributeVertex); // create model-modelVersion relationship - AtlasRelatedObjectId modelObject = createModelModelVersionRelation(existingModelVersionVertex, copyModelVersionVertex); + createModelModelVersionRelation(modelGuid, latestModelVersionEntity.getGuid()); // create modelVersion-entity relationship [with new entity] - createModelVersionModelEntityRelationship(copyModelVersionVertex, copyEntityVertex); + createModelVersionModelEntityRelationship(latestModelVersionVertex, modelENtityResponse.getCopyVertex()); // create modelVersion-entity relationship [with existing entities] - for (AtlasEntity entity : activeEntities) { - createModelVersionModelEntityRelationship(copyModelVersionVertex, entityRetriever.getEntityVertex(entity.getGuid())); - } + createModelVersionModelEntityRelationship(latestModelVersionVertex, existingEntities); + // create entity - attribute relation [with new attribute] - createModelEntityModelAttributeRelation(copyEntityVertex, copyAttributeVertex); + createModelEntityModelAttributeRelation(modelENtityResponse.getCopyVertex(), copyAttributeVertex); // create entity - attribute relation [with existing attributes] - createModelEntityModelAttributeRelation(copyEntityVertex, existingEntityAttributes); + createModelEntityModelAttributeRelation(modelENtityResponse.getCopyVertex(), existingAttributes); AtlasEntityType attributeType = typeRegistry.getEntityTypeByName(entityAttribute.getTypeName()); - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(copyEntity.getTypeName()); - AtlasEntityType modelVersionType = typeRegistry.getEntityTypeByName(existingModelVersionEntity.getTypeName()); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(modelENtityResponse.getCopyEntity().getTypeName()); + AtlasEntityType modelVersionType = typeRegistry.getEntityTypeByName(latestModelVersionEntity.getTypeName()); context.addCreated(copyAttribute.getGuid(), copyAttribute, attributeType, copyAttributeVertex); - context.addCreated(copyEntity.getGuid(), copyEntity, entityType, copyEntityVertex); - context.addCreated(copyModelVersion.getGuid(), copyModelVersion, modelVersionType, copyModelVersionVertex); - + context.addCreated(modelENtityResponse.getCopyEntity().getGuid(), modelENtityResponse.getCopyEntity(), + entityType, modelENtityResponse.getCopyVertex()); + context.addCreated(latestModelVersionEntity.getGuid(), + latestModelVersionEntity, modelVersionType, latestModelVersionVertex); - context.addUpdated(existingModelVersionEntity.getGuid(), existingModelVersionEntity, - modelVersionType, existingModelVersionVertex); - context.addUpdated(existingEntity.getGuid(), existingEntity, - entityType, existingEntityVertex); - // remove existing entity from context so it is not updated context.removeUpdated(entityAttribute.getGuid(), entityAttribute, entityType, vertexAttribute); // resolve references context.getDiscoveryContext(). addResolvedGuid( - modelObject.getGuid(), - entityRetriever.getEntityVertex(modelObject.getGuid())); + modelGuid, + entityRetriever.getEntityVertex(modelGuid)); } - } \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java index 1d8bc10bd3..047aa35903 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -19,8 +19,9 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections4.MapUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; +import org.elasticsearch.common.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,7 +63,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co break; case UPDATE: - updateDMEntity(entity, vertex, context); + updateDMEntities(entity, vertex, context); break; } } @@ -87,45 +88,23 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati int lastIndex = qualifiedNamePrefix.lastIndexOf("/"); String modelQualifiedName = qualifiedNamePrefix.substring(0, lastIndex); - String modelGuid = AtlasGraphUtilsV2.getGuidByUniqueAttributes( - typeRegistry.getEntityTypeByName(ATLAS_DM_DATA_MODEL), - (Map) new HashMap().put(QUALIFIED_NAME, modelQualifiedName)); + Map attrValues = new HashMap<>(); + attrValues.put(QUALIFIED_NAME, modelQualifiedName); - AtlasEntity.AtlasEntityWithExtInfo dataModel = entityRetriever.toAtlasEntityWithExtInfo(modelGuid, false); - List existingModelVersions = (List) dataModel.getEntity().getAttributes().get("dMVersions"); + AtlasVertex modelVertex = AtlasGraphUtilsV2.findByUniqueAttributes( + typeRegistry.getEntityTypeByName(ATLAS_DM_DATA_MODEL), attrValues); - String modelVersion = "v1"; - AtlasRelatedObjectId existingModelVersionObj = null; - - if (CollectionUtils.isNotEmpty(existingModelVersions)) { - int existingVersionNumber = existingModelVersions.size(); - modelVersion = "v" + (++existingVersionNumber); - - for (AtlasRelatedObjectId modelVersionObj : existingModelVersions) { - if (((int) modelVersionObj.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE) > 0) || - ((int) modelVersionObj.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE) > 0)) { - continue; - } - existingModelVersionObj = modelVersionObj; - } + if (modelVertex == null) { + throw new AtlasBaseException(AtlasErrorCode.DATA_MODEL_NOT_EXIST); } - AtlasEntity existingModelVersionEntity = null; - AtlasVertex existingModelVersionVertex = null; - List existingEntities = null; - ModelResponse modelVersionResponse = null; - + String modelGuid = AtlasGraphUtilsV2.getIdFromVertex(modelVertex); + ModelResponse modelVersionResponse = replicateModelVersion(modelGuid, modelQualifiedName, now); - if (existingModelVersionObj != null) { - existingModelVersionEntity = entityRetriever.toAtlasEntityWithExtInfo(existingModelVersionObj.getGuid()).getEntity(); - existingModelVersionVertex = entityRetriever.getEntityVertex(existingModelVersionObj.getGuid()); - setModelExpiredAtDates(existingModelVersionEntity, existingModelVersionVertex, now); - existingEntities= (List) existingModelVersionEntity.getRelationshipAttributes().get("dMEntities"); - modelVersionResponse = replicateModelVersion(existingModelVersionObj, now); - } else { + if (modelVersionResponse.getCopyEntity() == null) { String namespace = (String) entity.getAttributes().get(ATLAS_DM_NAMESPACE); modelVersionResponse = createEntity( - (modelQualifiedName + "/" + modelVersion), + (modelQualifiedName + "/" + "v1"), ATLAS_DM_VERSION_TYPE, namespace, context); @@ -136,17 +115,22 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati // model --- modelVersion relation - createModelModelVersionRelation(modelGuid,latestModelVersionEntity.getGuid()); + createModelModelVersionRelation(modelGuid, latestModelVersionEntity.getGuid()); // modelVersion --- entity relation createModelVersionModelEntityRelationship(latestModelVersionVertex, vertex); - // modelVersion --- entitiesOfExistingModelVersion - createModelVersionModelEntityRelationship(latestModelVersionVertex, existingEntities); + if (modelVersionResponse.getExistingEntity() != null) { + List existingEntities = (List) modelVersionResponse.getExistingEntity() + .getRelationshipAttributes() + .get("dMEntities"); + // modelVersion --- entitiesOfExistingModelVersion + createModelVersionModelEntityRelationship(latestModelVersionVertex, existingEntities); + } context.addCreated(latestModelVersionEntity.getGuid(), latestModelVersionEntity, - typeRegistry.getEntityTypeByName(ATLAS_DM_VERSION_TYPE), latestModelVersionVertex); + typeRegistry.getEntityTypeByName(ATLAS_DM_VERSION_TYPE), latestModelVersionVertex); // resolve references context.getDiscoveryContext(). addResolvedGuid( @@ -155,17 +139,64 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati } - private void updateDMEntities(){ + private void updateDMEntities(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + ModelResponse modelResponseParentEntity = updateDMEntity(entity, vertex, context); + + // case when a mapping is added + if (entity.getAppendRelationshipAttributes() != null) { + LinkedHashMap appendAttributes = (LinkedHashMap) entity.getAppendRelationshipAttributes(); + ModelResponse modelResponseRelatedEntity = null; + String guid = ""; + + for (String attribute : appendAttributes.keySet()) { + + if (appendAttributes.get(attribute) instanceof List) { + List> attributeList = (List>) appendAttributes.get(attribute); + + for (LinkedHashMap relationAttribute : attributeList) { + guid = (String) relationAttribute.get("guid"); + if (Strings.isEmpty(guid)) { + continue; + } + // update end2 + modelResponseRelatedEntity = updateDMEntity( + entityRetriever.toAtlasEntity(guid), + entityRetriever.getEntityVertex(guid), + context); + + relationAttribute.put("guid", modelResponseRelatedEntity.getCopyEntity().getGuid()); + } + } + + if (appendAttributes.get(attribute) instanceof LinkedHashMap) { + + LinkedHashMap attributeList = (LinkedHashMap) appendAttributes.get(attribute); + guid = (String) attributeList.get("guid"); + if (!Strings.isEmpty(guid)) { + + // update end2 + modelResponseRelatedEntity = updateDMEntity( + entityRetriever.toAtlasEntity(guid), + entityRetriever.getEntityVertex(guid), + context); + + attributeList.put("guid", modelResponseRelatedEntity.getCopyEntity().getGuid()); + } + } + + } + + modelResponseParentEntity.getCopyEntity().setAppendRelationshipAttributes(appendAttributes); + context.removeUpdatedWithRelationshipAttributes(entity); + context.setUpdatedWithRelationshipAttributes(modelResponseParentEntity.getCopyEntity()); + } } - private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { - // lets say entity has attribute : versionQualifiedName - // default/dm/1725359500/com.jpmc.ct.fri/RegulatoryReporting/entity1/epoch - // query with qualifiedName : default/dm/1725359500/com.jpmc.ct.fri/RegulatoryReporting/entity1 + private ModelResponse updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { if (!entity.getTypeName().equals(ATLAS_DM_ENTITY_TYPE)) { - return; + return new ModelResponse(entity, vertex); } String entityName = (String) entity.getAttribute(NAME); @@ -177,40 +208,57 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati long now = RequestContext.get().getRequestTime(); AtlasEntity.AtlasEntityWithExtInfo existingEntity = entityRetriever.toAtlasEntityWithExtInfo(vertex, false); - AtlasRelatedObjectId existingModelVersion = (AtlasRelatedObjectId) existingEntity.getEntity().getRelationshipAttributes().get("dMVersion"); List existingEntityAttributes = (List) existingEntity.getEntity().getRelationshipAttributes().get("dMAttributes"); - if (existingModelVersion == null) { - throw new AtlasBaseException(AtlasErrorCode.DATA_MODEL_VERSION_NOT_EXIST, existingModelVersion.getGuid()); - } - // create modelVersion v1 ---> v2 - ModelResponse modelVersionResponse = replicateModelVersion(existingModelVersion, now); - AtlasEntity existingModelVersionEntity = modelVersionResponse.getExistingEntity(); - AtlasEntity copyModelVersion = modelVersionResponse.getCopyEntity(); - AtlasVertex existingModelVersionVertex = modelVersionResponse.getExistingVertex(); - AtlasVertex copyModelVersionVertex = modelVersionResponse.getCopyVertex(); - // String modelVersionAttributeQualifiedName = (String) copyModelVersion.getAttribute(QUALIFIED_NAME); - List existingEntities = (List) existingModelVersionEntity.getRelationshipAttributes().get("dMEntities"); + // get model qualifiedName with qualifiedNamePrefix + String qualifiedNamePrefix = (String) entity.getAttributes().get(ATLAS_DM_QUALIFIED_NAME_PREFIX); + int lastIndex = qualifiedNamePrefix.lastIndexOf("/"); + String modelQualifiedName = qualifiedNamePrefix.substring(0, lastIndex); + String modelVersion = "v1"; + + + Map attrValues = new HashMap<>(); + attrValues.put(QUALIFIED_NAME, modelQualifiedName); + + AtlasVertex modelVertex = AtlasGraphUtilsV2.findByUniqueAttributes( + typeRegistry.getEntityTypeByName(ATLAS_DM_DATA_MODEL), attrValues); + + String modelGuid = AtlasGraphUtilsV2.getIdFromVertex(modelVertex); + ModelResponse modelVersionResponse = replicateModelVersion(modelGuid, modelQualifiedName, now); + + // model is not replicated successfully + if (modelVersionResponse.getCopyEntity() == null) { + String namespace = (String) entity.getAttributes().get(ATLAS_DM_NAMESPACE); + modelVersionResponse = createEntity( + (modelQualifiedName + "/" + modelVersion), + ATLAS_DM_VERSION_TYPE, + namespace, + context); + } - String entityQualifiedNamePrefix = (String) entity.getAttributes().get(ATLAS_DM_QUALIFIED_NAME_PREFIX); + AtlasEntity latestModelVersionEntity = modelVersionResponse.getCopyEntity(); + AtlasVertex latestModelVersionVertex = modelVersionResponse.getCopyVertex(); + AtlasEntity existingVersion = modelVersionResponse.getExistingEntity(); // create entity e1 ---> e1' - ModelResponse modelEntityResponse = replicateModelEntity(existingEntity.getEntity(), vertex, entityQualifiedNamePrefix, now); + ModelResponse modelEntityResponse = replicateModelEntity(existingEntity.getEntity(), vertex, qualifiedNamePrefix, now); AtlasVertex copyEntityVertex = modelEntityResponse.getCopyVertex(); AtlasEntity copyEntity = modelEntityResponse.getCopyEntity(); applyDiffs(entity, copyEntity, ATLAS_DM_ENTITY_TYPE); unsetExpiredDates(copyEntity, copyEntityVertex); // create model-modelVersion relation - AtlasRelatedObjectId modelObject = createModelModelVersionRelation(existingModelVersionVertex, copyModelVersionVertex); + createModelModelVersionRelation(modelGuid, latestModelVersionEntity.getGuid()); // create modelVersion-modelEntity relationship with new entity - createModelVersionModelEntityRelationship(copyModelVersionVertex, copyEntityVertex); + createModelVersionModelEntityRelationship(latestModelVersionVertex, copyEntityVertex); // create modelVersion-modelEntity relation with old entities which are not expired - createModelVersionModelEntityRelationship(copyModelVersionVertex, existingEntities); - + if (existingVersion != null) { + List existingEntities = (List) existingVersion.getRelationshipAttributes().get("dMEntities"); + createModelVersionModelEntityRelationship(latestModelVersionVertex, existingEntities); + } // create modelEntity-modelAttributeRelationship createModelEntityModelAttributeRelation(copyEntityVertex, existingEntityAttributes); @@ -220,25 +268,22 @@ private void updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati */ // previousEntity and previousModelVersion AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); - AtlasEntityType modelVersionType = typeRegistry.getEntityTypeByName(existingModelVersionEntity.getTypeName()); + AtlasEntityType modelVersionType = typeRegistry.getEntityTypeByName(modelVersionResponse.getCopyEntity().getTypeName()); context.addCreated(copyEntity.getGuid(), copyEntity, entityType, copyEntityVertex); - context.addCreated(copyModelVersion.getGuid(), copyModelVersion, modelVersionType, copyModelVersionVertex); - - context.addUpdated(entity.getGuid(), entity, entityType, vertex); - context.addUpdated(existingModelVersionEntity.getGuid(), existingModelVersionEntity, - modelVersionType, existingModelVersionVertex ); + context.addCreated(latestModelVersionEntity.getGuid(), latestModelVersionEntity, modelVersionType, latestModelVersionVertex); - // remove existing entity from context so it is not updated + //remove existing entity from context so it is not updated context.removeUpdated(entity.getGuid(), entity, entityType, vertex); // resolve references context.getDiscoveryContext(). addResolvedGuid( - modelObject.getGuid(), - entityRetriever.getEntityVertex(modelObject.getGuid())); + modelGuid, + modelVertex); + return new ModelResponse(copyEntity, copyEntityVertex); } } From 5db6ebd7fc4e41de8f3b8543bd2484c8c8fd0543 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 23 Sep 2024 12:47:22 +0530 Subject: [PATCH 30/55] fix date formattings --- .../model/AbstractModelPreProcessor.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index 5e330abae3..db10272e41 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -24,7 +24,6 @@ import org.mockito.internal.util.collections.ListUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import java.util.*; import static org.apache.atlas.repository.Constants.*; @@ -42,6 +41,7 @@ public abstract class AbstractModelPreProcessor implements PreProcessor { protected EntityGraphMapper entityGraphMapper; protected AtlasRelationshipStore atlasRelationshipStore; + public AbstractModelPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, EntityGraphMapper entityGraphMapper, AtlasRelationshipStore atlasRelationshipStore) { this.typeRegistry = typeRegistry; this.entityRetriever = entityRetriever; @@ -102,7 +102,7 @@ protected void setModelExpiredAtDates(AtlasEntity oldEntity, AtlasVertex oldVert protected void setQualifiedName(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { newEntity.setAttribute(QUALIFIED_NAME, value); - // AtlasGraphUtilsV2.setEncodedProperty(newVertex, QUALIFIED_NAME, value); + AtlasGraphUtilsV2.setEncodedProperty(newVertex, QUALIFIED_NAME, value); } protected void setName(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { @@ -143,9 +143,10 @@ protected ModelResponse replicateModelVersion(String modelGuid, String modelQual // get active model version for (AtlasRelatedObjectId modelVersionObj : existingModelVersions) { AtlasEntity modelVersionEntity = entityRetriever.toAtlasEntity(modelVersionObj.getGuid()); + Date expiredAtBusinessDate = (Date) modelVersionEntity.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE); + Date expiredAtSystemDate = (Date) modelVersionEntity.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE); - if (modelVersionEntity.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE) != null || - modelVersionEntity.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE) != null) { + if (expiredAtBusinessDate.getTime() > 0 || expiredAtSystemDate.getTime() > 0) { continue; } existingModelVersionObj = modelVersionObj; @@ -220,16 +221,16 @@ protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersio GraphHelper.getTypeName(modelVersionVertex))); for (AtlasRelatedObjectId existingEntity : existingEntities) { AtlasEntity entity = entityRetriever.toAtlasEntity(existingEntity.getGuid()); - if ( - (entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE) != null) || - (entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE) != null) - ) { + Date expiredAtBusinessDate = (Date) entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE); + Date expiredAtSystemDate = (Date) entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE); + if (expiredAtBusinessDate.getTime() > 0 || expiredAtSystemDate.getTime() > 0) { continue; } modelVersionEntityRelation.setEnd2(new AtlasObjectId( existingEntity.getGuid(), existingEntity.getTypeName() )); + atlasRelationshipStore.create(modelVersionEntityRelation); } } @@ -245,10 +246,9 @@ protected void createModelEntityModelAttributeRelation(AtlasVertex entity, List< GraphHelper.getTypeName(entity))); for (AtlasRelatedObjectId existingEntityAttribute : existingEntityAttributes) { AtlasEntity entityAttribute = entityRetriever.toAtlasEntity(existingEntityAttribute.getGuid()); - if ( - (entityAttribute.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE) != null) || - (entityAttribute.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE) != null) - ) { + Date expiredAtBusinessDate = (Date) entityAttribute.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE); + Date expiredAtSystemDate = (Date) entityAttribute.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE); + if (expiredAtBusinessDate.getTime() > 0 || expiredAtSystemDate.getTime() > 0) { continue; } modelEntityAttributeRelation.setEnd2( From 90751fb70f7980bb134ab61ea65dab151bc57367 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 23 Sep 2024 13:27:25 +0530 Subject: [PATCH 31/55] set allowed relationships for entity and attributes --- .../preprocessor/model/DMAttributePreprocessor.java | 8 +++++++- .../v2/preprocessor/model/DMEntityPreProcessor.java | 13 +++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java index 3c9d8cb1f2..32405030d6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java @@ -28,10 +28,16 @@ public class DMAttributePreprocessor extends AbstractModelPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(AbstractModelPreProcessor.class); + Set allowedRelationshipNames; public DMAttributePreprocessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, EntityGraphMapper entityGraphMapper, AtlasRelationshipStore atlasRelationshipStore) { super(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore); + allowedRelationshipNames = new HashSet<>(); + allowedRelationshipNames.add("dMMappedFromAttributes"); + allowedRelationshipNames.add("dMMappedToAttributes"); + allowedRelationshipNames.add("dMRelatedFromAttributes"); + allowedRelationshipNames.add("dMRelatedToAttributes"); } @@ -201,7 +207,7 @@ private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt entityQualifiedNamePrefix, now ); - if (modelENtityResponse.getExistingEntity()!=null && modelENtityResponse.getExistingEntity().getRelationshipAttributes()!=null){ + if (modelENtityResponse.getExistingEntity() != null && modelENtityResponse.getExistingEntity().getRelationshipAttributes() != null) { existingAttributes = (List) modelENtityResponse.getExistingEntity().getAttributes().get("dMAttributes"); } } else { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java index 047aa35903..4cfcbcd9fd 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -25,6 +25,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.text.ParseException; import java.time.Instant; import java.util.*; @@ -33,10 +34,16 @@ public class DMEntityPreProcessor extends AbstractModelPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(AbstractModelPreProcessor.class); + Set allowedRelationshipNames; public DMEntityPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, EntityGraphMapper entityGraphMapper, AtlasRelationshipStore atlasRelationshipStore) { super(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore); + allowedRelationshipNames = new HashSet<>(); + allowedRelationshipNames.add("dMMappedToEntities"); + allowedRelationshipNames.add("dMMappedFromEntities"); + allowedRelationshipNames.add("dMRelatedFromEntities"); + allowedRelationshipNames.add("dMRelatedToEntities"); } @@ -57,10 +64,6 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co switch (operation) { case CREATE: createDMEntity(entity, vertex, context); - // modelVersion --->modelName - // entity ---> modelVersion - // attribute ---> entityName - break; case UPDATE: updateDMEntities(entity, vertex, context); @@ -124,7 +127,6 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati List existingEntities = (List) modelVersionResponse.getExistingEntity() .getRelationshipAttributes() .get("dMEntities"); - // modelVersion --- entitiesOfExistingModelVersion createModelVersionModelEntityRelationship(latestModelVersionVertex, existingEntities); @@ -170,7 +172,6 @@ private void updateDMEntities(AtlasEntity entity, AtlasVertex vertex, EntityMuta } if (appendAttributes.get(attribute) instanceof LinkedHashMap) { - LinkedHashMap attributeList = (LinkedHashMap) appendAttributes.get(attribute); guid = (String) attributeList.get("guid"); if (!Strings.isEmpty(guid)) { From 2054edef4396f865f45b2539b65d9662146d1a2a Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 23 Sep 2024 16:54:51 +0530 Subject: [PATCH 32/55] Handle mapping for relationshipAttributes --- .../model/AbstractModelPreProcessor.java | 56 ++++++++++++++++--- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index db10272e41..352328a359 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -21,9 +21,11 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.ListUtils; import org.apache.commons.collections.MapUtils; +import org.elasticsearch.common.Strings; import org.mockito.internal.util.collections.ListUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.util.*; import static org.apache.atlas.repository.Constants.*; @@ -102,7 +104,7 @@ protected void setModelExpiredAtDates(AtlasEntity oldEntity, AtlasVertex oldVert protected void setQualifiedName(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { newEntity.setAttribute(QUALIFIED_NAME, value); - AtlasGraphUtilsV2.setEncodedProperty(newVertex, QUALIFIED_NAME, value); + AtlasGraphUtilsV2.setEncodedProperty(newVertex, QUALIFIED_NAME, value); } protected void setName(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { @@ -320,17 +322,38 @@ protected void copyAllAttributes(AtlasEntity source, AtlasEntity destination, lo destination.setClassifications(new ArrayList<>()); // Set empty list if source classifications are null or empty } + Set allowedRelations = allowedRelationshipsForEntityType(source.getTypeName()); + + Map destinationRelationAttributes = new HashMap<>(); + if (source.getRelationshipAttributes() != null) { + Map sourceAttributes = source.getRelationshipAttributes(); + for (String attribute : sourceAttributes.keySet()) { + if (allowedRelations.contains(attribute)) { + destinationRelationAttributes.put(attribute, sourceAttributes.get(attribute)); + } + } + } + + LinkedHashMap destinationAppendAttributes = new LinkedHashMap<>(); if (source.getAppendRelationshipAttributes() != null) { - destination.setAppendRelationshipAttributes(new HashMap<>(source.getAppendRelationshipAttributes())); - } else { - destination.setAppendRelationshipAttributes(new HashMap<>()); // Set empty map if source append relationship attributes are null + LinkedHashMap sourceAttributes = (LinkedHashMap) source.getAppendRelationshipAttributes(); + for (String attribute : sourceAttributes.keySet()) { + if (allowedRelations.contains(attribute)) { + destinationAppendAttributes.put(attribute, sourceAttributes.get(attribute)); + } + } } + LinkedHashMap destinationRemoveAttributes = new LinkedHashMap<>(); if (source.getRemoveRelationshipAttributes() != null) { - destination.setRemoveRelationshipAttributes(new HashMap<>(source.getRemoveRelationshipAttributes())); - } else { - destination.setRemoveRelationshipAttributes(new HashMap<>()); + LinkedHashMap sourceAttributes = (LinkedHashMap) source.getRemoveRelationshipAttributes(); + for (String attribute : sourceAttributes.keySet()) { + if (allowedRelations.contains(attribute)) { + destinationRemoveAttributes.put(attribute, sourceAttributes.get(attribute)); + } + } } + } @@ -379,4 +402,23 @@ protected void unsetExpiredDates(AtlasEntity latestEntity, AtlasVertex latestVer AtlasGraphUtilsV2.setEncodedProperty(latestVertex, ATLAS_DM_EXPIRED_AT_SYSTEM_DATE, 0); AtlasGraphUtilsV2.setEncodedProperty(latestVertex, ATLAS_DM_EXPIRED_AT_BUSINESS_DATE, 0); } + + private Set allowedRelationshipsForEntityType(String entityType) { + Set allowedRelationships = new HashSet<>(); + switch (entityType) { + case ATLAS_DM_ENTITY_TYPE: + allowedRelationships.add("dMMappedToEntities"); + allowedRelationships.add("dMMappedFromEntities"); + allowedRelationships.add("dMRelatedFromEntities"); + allowedRelationships.add("dMRelatedToEntities"); + break; + case ATLAS_DM_ATTRIBUTE_TYPE: + allowedRelationships.add("dMMappedFromAttributes"); + allowedRelationships.add("dMMappedToAttributes"); + allowedRelationships.add("dMRelatedFromAttributes"); + allowedRelationships.add("dMRelatedToAttributes"); + break; + } + return allowedRelationships; + } } From 5b5ef420b0c08d2391590a2fd7db0dd1a165d594 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 24 Sep 2024 06:11:29 +0530 Subject: [PATCH 33/55] remove relation --- .../repository/store/graph/v2/EntityMutationContext.java | 3 +++ .../v2/preprocessor/model/AbstractModelPreProcessor.java | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java index 0bf14635b3..f6a5df70e8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java @@ -101,6 +101,9 @@ public void removeUpdatedWithRelationshipAttributes(AtlasEntity entity){ entitiesUpdatedWithAppendRelationshipAttribute.remove(entity); } + public void removeUpdatedWithDeleteRelationshipAttributes(AtlasEntity entity){ + entitiesUpdatedWithRemoveRelationshipAttribute.remove(entity); + } public void addEntityToRestore(AtlasVertex vertex) { if (entitiesToRestore == null) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index 352328a359..c8ed042463 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -148,7 +148,7 @@ protected ModelResponse replicateModelVersion(String modelGuid, String modelQual Date expiredAtBusinessDate = (Date) modelVersionEntity.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE); Date expiredAtSystemDate = (Date) modelVersionEntity.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE); - if (expiredAtBusinessDate.getTime() > 0 || expiredAtSystemDate.getTime() > 0) { + if (expiredAtBusinessDate != null && expiredAtBusinessDate.getTime() > 0 || expiredAtSystemDate != null && expiredAtSystemDate.getTime() > 0) { continue; } existingModelVersionObj = modelVersionObj; @@ -225,7 +225,7 @@ protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersio AtlasEntity entity = entityRetriever.toAtlasEntity(existingEntity.getGuid()); Date expiredAtBusinessDate = (Date) entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE); Date expiredAtSystemDate = (Date) entity.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE); - if (expiredAtBusinessDate.getTime() > 0 || expiredAtSystemDate.getTime() > 0) { + if (expiredAtBusinessDate != null && expiredAtBusinessDate.getTime() > 0 || expiredAtSystemDate != null && expiredAtSystemDate.getTime() > 0) { continue; } modelVersionEntityRelation.setEnd2(new AtlasObjectId( @@ -250,7 +250,7 @@ protected void createModelEntityModelAttributeRelation(AtlasVertex entity, List< AtlasEntity entityAttribute = entityRetriever.toAtlasEntity(existingEntityAttribute.getGuid()); Date expiredAtBusinessDate = (Date) entityAttribute.getAttributes().get(ATLAS_DM_EXPIRED_AT_SYSTEM_DATE); Date expiredAtSystemDate = (Date) entityAttribute.getAttributes().get(ATLAS_DM_EXPIRED_AT_BUSINESS_DATE); - if (expiredAtBusinessDate.getTime() > 0 || expiredAtSystemDate.getTime() > 0) { + if (expiredAtBusinessDate != null && expiredAtBusinessDate.getTime() > 0 || expiredAtSystemDate != null && expiredAtSystemDate.getTime() > 0) { continue; } modelEntityAttributeRelation.setEnd2( @@ -403,7 +403,7 @@ protected void unsetExpiredDates(AtlasEntity latestEntity, AtlasVertex latestVer AtlasGraphUtilsV2.setEncodedProperty(latestVertex, ATLAS_DM_EXPIRED_AT_BUSINESS_DATE, 0); } - private Set allowedRelationshipsForEntityType(String entityType) { + protected Set allowedRelationshipsForEntityType(String entityType) { Set allowedRelationships = new HashSet<>(); switch (entityType) { case ATLAS_DM_ENTITY_TYPE: From 14cb2bb88381e20cc74d69769cf460e6e359db21 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 24 Sep 2024 06:12:22 +0530 Subject: [PATCH 34/55] append/Remove mappings --- .../model/DMEntityPreProcessor.java | 132 +++++++++++++----- 1 file changed, 97 insertions(+), 35 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java index 4cfcbcd9fd..fd40bd86fe 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -2,31 +2,25 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; -import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.instance.*; -import org.apache.atlas.repository.Constants; -import org.apache.atlas.repository.graph.AtlasGraphProvider; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasRelatedObjectId; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; +import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; -import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.CategoryPreProcessor; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.common.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.text.ParseException; -import java.time.Instant; import java.util.*; import static org.apache.atlas.repository.Constants.*; @@ -138,43 +132,74 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati addResolvedGuid( modelGuid, entityRetriever.getEntityVertex(modelGuid)); - } + /** + * Optimise this : validate why reference is not created in legacy code + */ + resolveReferences(entity, context); + } private void updateDMEntities(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { ModelResponse modelResponseParentEntity = updateDMEntity(entity, vertex, context); // case when a mapping is added if (entity.getAppendRelationshipAttributes() != null) { - LinkedHashMap appendAttributes = (LinkedHashMap) entity.getAppendRelationshipAttributes(); + Map appendRelationshipAttributes = processAppendRemoveRelationshipAttributes(entity, entity.getAppendRelationshipAttributes(), context); + modelResponseParentEntity.getCopyEntity().setAppendRelationshipAttributes(appendRelationshipAttributes); + context.removeUpdatedWithRelationshipAttributes(entity); + context.setUpdatedWithRelationshipAttributes(modelResponseParentEntity.getCopyEntity()); + } + + if (entity.getRemoveRelationshipAttributes() != null) { + Map appendRelationshipAttributes = processAppendRemoveRelationshipAttributes(entity, entity.getAppendRelationshipAttributes(), context); + modelResponseParentEntity.getCopyEntity().setRemoveRelationshipAttributes(appendRelationshipAttributes); + context.removeUpdatedWithDeleteRelationshipAttributes(entity); + context.setUpdatedWithRemoveRelationshipAttributes(modelResponseParentEntity.getCopyEntity()); + } + } + + private Map processAppendRemoveRelationshipAttributes(AtlasEntity entity, Map relationshipAttributes, EntityMutationContext context) throws AtlasBaseException { + Map appendAttributesDestination = new HashMap<>(); + if (relationshipAttributes != null) { + Map appendAttributesSource = (Map) relationshipAttributes; + ; ModelResponse modelResponseRelatedEntity = null; String guid = ""; + Set allowedRelations = allowedRelationshipsForEntityType(entity.getTypeName()); - for (String attribute : appendAttributes.keySet()) { + for (String attribute : appendAttributesSource.keySet()) { - if (appendAttributes.get(attribute) instanceof List) { - List> attributeList = (List>) appendAttributes.get(attribute); + if (appendAttributesSource.get(attribute) instanceof List) { - for (LinkedHashMap relationAttribute : attributeList) { + if (!allowedRelations.contains(attribute)) { + continue; + } + List> destList = new ArrayList<>(); + Map destMap = null; + + List> attributeList = (List>) appendAttributesSource.get(attribute); + + for (Map relationAttribute : attributeList) { guid = (String) relationAttribute.get("guid"); - if (Strings.isEmpty(guid)) { - continue; - } // update end2 modelResponseRelatedEntity = updateDMEntity( entityRetriever.toAtlasEntity(guid), entityRetriever.getEntityVertex(guid), context); - - relationAttribute.put("guid", modelResponseRelatedEntity.getCopyEntity().getGuid()); + //relationAttribute.put("guid", modelResponseRelatedEntity.getCopyEntity()); + destMap = new HashMap<>(relationAttribute); + guid = modelResponseRelatedEntity.getCopyEntity().getGuid(); + destMap.put("guid", guid); + //destMap.put(QUALIFIED_NAME, ) + context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); + destList.add(destMap); } - } - - if (appendAttributes.get(attribute) instanceof LinkedHashMap) { - LinkedHashMap attributeList = (LinkedHashMap) appendAttributes.get(attribute); - guid = (String) attributeList.get("guid"); - if (!Strings.isEmpty(guid)) { + appendAttributesDestination.put(attribute, destList); + } else { + if (appendAttributesSource.get(attribute) instanceof Map) { + LinkedHashMap attributeList = (LinkedHashMap) appendAttributesSource.get(attribute); + guid = (String) attributeList.get("guid"); // update end2 modelResponseRelatedEntity = updateDMEntity( @@ -182,19 +207,18 @@ private void updateDMEntities(AtlasEntity entity, AtlasVertex vertex, EntityMuta entityRetriever.getEntityVertex(guid), context); - attributeList.put("guid", modelResponseRelatedEntity.getCopyEntity().getGuid()); + Map destMap = new HashMap<>(attributeList); + destMap.put("guid", guid); + guid = modelResponseRelatedEntity.getCopyEntity().getGuid(); + context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); + appendAttributesDestination.put(attribute, destMap); } } - } - - modelResponseParentEntity.getCopyEntity().setAppendRelationshipAttributes(appendAttributes); - context.removeUpdatedWithRelationshipAttributes(entity); - context.setUpdatedWithRelationshipAttributes(modelResponseParentEntity.getCopyEntity()); } + return appendAttributesDestination; } - private ModelResponse updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { if (!entity.getTypeName().equals(ATLAS_DM_ENTITY_TYPE)) { return new ModelResponse(entity, vertex); @@ -284,8 +308,46 @@ private ModelResponse updateDMEntity(AtlasEntity entity, AtlasVertex vertex, Ent modelGuid, modelVertex); + /*** + * debug why its already not there in context + */ + resolveReferences(entity, context); + return new ModelResponse(copyEntity, copyEntityVertex); } + + private void resolveReferences(AtlasEntity entity, EntityMutationContext context) { + if (entity.getRelationshipAttributes() != null) { + Map appendAttributesSource = (Map) entity.getRelationshipAttributes(); + ModelResponse modelResponseRelatedEntity = null; + String guid = ""; + Set allowedRelations = allowedRelationshipsForEntityType(entity.getTypeName()); + + for (String attribute : appendAttributesSource.keySet()) { + + if (appendAttributesSource.get(attribute) instanceof List) { + + if (!allowedRelations.contains(attribute)) { + continue; + } + List> attributeList = (List>) appendAttributesSource.get(attribute); + + for (Map relationAttribute : attributeList) { + guid = (String) relationAttribute.get("guid"); + context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); + } + } else { + if (appendAttributesSource.get(attribute) instanceof Map) { + LinkedHashMap attributeList = (LinkedHashMap) appendAttributesSource.get(attribute); + guid = (String) attributeList.get("guid"); + context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); + } + } + } + } + + } } + From a347851308975f65b9c8731dbf0b3105ff9ff262 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 24 Sep 2024 06:41:00 +0530 Subject: [PATCH 35/55] Handle mapping updates --- .../model/AbstractModelPreProcessor.java | 32 +++++- .../model/DMAttributePreprocessor.java | 98 ++++++++++++++++++- .../model/DMEntityPreProcessor.java | 35 +------ 3 files changed, 127 insertions(+), 38 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index c8ed042463..7cc4ef7551 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -176,7 +176,6 @@ protected ModelResponse replicateModelEntity(AtlasEntity existingEntity, AtlasVe AtlasVertex copyEntityVertex = entityGraphMapper.createVertex(existingEntity); AtlasEntity copyEntity = entityRetriever.toAtlasEntity(copyEntityVertex); copyAllAttributes(existingEntity, copyEntity, epoch); - // copyEntity.setRelationshipAttributes(entity.getRelationshipAttributes()); setModelDates(copyEntity, copyEntityVertex, epoch); String entityQualifiedName = entityQualifiedNamePrefix + "_" + epoch; setQualifiedName(copyEntity, copyEntityVertex, entityQualifiedName); @@ -421,4 +420,35 @@ protected Set allowedRelationshipsForEntityType(String entityType) { } return allowedRelationships; } + protected void resolveReferences(AtlasEntity entity, EntityMutationContext context) { + if (entity.getRelationshipAttributes() != null) { + Map appendAttributesSource = (Map) entity.getRelationshipAttributes(); + ModelResponse modelResponseRelatedEntity = null; + String guid = ""; + Set allowedRelations = allowedRelationshipsForEntityType(entity.getTypeName()); + + for (String attribute : appendAttributesSource.keySet()) { + + if (appendAttributesSource.get(attribute) instanceof List) { + + if (!allowedRelations.contains(attribute)) { + continue; + } + List> attributeList = (List>) appendAttributesSource.get(attribute); + + for (Map relationAttribute : attributeList) { + guid = (String) relationAttribute.get("guid"); + context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); + } + } else { + if (appendAttributesSource.get(attribute) instanceof Map) { + LinkedHashMap attributeList = (LinkedHashMap) appendAttributesSource.get(attribute); + guid = (String) attributeList.get("guid"); + context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); + } + } + } + } + + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java index 32405030d6..a92a7abe22 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java @@ -58,7 +58,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co createDMAttribute(entity, vertex, context); break; case UPDATE: - updateDMAttribute(entity, vertex, context); + updateDMAttributes(entity, vertex, context); } } @@ -162,11 +162,97 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt modelGuid, entityRetriever.getEntityVertex(modelGuid)); + /*** + * debug why its already not there in context + */ + resolveReferences(entityAttribute, context); + + } + + private void updateDMAttributes(AtlasEntity entityAttribute, AtlasVertex vertexAttribute, EntityMutationContext context) throws AtlasBaseException { + ModelResponse modelResponseParentEntity = updateDMAttribute(entityAttribute, vertexAttribute, context); + + // case when a mapping is added + if (entityAttribute.getAppendRelationshipAttributes() != null) { + Map appendRelationshipAttributes = processAppendRemoveRelationshipAttributes(entityAttribute, entityAttribute.getAppendRelationshipAttributes(), context); + modelResponseParentEntity.getCopyEntity().setAppendRelationshipAttributes(appendRelationshipAttributes); + context.removeUpdatedWithRelationshipAttributes(entityAttribute); + context.setUpdatedWithRelationshipAttributes(modelResponseParentEntity.getCopyEntity()); + } + + if (entityAttribute.getRemoveRelationshipAttributes() != null) { + Map appendRelationshipAttributes = processAppendRemoveRelationshipAttributes(entityAttribute, entityAttribute.getAppendRelationshipAttributes(), context); + modelResponseParentEntity.getCopyEntity().setRemoveRelationshipAttributes(appendRelationshipAttributes); + context.removeUpdatedWithDeleteRelationshipAttributes(entityAttribute); + context.setUpdatedWithRemoveRelationshipAttributes(modelResponseParentEntity.getCopyEntity()); + } } - private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAttribute, EntityMutationContext context) throws AtlasBaseException { + private Map processAppendRemoveRelationshipAttributes(AtlasEntity entity, Map relationshipAttributes, EntityMutationContext context) throws AtlasBaseException { + Map appendAttributesDestination = new HashMap<>(); + if (relationshipAttributes != null) { + Map appendAttributesSource = (Map) relationshipAttributes; + ; + ModelResponse modelResponseRelatedEntity = null; + String guid = ""; + Set allowedRelations = allowedRelationshipsForEntityType(entity.getTypeName()); + + for (String attribute : appendAttributesSource.keySet()) { + + if (appendAttributesSource.get(attribute) instanceof List) { + + if (!allowedRelations.contains(attribute)) { + continue; + } + List> destList = new ArrayList<>(); + Map destMap = null; + + List> attributeList = (List>) appendAttributesSource.get(attribute); + + for (Map relationAttribute : attributeList) { + guid = (String) relationAttribute.get("guid"); + + // update end2 + modelResponseRelatedEntity = updateDMAttribute( + entityRetriever.toAtlasEntity(guid), + entityRetriever.getEntityVertex(guid), + context); + //relationAttribute.put("guid", modelResponseRelatedEntity.getCopyEntity()); + destMap = new HashMap<>(relationAttribute); + guid = modelResponseRelatedEntity.getCopyEntity().getGuid(); + destMap.put("guid", guid); + //destMap.put(QUALIFIED_NAME, ) + context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); + destList.add(destMap); + } + appendAttributesDestination.put(attribute, destList); + } else { + if (appendAttributesSource.get(attribute) instanceof Map) { + LinkedHashMap attributeList = (LinkedHashMap) appendAttributesSource.get(attribute); + guid = (String) attributeList.get("guid"); + + // update end2 + modelResponseRelatedEntity = updateDMAttribute( + entityRetriever.toAtlasEntity(guid), + entityRetriever.getEntityVertex(guid), + context); + + Map destMap = new HashMap<>(attributeList); + destMap.put("guid", guid); + guid = modelResponseRelatedEntity.getCopyEntity().getGuid(); + context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); + appendAttributesDestination.put(attribute, destMap); + } + } + } + } + return appendAttributesDestination; + } + + + private ModelResponse updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAttribute, EntityMutationContext context) throws AtlasBaseException { if (!entityAttribute.getTypeName().equals(ATLAS_DM_ATTRIBUTE_TYPE)) { - return; + return new ModelResponse(entityAttribute, vertexAttribute); } String attributeName = (String) entityAttribute.getAttribute(NAME); @@ -285,5 +371,11 @@ private void updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt addResolvedGuid( modelGuid, entityRetriever.getEntityVertex(modelGuid)); + + /*** + * debug why its already not there in context + */ + resolveReferences(entityAttribute, context); + return new ModelResponse(copyAttribute, copyAttributeVertex); } } \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java index fd40bd86fe..c1a4ffed13 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -9,7 +9,6 @@ import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; -import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; @@ -151,7 +150,7 @@ private void updateDMEntities(AtlasEntity entity, AtlasVertex vertex, EntityMuta } if (entity.getRemoveRelationshipAttributes() != null) { - Map appendRelationshipAttributes = processAppendRemoveRelationshipAttributes(entity, entity.getAppendRelationshipAttributes(), context); + Map appendRelationshipAttributes = processAppendRemoveRelationshipAttributes(entity, entity.getRemoveRelationshipAttributes(), context); modelResponseParentEntity.getCopyEntity().setRemoveRelationshipAttributes(appendRelationshipAttributes); context.removeUpdatedWithDeleteRelationshipAttributes(entity); context.setUpdatedWithRemoveRelationshipAttributes(modelResponseParentEntity.getCopyEntity()); @@ -315,38 +314,6 @@ private ModelResponse updateDMEntity(AtlasEntity entity, AtlasVertex vertex, Ent return new ModelResponse(copyEntity, copyEntityVertex); } - - private void resolveReferences(AtlasEntity entity, EntityMutationContext context) { - if (entity.getRelationshipAttributes() != null) { - Map appendAttributesSource = (Map) entity.getRelationshipAttributes(); - ModelResponse modelResponseRelatedEntity = null; - String guid = ""; - Set allowedRelations = allowedRelationshipsForEntityType(entity.getTypeName()); - - for (String attribute : appendAttributesSource.keySet()) { - - if (appendAttributesSource.get(attribute) instanceof List) { - - if (!allowedRelations.contains(attribute)) { - continue; - } - List> attributeList = (List>) appendAttributesSource.get(attribute); - - for (Map relationAttribute : attributeList) { - guid = (String) relationAttribute.get("guid"); - context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); - } - } else { - if (appendAttributesSource.get(attribute) instanceof Map) { - LinkedHashMap attributeList = (LinkedHashMap) appendAttributesSource.get(attribute); - guid = (String) attributeList.get("guid"); - context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); - } - } - } - } - - } } From 98da1361426d3d6f630c5cf3ac02c5af0f41616e Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 24 Sep 2024 09:02:24 +0530 Subject: [PATCH 36/55] update DM constants --- .../apache/atlas/repository/Constants.java | 4 +++- .../DMAttributeAssociationPreprocessor.java | 2 ++ .../DMEntityAssociationPreProcessor.java | 21 +++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributeAssociationPreprocessor.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityAssociationPreProcessor.java diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 47a01bf08f..78332a1edb 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -214,8 +214,10 @@ public final class Constants { public static final String ATLAS_DM_VERSION_TYPE = "DMVersion"; - public static final String ATLAS_DM_QUALIFIED_NAME_PREFIX = "dMQualifiedNamePrefix"; + public static final String ATLAS_DM_ENTITY_ASSOCIATION_TYPE= "DMEntityAssociation"; + public static final String ATLAS_DM_ATTRIBUTE_ASSOCIATION_TYPE= "DMAttributeAssociation"; + public static final String ATLAS_DM_QUALIFIED_NAME_PREFIX = "dMQualifiedNamePrefix"; public static final String ATLAS_DM_NAMESPACE = "dMDataModelNamespace"; public static final String ATLAS_DM_EXPIRED_AT_SYSTEM_DATE = "dMDataModelExpiredAtSystemDate"; public static final String ATLAS_DM_EXPIRED_AT_BUSINESS_DATE = "dMDataModelExpiredAtBusinessDate"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributeAssociationPreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributeAssociationPreprocessor.java new file mode 100644 index 0000000000..2ce06bdcad --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributeAssociationPreprocessor.java @@ -0,0 +1,2 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.model;public class DMAttributeAssociationPreprocessor { +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityAssociationPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityAssociationPreProcessor.java new file mode 100644 index 0000000000..e893fdc614 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityAssociationPreProcessor.java @@ -0,0 +1,21 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.model; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; +import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.type.AtlasTypeRegistry; + +public class EntityAssociationPreProcessor extends AbstractModelPreProcessor{ + public EntityAssociationPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, EntityGraphMapper entityGraphMapper, AtlasRelationshipStore atlasRelationshipStore) { + super(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore); + } + + @Override + public void processAttributes(AtlasStruct entity, EntityMutationContext context, EntityMutations.EntityOperation operation) throws AtlasBaseException { + + } +} From 126da8e16d88097f415cf1e7b4b29f77b63204f2 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 24 Sep 2024 09:08:40 +0530 Subject: [PATCH 37/55] associations for entity and attribute --- .../model/AbstractModelPreProcessor.java | 371 ++++++++++++++++-- .../DMAttributeAssociationPreprocessor.java | 104 ++++- .../model/DMAttributePreprocessor.java | 212 +--------- .../DMEntityAssociationPreProcessor.java | 88 ++++- .../model/DMEntityPreProcessor.java | 177 +-------- 5 files changed, 538 insertions(+), 414 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index 7cc4ef7551..92f498384b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -21,6 +21,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.ListUtils; import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; import org.elasticsearch.common.Strings; import org.mockito.internal.util.collections.ListUtil; import org.slf4j.Logger; @@ -30,6 +31,7 @@ import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; public abstract class AbstractModelPreProcessor implements PreProcessor { @@ -188,7 +190,6 @@ protected ModelResponse replicateModelAttribute(AtlasEntity existingAttribute, A AtlasVertex copyAttributeVertex = entityGraphMapper.createVertex(existingAttribute); AtlasEntity copyAttributeEntity = entityRetriever.toAtlasEntity(copyAttributeVertex); copyAllAttributes(existingAttribute, copyAttributeEntity, epoch); - // copyEntity.setRelationshipAttributes(entity.getRelationshipAttributes()); setModelDates(copyAttributeEntity, copyAttributeVertex, epoch); String attributeQualifiedName = attributeQualifiedNamePrefix + "_" + epoch; setQualifiedName(copyAttributeEntity, copyAttributeVertex, attributeQualifiedName); @@ -321,41 +322,28 @@ protected void copyAllAttributes(AtlasEntity source, AtlasEntity destination, lo destination.setClassifications(new ArrayList<>()); // Set empty list if source classifications are null or empty } - Set allowedRelations = allowedRelationshipsForEntityType(source.getTypeName()); + String entityType = source.getTypeName(); - Map destinationRelationAttributes = new HashMap<>(); - if (source.getRelationshipAttributes() != null) { - Map sourceAttributes = source.getRelationshipAttributes(); - for (String attribute : sourceAttributes.keySet()) { - if (allowedRelations.contains(attribute)) { - destinationRelationAttributes.put(attribute, sourceAttributes.get(attribute)); - } - } - } + copyRelationshipAttributes(source.getRelationshipAttributes(), destination, entityType); + copyRelationshipAttributes(source.getAppendRelationshipAttributes(), destination, entityType); + copyRelationshipAttributes(source.getRemoveRelationshipAttributes(), destination, entityType); + } - LinkedHashMap destinationAppendAttributes = new LinkedHashMap<>(); - if (source.getAppendRelationshipAttributes() != null) { - LinkedHashMap sourceAttributes = (LinkedHashMap) source.getAppendRelationshipAttributes(); - for (String attribute : sourceAttributes.keySet()) { - if (allowedRelations.contains(attribute)) { - destinationAppendAttributes.put(attribute, sourceAttributes.get(attribute)); - } - } + private void copyRelationshipAttributes(Map sourceAttributes, AtlasEntity destination, String entityType) { + if (MapUtils.isNotEmpty(sourceAttributes)) { + return; } - LinkedHashMap destinationRemoveAttributes = new LinkedHashMap<>(); - if (source.getRemoveRelationshipAttributes() != null) { - LinkedHashMap sourceAttributes = (LinkedHashMap) source.getRemoveRelationshipAttributes(); - for (String attribute : sourceAttributes.keySet()) { - if (allowedRelations.contains(attribute)) { - destinationRemoveAttributes.put(attribute, sourceAttributes.get(attribute)); - } + Map destinationAttributes = new HashMap<>(); + Set allowedRelations = allowedRelationshipsForEntityType(entityType); + for (String attribute : sourceAttributes.keySet()) { + if (allowedRelations.contains(attribute)) { + destinationAttributes.put(attribute, sourceAttributes.get(attribute)); } } - + destination.setAttributes(destinationAttributes); } - public static void replaceAttributes(Map existingAttributes, Map diffAttributes) { if (MapUtils.isEmpty(diffAttributes)) { return; @@ -420,9 +408,175 @@ protected Set allowedRelationshipsForEntityType(String entityType) { } return allowedRelationships; } - protected void resolveReferences(AtlasEntity entity, EntityMutationContext context) { - if (entity.getRelationshipAttributes() != null) { - Map appendAttributesSource = (Map) entity.getRelationshipAttributes(); + + protected ModelResponse updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + if (!entity.getTypeName().equals(ATLAS_DM_ENTITY_TYPE)) { + return new ModelResponse(entity, vertex); + } + + String entityName = (String) entity.getAttribute(NAME); + + if (StringUtils.isEmpty(entityName) || isNameInvalid(entityName)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); + } + + + long now = RequestContext.get().getRequestTime(); + AtlasEntity.AtlasEntityWithExtInfo existingEntity = entityRetriever.toAtlasEntityWithExtInfo(vertex, false); + List existingEntityAttributes = (List) existingEntity.getEntity().getRelationshipAttributes().get("dMAttributes"); + + + // get model qualifiedName with qualifiedNamePrefix + String qualifiedNamePrefix = (String) entity.getAttributes().get(ATLAS_DM_QUALIFIED_NAME_PREFIX); + int lastIndex = qualifiedNamePrefix.lastIndexOf("/"); + String modelQualifiedName = qualifiedNamePrefix.substring(0, lastIndex); + String modelVersion = "v1"; + + + Map attrValues = new HashMap<>(); + attrValues.put(QUALIFIED_NAME, modelQualifiedName); + + AtlasVertex modelVertex = AtlasGraphUtilsV2.findByUniqueAttributes( + typeRegistry.getEntityTypeByName(ATLAS_DM_DATA_MODEL), attrValues); + + String modelGuid = AtlasGraphUtilsV2.getIdFromVertex(modelVertex); + ModelResponse modelVersionResponse = replicateModelVersion(modelGuid, modelQualifiedName, now); + + // model is not replicated successfully + if (modelVersionResponse.getCopyEntity() == null) { + String namespace = (String) entity.getAttributes().get(ATLAS_DM_NAMESPACE); + modelVersionResponse = createEntity( + (modelQualifiedName + "/" + modelVersion), + ATLAS_DM_VERSION_TYPE, + namespace, + context); + } + + AtlasEntity latestModelVersionEntity = modelVersionResponse.getCopyEntity(); + AtlasVertex latestModelVersionVertex = modelVersionResponse.getCopyVertex(); + AtlasEntity existingVersion = modelVersionResponse.getExistingEntity(); + + // create entity e1 ---> e1' + ModelResponse modelEntityResponse = replicateModelEntity(existingEntity.getEntity(), vertex, qualifiedNamePrefix, now); + AtlasVertex copyEntityVertex = modelEntityResponse.getCopyVertex(); + AtlasEntity copyEntity = modelEntityResponse.getCopyEntity(); + applyDiffs(entity, copyEntity, ATLAS_DM_ENTITY_TYPE); + unsetExpiredDates(copyEntity, copyEntityVertex); + + // create model-modelVersion relation + createModelModelVersionRelation(modelGuid, latestModelVersionEntity.getGuid()); + + // create modelVersion-modelEntity relationship with new entity + createModelVersionModelEntityRelationship(latestModelVersionVertex, copyEntityVertex); + + // create modelVersion-modelEntity relation with old entities which are not expired + if (existingVersion != null) { + List existingEntities = (List) existingVersion.getRelationshipAttributes().get("dMEntities"); + createModelVersionModelEntityRelationship(latestModelVersionVertex, existingEntities); + } + // create modelEntity-modelAttributeRelationship + createModelEntityModelAttributeRelation(copyEntityVertex, existingEntityAttributes); + + + /** + * update context + */ + // previousEntity and previousModelVersion + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); + AtlasEntityType modelVersionType = typeRegistry.getEntityTypeByName(modelVersionResponse.getCopyEntity().getTypeName()); + + context.addCreated(copyEntity.getGuid(), copyEntity, entityType, copyEntityVertex); + context.addCreated(latestModelVersionEntity.getGuid(), latestModelVersionEntity, modelVersionType, latestModelVersionVertex); + + //remove existing entity from context so it is not updated + context.removeUpdated(entity.getGuid(), entity, + entityType, vertex); + + // resolve references + context.getDiscoveryContext(). + addResolvedGuid( + modelGuid, + modelVertex); + + return new ModelResponse(copyEntity, copyEntityVertex); + } + + protected Map processRelationshipAttributesForEntity(AtlasEntity entity, Map relationshipAttributes, EntityMutationContext context) throws AtlasBaseException { + Map appendAttributesDestination = new HashMap<>(); + if (relationshipAttributes != null) { + Map appendAttributesSource = (Map) relationshipAttributes; + ; + ModelResponse modelResponseRelatedEntity = null; + String guid = ""; + Set allowedRelations = allowedRelationshipsForEntityType(entity.getTypeName()); + + for (String attribute : appendAttributesSource.keySet()) { + + if (appendAttributesSource.get(attribute) instanceof List) { + + if (!allowedRelations.contains(attribute)) { + continue; + } + List> destList = new ArrayList<>(); + Map destMap = null; + + List> attributeList = (List>) appendAttributesSource.get(attribute); + + for (Map relationAttribute : attributeList) { + guid = (String) relationAttribute.get("guid"); + + // update end2 + modelResponseRelatedEntity = updateDMEntity( + entityRetriever.toAtlasEntity(guid), + entityRetriever.getEntityVertex(guid), + context); + //relationAttribute.put("guid", modelResponseRelatedEntity.getCopyEntity()); + destMap = new HashMap<>(relationAttribute); + guid = modelResponseRelatedEntity.getCopyEntity().getGuid(); + destMap.put("guid", guid); + //destMap.put(QUALIFIED_NAME, ) + context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); + destList.add(destMap); + } + appendAttributesDestination.put(attribute, destList); + } else { + if (appendAttributesSource.get(attribute) instanceof Map) { + LinkedHashMap attributeList = (LinkedHashMap) appendAttributesSource.get(attribute); + guid = (String) attributeList.get("guid"); + + // update end2 + modelResponseRelatedEntity = updateDMEntity( + entityRetriever.toAtlasEntity(guid), + entityRetriever.getEntityVertex(guid), + context); + + Map destMap = new HashMap<>(attributeList); + destMap.put("guid", guid); + guid = modelResponseRelatedEntity.getCopyEntity().getGuid(); + context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); + appendAttributesDestination.put(attribute, destMap); + } + } + } + } + return appendAttributesDestination; + } + + protected ModelResponse replicateDMAssociation(AtlasEntity existingEntity, AtlasVertex existingEntityVertex, long epoch) throws AtlasBaseException { + AtlasVertex copyEntityVertex = entityGraphMapper.createVertex(existingEntity); + AtlasEntity copyEntity = entityRetriever.toAtlasEntity(copyEntityVertex); + copyAllAttributes(existingEntity, copyEntity, epoch); + setModelDates(copyEntity, copyEntityVertex, epoch); + setModelDates(copyEntity, copyEntityVertex, epoch); + setModelExpiredAtDates(existingEntity, existingEntityVertex, epoch); + return new ModelResponse(existingEntity, copyEntity, existingEntityVertex, copyEntityVertex); + } + + protected Map processRelationshipAttributesForAttribute(AtlasEntity entity, Map relationshipAttributes, EntityMutationContext context) throws AtlasBaseException { + Map appendAttributesDestination = new HashMap<>(); + if (relationshipAttributes != null) { + Map appendAttributesSource = (Map) relationshipAttributes; + ; ModelResponse modelResponseRelatedEntity = null; String guid = ""; Set allowedRelations = allowedRelationshipsForEntityType(entity.getTypeName()); @@ -434,21 +588,176 @@ protected void resolveReferences(AtlasEntity entity, EntityMutationContext conte if (!allowedRelations.contains(attribute)) { continue; } + List> destList = new ArrayList<>(); + Map destMap = null; + List> attributeList = (List>) appendAttributesSource.get(attribute); for (Map relationAttribute : attributeList) { guid = (String) relationAttribute.get("guid"); + + // update end2 + modelResponseRelatedEntity = updateDMAttribute( + entityRetriever.toAtlasEntity(guid), + entityRetriever.getEntityVertex(guid), + context); + //relationAttribute.put("guid", modelResponseRelatedEntity.getCopyEntity()); + destMap = new HashMap<>(relationAttribute); + guid = modelResponseRelatedEntity.getCopyEntity().getGuid(); + destMap.put("guid", guid); + //destMap.put(QUALIFIED_NAME, ) context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); + destList.add(destMap); } + appendAttributesDestination.put(attribute, destList); } else { if (appendAttributesSource.get(attribute) instanceof Map) { LinkedHashMap attributeList = (LinkedHashMap) appendAttributesSource.get(attribute); guid = (String) attributeList.get("guid"); + + // update end2 + modelResponseRelatedEntity = updateDMAttribute( + entityRetriever.toAtlasEntity(guid), + entityRetriever.getEntityVertex(guid), + context); + + Map destMap = new HashMap<>(attributeList); + destMap.put("guid", guid); + guid = modelResponseRelatedEntity.getCopyEntity().getGuid(); context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); + appendAttributesDestination.put(attribute, destMap); } } } } + return appendAttributesDestination; + } + + protected ModelResponse updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAttribute, EntityMutationContext context) throws AtlasBaseException { + if (!entityAttribute.getTypeName().equals(ATLAS_DM_ATTRIBUTE_TYPE)) { + return new ModelResponse(entityAttribute, vertexAttribute); + } + + String attributeName = (String) entityAttribute.getAttribute(NAME); + + if (StringUtils.isEmpty(attributeName) || isNameInvalid(attributeName)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); + } + + long now = RequestContext.get().getRequestTime(); + + + // get entity qualifiedName with qualifiedNamePrefix + String attributeQualifiedNamePrefix = (String) entityAttribute.getAttributes().get(ATLAS_DM_QUALIFIED_NAME_PREFIX); + int lastIndex = attributeQualifiedNamePrefix.lastIndexOf("/"); + String entityQualifiedNamePrefix = attributeQualifiedNamePrefix.substring(0, lastIndex); + String namespace = (String) entityAttribute.getAttributes().get(ATLAS_DM_NAMESPACE); + String modelVersion = "v2"; + + ModelResponse modelENtityResponse = null; + AtlasVertex latestEntityVertex = AtlasGraphUtilsV2.findLatestEntityAttributeVerticesByType(ATLAS_DM_ENTITY_TYPE, entityQualifiedNamePrefix); + + // get model qualifiedName with qualifiedNamePrefix + lastIndex = entityQualifiedNamePrefix.lastIndexOf("/"); + String modelQualifiedName = entityQualifiedNamePrefix.substring(0, lastIndex); + Map attrValues = new HashMap<>(); + attrValues.put(QUALIFIED_NAME, modelQualifiedName); + + String modelGuid = AtlasGraphUtilsV2.getGuidByUniqueAttributes( + typeRegistry.getEntityTypeByName(ATLAS_DM_DATA_MODEL), + attrValues); + + List existingAttributes = null; + + if (latestEntityVertex != null) { + modelENtityResponse = replicateModelEntity( + entityRetriever.toAtlasEntity(latestEntityVertex), + latestEntityVertex, + entityQualifiedNamePrefix, + now + ); + if (modelENtityResponse.getExistingEntity() != null && modelENtityResponse.getExistingEntity().getRelationshipAttributes() != null) { + existingAttributes = (List) modelENtityResponse.getExistingEntity().getAttributes().get("dMAttributes"); + } + } else { + modelENtityResponse = createEntity( + attributeQualifiedNamePrefix + "_" + now, + ATLAS_DM_ATTRIBUTE_TYPE, + namespace, + context + ); + } + + ModelResponse modelVersionResponse = replicateModelVersion(modelGuid, modelQualifiedName, now); + + if (modelVersionResponse.getCopyEntity() == null) { + modelVersionResponse = createEntity( + (modelQualifiedName + "/" + modelVersion), + ATLAS_DM_VERSION_TYPE, + namespace, + context); + } + AtlasEntity latestModelVersionEntity = modelVersionResponse.getCopyEntity(); + AtlasVertex latestModelVersionVertex = modelVersionResponse.getCopyVertex(); + + List existingEntities = null; + + if (modelVersionResponse.getExistingEntity() != null && modelVersionResponse.getExistingEntity().getRelationshipAttributes() != null) { + existingEntities = (List) modelVersionResponse.getExistingEntity().getRelationshipAttributes().get("dMEntities"); + } + + AtlasEntity existingEntityAttributeWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(entityAttribute.getGuid(), false).getEntity(); + // create attribute a1 ---> a1' + ModelResponse modelAttributeResponse = replicateModelAttribute( + existingEntityAttributeWithExtInfo, + entityRetriever.getEntityVertex(entityAttribute.getGuid()), + attributeQualifiedNamePrefix, + now); + + AtlasVertex copyAttributeVertex = modelAttributeResponse.getCopyVertex(); + AtlasEntity copyAttribute = modelAttributeResponse.getCopyEntity(); + applyDiffs(entityAttribute, copyAttribute, ATLAS_DM_ATTRIBUTE_TYPE); + unsetExpiredDates(copyAttribute, copyAttributeVertex); + + // create model-modelVersion relationship + createModelModelVersionRelation(modelGuid, latestModelVersionEntity.getGuid()); + + // create modelVersion-entity relationship [with new entity] + createModelVersionModelEntityRelationship(latestModelVersionVertex, modelENtityResponse.getCopyVertex()); + + // create modelVersion-entity relationship [with existing entities] + createModelVersionModelEntityRelationship(latestModelVersionVertex, existingEntities); + + + // create entity - attribute relation [with new attribute] + createModelEntityModelAttributeRelation(modelENtityResponse.getCopyVertex(), copyAttributeVertex); + + // create entity - attribute relation [with existing attributes] + createModelEntityModelAttributeRelation(modelENtityResponse.getCopyVertex(), existingAttributes); + + AtlasEntityType attributeType = typeRegistry.getEntityTypeByName(entityAttribute.getTypeName()); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(modelENtityResponse.getCopyEntity().getTypeName()); + AtlasEntityType modelVersionType = typeRegistry.getEntityTypeByName(latestModelVersionEntity.getTypeName()); + + context.addCreated(copyAttribute.getGuid(), copyAttribute, attributeType, copyAttributeVertex); + context.addCreated(modelENtityResponse.getCopyEntity().getGuid(), modelENtityResponse.getCopyEntity(), + entityType, modelENtityResponse.getCopyVertex()); + context.addCreated(latestModelVersionEntity.getGuid(), + latestModelVersionEntity, modelVersionType, latestModelVersionVertex); + + context.removeUpdated(entityAttribute.getGuid(), entityAttribute, + entityType, vertexAttribute); + + // resolve references + context.getDiscoveryContext(). + addResolvedGuid( + modelGuid, + entityRetriever.getEntityVertex(modelGuid)); + + return new ModelResponse(copyAttribute, copyAttributeVertex); } + } + + diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributeAssociationPreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributeAssociationPreprocessor.java index 2ce06bdcad..6341e7ff5c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributeAssociationPreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributeAssociationPreprocessor.java @@ -1,2 +1,104 @@ -package org.apache.atlas.repository.store.graph.v2.preprocessor.model;public class DMAttributeAssociationPreprocessor { +package org.apache.atlas.repository.store.graph.v2.preprocessor.model; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; +import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.Constants.ATLAS_DM_ENTITY_ASSOCIATION_TYPE; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; + +public class DMAttributeAssociationPreprocessor extends AbstractModelPreProcessor{ + private static final Logger LOG = LoggerFactory.getLogger(DMAttributePreprocessor.class); + + public DMAttributeAssociationPreprocessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, EntityGraphMapper entityGraphMapper, AtlasRelationshipStore atlasRelationshipStore) { + super(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore); + } + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, + EntityMutations.EntityOperation operation) throws AtlasBaseException { + + if (LOG.isDebugEnabled()) { + LOG.debug("ModelPreProcessor.processAttributes: pre processing {}, {}", + entityStruct.getAttribute(QUALIFIED_NAME), operation); + } + + AtlasEntity entity = (AtlasEntity) entityStruct; + AtlasVertex vertex = context.getVertex(entity.getGuid()); + + switch (operation) { + case CREATE: + createDMAttributeAssociation(entity, vertex, context); + break; + case UPDATE: + updateDMAttributeAssociation(entity, vertex, context); + } + } + + private void createDMAttributeAssociation(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + if (!entity.getTypeName().equals(ATLAS_DM_ATTRIBUTE_TYPE)) { + return; + } + + String entityName = (String) entity.getAttribute(NAME); + + if (StringUtils.isEmpty(entityName) || isNameInvalid(entityName)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); + } + + entity.setRelationshipAttributes( + processRelationshipAttributesForAttribute(entity, entity.getRelationshipAttributes(), context)); + } + private void updateDMAttributeAssociation(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + if (!entity.getTypeName().equals(ATLAS_DM_ENTITY_TYPE)) { + return; + } + + String entityName = (String) entity.getAttribute(NAME); + + if (StringUtils.isEmpty(entityName) || isNameInvalid(entityName)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); + } + + + long now = RequestContext.get().getRequestTime(); + + ModelResponse modelResponse = replicateDMAssociation(entity, vertex, now); + AtlasEntity copyEntity = modelResponse.getCopyEntity(); + AtlasVertex copyVertex = modelResponse.getCopyVertex(); + applyDiffs(entity, copyEntity, ATLAS_DM_ATTRIBUTE_ASSOCIATION_TYPE); + unsetExpiredDates(copyEntity, copyVertex); + + + // case when a mapping is added + if (entity.getAppendRelationshipAttributes() != null) { + Map appendRelationshipAttributes = processRelationshipAttributesForAttribute(entity, entity.getAppendRelationshipAttributes(), context); + modelResponse.getCopyEntity().setAppendRelationshipAttributes(appendRelationshipAttributes); + context.removeUpdatedWithRelationshipAttributes(entity); + context.setUpdatedWithRelationshipAttributes(modelResponse.getCopyEntity()); + } + + if (entity.getRemoveRelationshipAttributes() != null) { + Map appendRelationshipAttributes = processRelationshipAttributesForAttribute(entity, entity.getRemoveRelationshipAttributes(), context); + modelResponse.getCopyEntity().setRemoveRelationshipAttributes(appendRelationshipAttributes); + context.removeUpdatedWithDeleteRelationshipAttributes(entity); + context.setUpdatedWithRemoveRelationshipAttributes(modelResponse.getCopyEntity()); + } + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java index a92a7abe22..47ca78c51f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java @@ -20,24 +20,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.time.Instant; import java.util.*; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; public class DMAttributePreprocessor extends AbstractModelPreProcessor { - private static final Logger LOG = LoggerFactory.getLogger(AbstractModelPreProcessor.class); - Set allowedRelationshipNames; - + private static final Logger LOG = LoggerFactory.getLogger(DMAttributePreprocessor.class); public DMAttributePreprocessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, EntityGraphMapper entityGraphMapper, AtlasRelationshipStore atlasRelationshipStore) { super(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore); - allowedRelationshipNames = new HashSet<>(); - allowedRelationshipNames.add("dMMappedFromAttributes"); - allowedRelationshipNames.add("dMMappedToAttributes"); - allowedRelationshipNames.add("dMRelatedFromAttributes"); - allowedRelationshipNames.add("dMRelatedToAttributes"); } @@ -162,11 +154,7 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt modelGuid, entityRetriever.getEntityVertex(modelGuid)); - /*** - * debug why its already not there in context - */ - resolveReferences(entityAttribute, context); - + entityAttribute.setRelationshipAttributes(processRelationshipAttributesForAttribute(entityAttribute, entityAttribute.getRelationshipAttributes(), context)); } private void updateDMAttributes(AtlasEntity entityAttribute, AtlasVertex vertexAttribute, EntityMutationContext context) throws AtlasBaseException { @@ -174,208 +162,18 @@ private void updateDMAttributes(AtlasEntity entityAttribute, AtlasVertex vertexA // case when a mapping is added if (entityAttribute.getAppendRelationshipAttributes() != null) { - Map appendRelationshipAttributes = processAppendRemoveRelationshipAttributes(entityAttribute, entityAttribute.getAppendRelationshipAttributes(), context); + Map appendRelationshipAttributes = processRelationshipAttributesForAttribute(entityAttribute, entityAttribute.getAppendRelationshipAttributes(), context); modelResponseParentEntity.getCopyEntity().setAppendRelationshipAttributes(appendRelationshipAttributes); context.removeUpdatedWithRelationshipAttributes(entityAttribute); context.setUpdatedWithRelationshipAttributes(modelResponseParentEntity.getCopyEntity()); } if (entityAttribute.getRemoveRelationshipAttributes() != null) { - Map appendRelationshipAttributes = processAppendRemoveRelationshipAttributes(entityAttribute, entityAttribute.getAppendRelationshipAttributes(), context); + Map appendRelationshipAttributes = processRelationshipAttributesForAttribute(entityAttribute, entityAttribute.getAppendRelationshipAttributes(), context); modelResponseParentEntity.getCopyEntity().setRemoveRelationshipAttributes(appendRelationshipAttributes); context.removeUpdatedWithDeleteRelationshipAttributes(entityAttribute); context.setUpdatedWithRemoveRelationshipAttributes(modelResponseParentEntity.getCopyEntity()); } } - private Map processAppendRemoveRelationshipAttributes(AtlasEntity entity, Map relationshipAttributes, EntityMutationContext context) throws AtlasBaseException { - Map appendAttributesDestination = new HashMap<>(); - if (relationshipAttributes != null) { - Map appendAttributesSource = (Map) relationshipAttributes; - ; - ModelResponse modelResponseRelatedEntity = null; - String guid = ""; - Set allowedRelations = allowedRelationshipsForEntityType(entity.getTypeName()); - - for (String attribute : appendAttributesSource.keySet()) { - - if (appendAttributesSource.get(attribute) instanceof List) { - - if (!allowedRelations.contains(attribute)) { - continue; - } - List> destList = new ArrayList<>(); - Map destMap = null; - - List> attributeList = (List>) appendAttributesSource.get(attribute); - - for (Map relationAttribute : attributeList) { - guid = (String) relationAttribute.get("guid"); - - // update end2 - modelResponseRelatedEntity = updateDMAttribute( - entityRetriever.toAtlasEntity(guid), - entityRetriever.getEntityVertex(guid), - context); - //relationAttribute.put("guid", modelResponseRelatedEntity.getCopyEntity()); - destMap = new HashMap<>(relationAttribute); - guid = modelResponseRelatedEntity.getCopyEntity().getGuid(); - destMap.put("guid", guid); - //destMap.put(QUALIFIED_NAME, ) - context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); - destList.add(destMap); - } - appendAttributesDestination.put(attribute, destList); - } else { - if (appendAttributesSource.get(attribute) instanceof Map) { - LinkedHashMap attributeList = (LinkedHashMap) appendAttributesSource.get(attribute); - guid = (String) attributeList.get("guid"); - - // update end2 - modelResponseRelatedEntity = updateDMAttribute( - entityRetriever.toAtlasEntity(guid), - entityRetriever.getEntityVertex(guid), - context); - - Map destMap = new HashMap<>(attributeList); - destMap.put("guid", guid); - guid = modelResponseRelatedEntity.getCopyEntity().getGuid(); - context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); - appendAttributesDestination.put(attribute, destMap); - } - } - } - } - return appendAttributesDestination; - } - - - private ModelResponse updateDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAttribute, EntityMutationContext context) throws AtlasBaseException { - if (!entityAttribute.getTypeName().equals(ATLAS_DM_ATTRIBUTE_TYPE)) { - return new ModelResponse(entityAttribute, vertexAttribute); - } - - String attributeName = (String) entityAttribute.getAttribute(NAME); - - if (StringUtils.isEmpty(attributeName) || isNameInvalid(attributeName)) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); - } - - long now = RequestContext.get().getRequestTime(); - - - // get entity qualifiedName with qualifiedNamePrefix - String attributeQualifiedNamePrefix = (String) entityAttribute.getAttributes().get(ATLAS_DM_QUALIFIED_NAME_PREFIX); - int lastIndex = attributeQualifiedNamePrefix.lastIndexOf("/"); - String entityQualifiedNamePrefix = attributeQualifiedNamePrefix.substring(0, lastIndex); - String namespace = (String) entityAttribute.getAttributes().get(ATLAS_DM_NAMESPACE); - String modelVersion = "v2"; - - ModelResponse modelENtityResponse = null; - AtlasVertex latestEntityVertex = AtlasGraphUtilsV2.findLatestEntityAttributeVerticesByType(ATLAS_DM_ENTITY_TYPE, entityQualifiedNamePrefix); - - // get model qualifiedName with qualifiedNamePrefix - lastIndex = entityQualifiedNamePrefix.lastIndexOf("/"); - String modelQualifiedName = entityQualifiedNamePrefix.substring(0, lastIndex); - Map attrValues = new HashMap<>(); - attrValues.put(QUALIFIED_NAME, modelQualifiedName); - - String modelGuid = AtlasGraphUtilsV2.getGuidByUniqueAttributes( - typeRegistry.getEntityTypeByName(ATLAS_DM_DATA_MODEL), - attrValues); - - List existingAttributes = null; - - if (latestEntityVertex != null) { - modelENtityResponse = replicateModelEntity( - entityRetriever.toAtlasEntity(latestEntityVertex), - latestEntityVertex, - entityQualifiedNamePrefix, - now - ); - if (modelENtityResponse.getExistingEntity() != null && modelENtityResponse.getExistingEntity().getRelationshipAttributes() != null) { - existingAttributes = (List) modelENtityResponse.getExistingEntity().getAttributes().get("dMAttributes"); - } - } else { - modelENtityResponse = createEntity( - attributeQualifiedNamePrefix + "_" + now, - ATLAS_DM_ATTRIBUTE_TYPE, - namespace, - context - ); - } - - ModelResponse modelVersionResponse = replicateModelVersion(modelGuid, modelQualifiedName, now); - - if (modelVersionResponse.getCopyEntity() == null) { - modelVersionResponse = createEntity( - (modelQualifiedName + "/" + modelVersion), - ATLAS_DM_VERSION_TYPE, - namespace, - context); - } - AtlasEntity latestModelVersionEntity = modelVersionResponse.getCopyEntity(); - AtlasVertex latestModelVersionVertex = modelVersionResponse.getCopyVertex(); - - List existingEntities = null; - - if (modelVersionResponse.getExistingEntity() != null && modelVersionResponse.getExistingEntity().getRelationshipAttributes() != null) { - existingEntities = (List) modelVersionResponse.getExistingEntity().getRelationshipAttributes().get("dMEntities"); - } - - AtlasEntity existingEntityAttributeWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(entityAttribute.getGuid(), false).getEntity(); - - // create attribute a1 ---> a1' - ModelResponse modelAttributeResponse = replicateModelAttribute( - existingEntityAttributeWithExtInfo, - entityRetriever.getEntityVertex(entityAttribute.getGuid()), - attributeQualifiedNamePrefix, - now); - - AtlasVertex copyAttributeVertex = modelAttributeResponse.getCopyVertex(); - AtlasEntity copyAttribute = modelAttributeResponse.getCopyEntity(); - applyDiffs(entityAttribute, copyAttribute, ATLAS_DM_ATTRIBUTE_TYPE); - unsetExpiredDates(copyAttribute, copyAttributeVertex); - - // create model-modelVersion relationship - createModelModelVersionRelation(modelGuid, latestModelVersionEntity.getGuid()); - - // create modelVersion-entity relationship [with new entity] - createModelVersionModelEntityRelationship(latestModelVersionVertex, modelENtityResponse.getCopyVertex()); - - // create modelVersion-entity relationship [with existing entities] - createModelVersionModelEntityRelationship(latestModelVersionVertex, existingEntities); - - - // create entity - attribute relation [with new attribute] - createModelEntityModelAttributeRelation(modelENtityResponse.getCopyVertex(), copyAttributeVertex); - - // create entity - attribute relation [with existing attributes] - createModelEntityModelAttributeRelation(modelENtityResponse.getCopyVertex(), existingAttributes); - - AtlasEntityType attributeType = typeRegistry.getEntityTypeByName(entityAttribute.getTypeName()); - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(modelENtityResponse.getCopyEntity().getTypeName()); - AtlasEntityType modelVersionType = typeRegistry.getEntityTypeByName(latestModelVersionEntity.getTypeName()); - - context.addCreated(copyAttribute.getGuid(), copyAttribute, attributeType, copyAttributeVertex); - context.addCreated(modelENtityResponse.getCopyEntity().getGuid(), modelENtityResponse.getCopyEntity(), - entityType, modelENtityResponse.getCopyVertex()); - context.addCreated(latestModelVersionEntity.getGuid(), - latestModelVersionEntity, modelVersionType, latestModelVersionVertex); - - context.removeUpdated(entityAttribute.getGuid(), entityAttribute, - entityType, vertexAttribute); - - // resolve references - context.getDiscoveryContext(). - addResolvedGuid( - modelGuid, - entityRetriever.getEntityVertex(modelGuid)); - - /*** - * debug why its already not there in context - */ - resolveReferences(entityAttribute, context); - return new ModelResponse(copyAttribute, copyAttributeVertex); - } -} \ No newline at end of file +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityAssociationPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityAssociationPreProcessor.java index e893fdc614..a7d8f6a189 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityAssociationPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityAssociationPreProcessor.java @@ -1,21 +1,103 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.model; +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class EntityAssociationPreProcessor extends AbstractModelPreProcessor{ - public EntityAssociationPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, EntityGraphMapper entityGraphMapper, AtlasRelationshipStore atlasRelationshipStore) { +import java.util.*; + +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; + +public class DMEntityAssociationPreProcessor extends AbstractModelPreProcessor { + + private static final Logger LOG = LoggerFactory.getLogger(DMAttributePreprocessor.class); + + public DMEntityAssociationPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, EntityGraphMapper entityGraphMapper, AtlasRelationshipStore atlasRelationshipStore) { super(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore); } @Override - public void processAttributes(AtlasStruct entity, EntityMutationContext context, EntityMutations.EntityOperation operation) throws AtlasBaseException { + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, + EntityMutations.EntityOperation operation) throws AtlasBaseException { + + if (LOG.isDebugEnabled()) { + LOG.debug("ModelPreProcessor.processAttributes: pre processing {}, {}", + entityStruct.getAttribute(QUALIFIED_NAME), operation); + } + + AtlasEntity entity = (AtlasEntity) entityStruct; + AtlasVertex vertex = context.getVertex(entity.getGuid()); + + switch (operation) { + case CREATE: + createDMEntityAssociation(entity, vertex, context); + break; + case UPDATE: + updateDMEntityAssociation(entity, vertex, context); + } + } + + private void createDMEntityAssociation(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + if (!entity.getTypeName().equals(ATLAS_DM_ENTITY_ASSOCIATION_TYPE)) { + return; + } + + String entityName = (String) entity.getAttribute(NAME); + + if (StringUtils.isEmpty(entityName) || isNameInvalid(entityName)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); + } + + entity.setRelationshipAttributes( + processRelationshipAttributesForEntity(entity, entity.getRelationshipAttributes(), context)); + } + private void updateDMEntityAssociation(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + if (!entity.getTypeName().equals(ATLAS_DM_ENTITY_TYPE)) { + return; + } + + String entityName = (String) entity.getAttribute(NAME); + + if (StringUtils.isEmpty(entityName) || isNameInvalid(entityName)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); + } + + + long now = RequestContext.get().getRequestTime(); + + ModelResponse modelResponse = replicateDMAssociation(entity, vertex, now); + AtlasEntity copyEntity = modelResponse.getCopyEntity(); + AtlasVertex copyVertex = modelResponse.getCopyVertex(); + applyDiffs(entity, copyEntity, ATLAS_DM_ENTITY_ASSOCIATION_TYPE); + unsetExpiredDates(copyEntity, copyVertex); + + + // case when a mapping is added + if (entity.getAppendRelationshipAttributes() != null) { + Map appendRelationshipAttributes = processRelationshipAttributesForEntity(entity, entity.getAppendRelationshipAttributes(), context); + modelResponse.getCopyEntity().setAppendRelationshipAttributes(appendRelationshipAttributes); + context.removeUpdatedWithRelationshipAttributes(entity); + context.setUpdatedWithRelationshipAttributes(modelResponse.getCopyEntity()); + } + if (entity.getRemoveRelationshipAttributes() != null) { + Map appendRelationshipAttributes = processRelationshipAttributesForEntity(entity, entity.getRemoveRelationshipAttributes(), context); + modelResponse.getCopyEntity().setRemoveRelationshipAttributes(appendRelationshipAttributes); + context.removeUpdatedWithDeleteRelationshipAttributes(entity); + context.setUpdatedWithRemoveRelationshipAttributes(modelResponse.getCopyEntity()); + } } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java index c1a4ffed13..4baf0d5675 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -13,7 +13,6 @@ import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; -import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; @@ -26,17 +25,11 @@ import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; public class DMEntityPreProcessor extends AbstractModelPreProcessor { - private static final Logger LOG = LoggerFactory.getLogger(AbstractModelPreProcessor.class); - Set allowedRelationshipNames; + private static final Logger LOG = LoggerFactory.getLogger(DMEntityPreProcessor.class); public DMEntityPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, EntityGraphMapper entityGraphMapper, AtlasRelationshipStore atlasRelationshipStore) { super(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore); - allowedRelationshipNames = new HashSet<>(); - allowedRelationshipNames.add("dMMappedToEntities"); - allowedRelationshipNames.add("dMMappedFromEntities"); - allowedRelationshipNames.add("dMRelatedFromEntities"); - allowedRelationshipNames.add("dMRelatedToEntities"); } @@ -132,10 +125,8 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati modelGuid, entityRetriever.getEntityVertex(modelGuid)); - /** - * Optimise this : validate why reference is not created in legacy code - */ - resolveReferences(entity, context); + entity.setRelationshipAttributes( + processRelationshipAttributesForEntity(entity, entity.getRelationshipAttributes(), context)); } private void updateDMEntities(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { @@ -143,177 +134,19 @@ private void updateDMEntities(AtlasEntity entity, AtlasVertex vertex, EntityMuta // case when a mapping is added if (entity.getAppendRelationshipAttributes() != null) { - Map appendRelationshipAttributes = processAppendRemoveRelationshipAttributes(entity, entity.getAppendRelationshipAttributes(), context); + Map appendRelationshipAttributes = processRelationshipAttributesForEntity(entity, entity.getAppendRelationshipAttributes(), context); modelResponseParentEntity.getCopyEntity().setAppendRelationshipAttributes(appendRelationshipAttributes); context.removeUpdatedWithRelationshipAttributes(entity); context.setUpdatedWithRelationshipAttributes(modelResponseParentEntity.getCopyEntity()); } if (entity.getRemoveRelationshipAttributes() != null) { - Map appendRelationshipAttributes = processAppendRemoveRelationshipAttributes(entity, entity.getRemoveRelationshipAttributes(), context); + Map appendRelationshipAttributes = processRelationshipAttributesForEntity(entity, entity.getRemoveRelationshipAttributes(), context); modelResponseParentEntity.getCopyEntity().setRemoveRelationshipAttributes(appendRelationshipAttributes); context.removeUpdatedWithDeleteRelationshipAttributes(entity); context.setUpdatedWithRemoveRelationshipAttributes(modelResponseParentEntity.getCopyEntity()); } } - - private Map processAppendRemoveRelationshipAttributes(AtlasEntity entity, Map relationshipAttributes, EntityMutationContext context) throws AtlasBaseException { - Map appendAttributesDestination = new HashMap<>(); - if (relationshipAttributes != null) { - Map appendAttributesSource = (Map) relationshipAttributes; - ; - ModelResponse modelResponseRelatedEntity = null; - String guid = ""; - Set allowedRelations = allowedRelationshipsForEntityType(entity.getTypeName()); - - for (String attribute : appendAttributesSource.keySet()) { - - if (appendAttributesSource.get(attribute) instanceof List) { - - if (!allowedRelations.contains(attribute)) { - continue; - } - List> destList = new ArrayList<>(); - Map destMap = null; - - List> attributeList = (List>) appendAttributesSource.get(attribute); - - for (Map relationAttribute : attributeList) { - guid = (String) relationAttribute.get("guid"); - - // update end2 - modelResponseRelatedEntity = updateDMEntity( - entityRetriever.toAtlasEntity(guid), - entityRetriever.getEntityVertex(guid), - context); - //relationAttribute.put("guid", modelResponseRelatedEntity.getCopyEntity()); - destMap = new HashMap<>(relationAttribute); - guid = modelResponseRelatedEntity.getCopyEntity().getGuid(); - destMap.put("guid", guid); - //destMap.put(QUALIFIED_NAME, ) - context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); - destList.add(destMap); - } - appendAttributesDestination.put(attribute, destList); - } else { - if (appendAttributesSource.get(attribute) instanceof Map) { - LinkedHashMap attributeList = (LinkedHashMap) appendAttributesSource.get(attribute); - guid = (String) attributeList.get("guid"); - - // update end2 - modelResponseRelatedEntity = updateDMEntity( - entityRetriever.toAtlasEntity(guid), - entityRetriever.getEntityVertex(guid), - context); - - Map destMap = new HashMap<>(attributeList); - destMap.put("guid", guid); - guid = modelResponseRelatedEntity.getCopyEntity().getGuid(); - context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); - appendAttributesDestination.put(attribute, destMap); - } - } - } - } - return appendAttributesDestination; - } - - private ModelResponse updateDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { - if (!entity.getTypeName().equals(ATLAS_DM_ENTITY_TYPE)) { - return new ModelResponse(entity, vertex); - } - - String entityName = (String) entity.getAttribute(NAME); - - if (StringUtils.isEmpty(entityName) || isNameInvalid(entityName)) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); - } - - - long now = RequestContext.get().getRequestTime(); - AtlasEntity.AtlasEntityWithExtInfo existingEntity = entityRetriever.toAtlasEntityWithExtInfo(vertex, false); - List existingEntityAttributes = (List) existingEntity.getEntity().getRelationshipAttributes().get("dMAttributes"); - - - // get model qualifiedName with qualifiedNamePrefix - String qualifiedNamePrefix = (String) entity.getAttributes().get(ATLAS_DM_QUALIFIED_NAME_PREFIX); - int lastIndex = qualifiedNamePrefix.lastIndexOf("/"); - String modelQualifiedName = qualifiedNamePrefix.substring(0, lastIndex); - String modelVersion = "v1"; - - - Map attrValues = new HashMap<>(); - attrValues.put(QUALIFIED_NAME, modelQualifiedName); - - AtlasVertex modelVertex = AtlasGraphUtilsV2.findByUniqueAttributes( - typeRegistry.getEntityTypeByName(ATLAS_DM_DATA_MODEL), attrValues); - - String modelGuid = AtlasGraphUtilsV2.getIdFromVertex(modelVertex); - ModelResponse modelVersionResponse = replicateModelVersion(modelGuid, modelQualifiedName, now); - - // model is not replicated successfully - if (modelVersionResponse.getCopyEntity() == null) { - String namespace = (String) entity.getAttributes().get(ATLAS_DM_NAMESPACE); - modelVersionResponse = createEntity( - (modelQualifiedName + "/" + modelVersion), - ATLAS_DM_VERSION_TYPE, - namespace, - context); - } - - AtlasEntity latestModelVersionEntity = modelVersionResponse.getCopyEntity(); - AtlasVertex latestModelVersionVertex = modelVersionResponse.getCopyVertex(); - AtlasEntity existingVersion = modelVersionResponse.getExistingEntity(); - - // create entity e1 ---> e1' - ModelResponse modelEntityResponse = replicateModelEntity(existingEntity.getEntity(), vertex, qualifiedNamePrefix, now); - AtlasVertex copyEntityVertex = modelEntityResponse.getCopyVertex(); - AtlasEntity copyEntity = modelEntityResponse.getCopyEntity(); - applyDiffs(entity, copyEntity, ATLAS_DM_ENTITY_TYPE); - unsetExpiredDates(copyEntity, copyEntityVertex); - - // create model-modelVersion relation - createModelModelVersionRelation(modelGuid, latestModelVersionEntity.getGuid()); - - // create modelVersion-modelEntity relationship with new entity - createModelVersionModelEntityRelationship(latestModelVersionVertex, copyEntityVertex); - - // create modelVersion-modelEntity relation with old entities which are not expired - if (existingVersion != null) { - List existingEntities = (List) existingVersion.getRelationshipAttributes().get("dMEntities"); - createModelVersionModelEntityRelationship(latestModelVersionVertex, existingEntities); - } - // create modelEntity-modelAttributeRelationship - createModelEntityModelAttributeRelation(copyEntityVertex, existingEntityAttributes); - - - /** - * update context - */ - // previousEntity and previousModelVersion - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); - AtlasEntityType modelVersionType = typeRegistry.getEntityTypeByName(modelVersionResponse.getCopyEntity().getTypeName()); - - context.addCreated(copyEntity.getGuid(), copyEntity, entityType, copyEntityVertex); - context.addCreated(latestModelVersionEntity.getGuid(), latestModelVersionEntity, modelVersionType, latestModelVersionVertex); - - //remove existing entity from context so it is not updated - context.removeUpdated(entity.getGuid(), entity, - entityType, vertex); - - // resolve references - context.getDiscoveryContext(). - addResolvedGuid( - modelGuid, - modelVertex); - - /*** - * debug why its already not there in context - */ - resolveReferences(entity, context); - - return new ModelResponse(copyEntity, copyEntityVertex); - } } From d79cc0dbc219810f9a5d1ddf83374943c1cabb95 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 24 Sep 2024 12:59:01 +0530 Subject: [PATCH 38/55] expose association preprocessors --- .../store/graph/v2/AtlasEntityStoreV2.java | 258 +++++++++--------- .../model/AbstractModelPreProcessor.java | 11 +- .../model/DMAttributePreprocessor.java | 2 +- 3 files changed, 144 insertions(+), 127 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 71b810cc79..e609d2ef3f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -68,6 +68,7 @@ import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.GlossaryPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.TermPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.model.DMAttributePreprocessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.model.DMEntityAssociationPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.model.DMEntityPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.LinkPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.ReadmePreProcessor; @@ -112,7 +113,6 @@ import static org.apache.atlas.type.Constants.*; - @Component public class AtlasEntityStoreV2 implements AtlasEntityStore { private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityStoreV2.class); @@ -122,16 +122,16 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore { private static final String ATTR_MEANINGS = "meanings"; - private final AtlasGraph graph; - private final DeleteHandlerDelegate deleteDelegate; - private final RestoreHandlerV1 restoreHandlerV1; - private final AtlasTypeRegistry typeRegistry; + private final AtlasGraph graph; + private final DeleteHandlerDelegate deleteDelegate; + private final RestoreHandlerV1 restoreHandlerV1; + private final AtlasTypeRegistry typeRegistry; private final IAtlasEntityChangeNotifier entityChangeNotifier; - private final EntityGraphMapper entityGraphMapper; - private final EntityGraphRetriever entityRetriever; - private boolean storeDifferentialAudits; - private final GraphHelper graphHelper; - private final TaskManagement taskManagement; + private final EntityGraphMapper entityGraphMapper; + private final EntityGraphRetriever entityRetriever; + private boolean storeDifferentialAudits; + private final GraphHelper graphHelper; + private final TaskManagement taskManagement; private EntityDiscoveryService discovery; private final AtlasRelationshipStore atlasRelationshipStore; private final FeatureFlagStore featureFlagStore; @@ -145,15 +145,15 @@ public AtlasEntityStoreV2(AtlasGraph graph, DeleteHandlerDelegate deleteDelegate AtlasRelationshipStore atlasRelationshipStore, FeatureFlagStore featureFlagStore, IAtlasMinimalChangeNotifier atlasAlternateChangeNotifier) { - this.graph = graph; - this.deleteDelegate = deleteDelegate; - this.restoreHandlerV1 = restoreHandlerV1; - this.typeRegistry = typeRegistry; + this.graph = graph; + this.deleteDelegate = deleteDelegate; + this.restoreHandlerV1 = restoreHandlerV1; + this.typeRegistry = typeRegistry; this.entityChangeNotifier = entityChangeNotifier; - this.entityGraphMapper = entityGraphMapper; - this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry); + this.entityGraphMapper = entityGraphMapper; + this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry); this.storeDifferentialAudits = STORE_DIFFERENTIAL_AUDITS.getBoolean(); - this.graphHelper = new GraphHelper(graph); + this.graphHelper = new GraphHelper(graph); this.taskManagement = taskManagement; this.atlasRelationshipStore = atlasRelationshipStore; this.featureFlagStore = featureFlagStore; @@ -285,8 +285,8 @@ public AtlasEntitiesWithExtInfo getByIds(List guids, boolean isMinExtInf AtlasEntitiesWithExtInfo ret = entityRetriever.toAtlasEntitiesWithExtInfo(guids, isMinExtInfo); - if(ret != null){ - for(String guid : guids) { + if (ret != null) { + for (String guid : guids) { AtlasEntity entity = ret.getEntity(guid); try { AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_READ, new AtlasEntityHeader(entity)), "read entity: guid=", guid); @@ -318,7 +318,7 @@ public AtlasEntitiesWithExtInfo getByIds(List guids, boolean isMinExtInf @Override @GraphTransaction - public AtlasEntitiesWithExtInfo getEntitiesByUniqueAttributes(AtlasEntityType entityType, List> uniqueAttributes , boolean isMinExtInfo, boolean ignoreRelationships) throws AtlasBaseException { + public AtlasEntitiesWithExtInfo getEntitiesByUniqueAttributes(AtlasEntityType entityType, List> uniqueAttributes, boolean isMinExtInfo, boolean ignoreRelationships) throws AtlasBaseException { if (LOG.isDebugEnabled()) { LOG.debug("==> getEntitiesByUniqueAttributes({}, {})", entityType.getTypeName(), uniqueAttributes); } @@ -377,7 +377,7 @@ public AtlasEntityWithExtInfo getByUniqueAttributes(AtlasEntityType entityType, @Override @GraphTransaction public AtlasEntityHeader getAtlasEntityHeaderWithoutAuthorization(String guid, String qualifiedName, String typeName) throws AtlasBaseException { - return extractEntityHeader( guid, qualifiedName, typeName); + return extractEntityHeader(guid, qualifiedName, typeName); } @Override @@ -409,6 +409,7 @@ public AtlasEntityHeader getEntityHeaderByUniqueAttributes(AtlasEntityType entit /** * Check state of entities in the store + * * @param request AtlasCheckStateRequest * @return AtlasCheckStateResult * @throws AtlasBaseException @@ -439,7 +440,7 @@ public EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean @Override @GraphTransaction - public EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean replaceClassifications, + public EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean replaceClassifications, boolean replaceBusinessAttributes, boolean isOverwriteBusinessAttributes) throws AtlasBaseException { return createOrUpdate(entityStream, false, replaceClassifications, replaceBusinessAttributes, isOverwriteBusinessAttributes); } @@ -505,7 +506,7 @@ public EntityMutationResponse updateByUniqueAttributes(AtlasEntityType entityTyp throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "no entity to update."); } - String guid = AtlasGraphUtilsV2.getGuidByUniqueAttributes(graph, entityType, uniqAttributes); + String guid = AtlasGraphUtilsV2.getGuidByUniqueAttributes(graph, entityType, uniqAttributes); AtlasEntity entity = updatedEntityInfo.getEntity(); entity.setGuid(guid); @@ -523,9 +524,9 @@ public EntityMutationResponse updateEntityAttributeByGuid(String guid, String at LOG.debug("==> updateEntityAttributeByGuid({}, {}, {})", guid, attrName, attrValue); } - AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeaderWithClassifications(guid); - AtlasEntityType entityType = (AtlasEntityType) typeRegistry.getType(entity.getTypeName()); - AtlasAttribute attr = entityType.getAttribute(attrName); + AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeaderWithClassifications(guid); + AtlasEntityType entityType = (AtlasEntityType) typeRegistry.getType(entity.getTypeName()); + AtlasAttribute attr = entityType.getAttribute(attrName); AtlasAuthorizationUtils.verifyUpdateEntityAccess(typeRegistry, entity, "update entity ByUniqueAttributes : guid=" + guid); @@ -537,7 +538,7 @@ public EntityMutationResponse updateEntityAttributeByGuid(String guid, String at } } - AtlasType attrType = attr.getAttributeType(); + AtlasType attrType = attr.getAttributeType(); AtlasEntity updateEntity = new AtlasEntity(); updateEntity.setGuid(guid); @@ -577,7 +578,7 @@ public EntityMutationResponse deleteById(final String guid) throws AtlasBaseExce } Collection deletionCandidates = new ArrayList<>(); - AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(graph, guid); + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(graph, guid); if (vertex != null) { AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(vertex); @@ -595,7 +596,7 @@ public EntityMutationResponse deleteById(final String guid) throws AtlasBaseExce EntityMutationResponse ret = deleteVertices(deletionCandidates); - if(ret.getDeletedEntities()!=null) + if (ret.getDeletedEntities() != null) processTermEntityDeletion(ret.getDeletedEntities()); // Notify the change listeners @@ -639,7 +640,7 @@ public EntityMutationResponse deleteByIds(final List guids) throws Atlas EntityMutationResponse ret = deleteVertices(deletionCandidates); - if(ret.getDeletedEntities() != null) + if (ret.getDeletedEntities() != null) processTermEntityDeletion(ret.getDeletedEntities()); // Notify the change listeners @@ -732,7 +733,7 @@ public EntityMutationResponse deleteByUniqueAttributes(AtlasEntityType entityTyp } Collection deletionCandidates = new ArrayList<>(); - AtlasVertex vertex = AtlasGraphUtilsV2.findByUniqueAttributes(graph, entityType, uniqAttributes); + AtlasVertex vertex = AtlasGraphUtilsV2.findByUniqueAttributes(graph, entityType, uniqAttributes); if (vertex != null) { AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(vertex); @@ -751,7 +752,7 @@ public EntityMutationResponse deleteByUniqueAttributes(AtlasEntityType entityTyp EntityMutationResponse ret = deleteVertices(deletionCandidates); - if(ret.getDeletedEntities()!=null) + if (ret.getDeletedEntities() != null) processTermEntityDeletion(ret.getDeletedEntities()); // Notify the change listeners @@ -818,19 +819,19 @@ public EntityMutationResponse deleteByUniqueAttributes(List objec return ret; } - private void processTermEntityDeletion(List deletedEntities) throws AtlasBaseException{ - for(AtlasEntityHeader entity:deletedEntities){ - if(ATLAS_GLOSSARY_TERM_ENTITY_TYPE.equals(entity.getTypeName())){ + private void processTermEntityDeletion(List deletedEntities) throws AtlasBaseException { + for (AtlasEntityHeader entity : deletedEntities) { + if (ATLAS_GLOSSARY_TERM_ENTITY_TYPE.equals(entity.getTypeName())) { - String termQualifiedName = entity.getAttribute(QUALIFIED_NAME).toString(); - String termName = entity.getAttribute(NAME).toString(); - String guid = entity.getGuid(); - Boolean isHardDelete = DeleteType.HARD.name().equals(entity.getDeleteHandler()); + String termQualifiedName = entity.getAttribute(QUALIFIED_NAME).toString(); + String termName = entity.getAttribute(NAME).toString(); + String guid = entity.getGuid(); + Boolean isHardDelete = DeleteType.HARD.name().equals(entity.getDeleteHandler()); - if(checkEntityTermAssociation(termQualifiedName)){ - if(DEFERRED_ACTION_ENABLED && taskManagement!=null){ + if (checkEntityTermAssociation(termQualifiedName)) { + if (DEFERRED_ACTION_ENABLED && taskManagement != null) { createAndQueueTask(termName, termQualifiedName, guid, isHardDelete); - }else{ + } else { updateMeaningsNamesInEntitiesOnTermDelete(termName, termQualifiedName, guid); } } @@ -838,11 +839,11 @@ private void processTermEntityDeletion(List deletedEntities) } } - private boolean checkEntityTermAssociation(String termQName) throws AtlasBaseException{ + private boolean checkEntityTermAssociation(String termQName) throws AtlasBaseException { List entityHeader; try { - entityHeader = discovery.searchUsingTermQualifiedName(0, 1, termQName,null, null); + entityHeader = discovery.searchUsingTermQualifiedName(0, 1, termQName, null, null); } catch (AtlasBaseException e) { throw e; } @@ -854,10 +855,10 @@ private boolean checkEntityTermAssociation(String termQName) throws AtlasBaseExc public void updateMeaningsNamesInEntitiesOnTermDelete(String termName, String termQName, String termGuid) throws AtlasBaseException { int from = 0; - Set attributes = new HashSet(){{ + Set attributes = new HashSet() {{ add(ATTR_MEANINGS); }}; - Set relationAttributes = new HashSet(){{ + Set relationAttributes = new HashSet() {{ add(STATE_PROPERTY_KEY); add(NAME); }}; @@ -892,13 +893,13 @@ public void updateMeaningsNamesInEntitiesOnTermDelete(String termName, String te } - public void createAndQueueTask(String termName, String termQName, String termGuid, Boolean isHardDelete){ + public void createAndQueueTask(String termName, String termQName, String termGuid, Boolean isHardDelete) { String taskType = isHardDelete ? UPDATE_ENTITY_MEANINGS_ON_TERM_HARD_DELETE : UPDATE_ENTITY_MEANINGS_ON_TERM_SOFT_DELETE; String currentUser = RequestContext.getCurrentUser(); Map taskParams = MeaningsTask.toParameters(termName, termQName, termGuid); AtlasTask task = taskManagement.createTask(taskType, currentUser, taskParams); - if(!isHardDelete){ + if (!isHardDelete) { AtlasVertex termVertex = AtlasGraphUtilsV2.findByGuid(termGuid); AtlasGraphUtilsV2.addEncodedProperty(termVertex, PENDING_TASKS_PROPERTY_KEY, task.getGuid()); } @@ -909,7 +910,7 @@ public void createAndQueueTask(String termName, String termQName, String termGui @Override @GraphTransaction - public String getGuidByUniqueAttributes(AtlasEntityType entityType, Map uniqAttributes) throws AtlasBaseException{ + public String getGuidByUniqueAttributes(AtlasEntityType entityType, Map uniqAttributes) throws AtlasBaseException { return AtlasGraphUtilsV2.getGuidByUniqueAttributes(graph, entityType, uniqAttributes); } @@ -964,7 +965,7 @@ public void addClassifications(final String guid, final List> bm : businessAttrbutes.entrySet()) { @@ -1228,7 +1229,7 @@ public void addOrUpdateBusinessAttributesByDisplayName(String guid, Map attributes = new HashMap<>(); + Map attributes = new HashMap<>(); for (Map.Entry incomingAttrs : bm.getValue().entrySet()) { AtlasAttribute atlasAttribute = bmType.getAllAttributes().get(incomingAttrs.getKey()); @@ -1286,8 +1287,8 @@ public void removeBusinessAttributes(String guid, Map labels) throws AtlasBaseException validateLabels(labels); - AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(entityVertex); - Set addedLabels = Collections.emptySet(); - Set removedLabels = Collections.emptySet(); + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(entityVertex); + Set addedLabels = Collections.emptySet(); + Set removedLabels = Collections.emptySet(); if (CollectionUtils.isEmpty(entityHeader.getLabels())) { addedLabels = labels; } else if (CollectionUtils.isEmpty(labels)) { removedLabels = entityHeader.getLabels(); } else { - addedLabels = new HashSet(CollectionUtils.subtract(labels, entityHeader.getLabels())); + addedLabels = new HashSet(CollectionUtils.subtract(labels, entityHeader.getLabels())); removedLabels = new HashSet(CollectionUtils.subtract(entityHeader.getLabels(), labels)); } @@ -1376,7 +1377,7 @@ public void removeLabels(String guid, Set labels) throws AtlasBaseExcept throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid); } - AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(entityVertex); + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(entityVertex); AtlasEntityAccessRequestBuilder requestBuilder = new AtlasEntityAccessRequestBuilder(typeRegistry, AtlasPrivilege.ENTITY_REMOVE_LABEL, entityHeader); for (String label : labels) { @@ -1415,7 +1416,7 @@ public void addLabels(String guid, Set labels) throws AtlasBaseException throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid); } - AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(entityVertex); + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(entityVertex); AtlasEntityAccessRequestBuilder requestBuilder = new AtlasEntityAccessRequestBuilder(typeRegistry, AtlasPrivilege.ENTITY_ADD_LABEL, entityHeader); for (String label : labels) { @@ -1466,17 +1467,17 @@ private EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean if (CollectionUtils.isNotEmpty(context.getUpdatedEntities())) { MetricRecorder checkForUnchangedEntities = RequestContext.get().startMetricRecord("checkForUnchangedEntities"); - List entitiesToSkipUpdate = new ArrayList<>(); - AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), !replaceClassifications, !replaceBusinessAttributes); - RequestContext reqContext = RequestContext.get(); + List entitiesToSkipUpdate = new ArrayList<>(); + AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), !replaceClassifications, !replaceBusinessAttributes); + RequestContext reqContext = RequestContext.get(); for (AtlasEntity entity : context.getUpdatedEntities()) { if (entity.getStatus() == AtlasEntity.Status.DELETED) {// entity status could be updated during import continue; } - AtlasVertex storedVertex = context.getVertex(entity.getGuid()); - AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, storedVertex, !storeDifferentialAudits); + AtlasVertex storedVertex = context.getVertex(entity.getGuid()); + AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, storedVertex, !storeDifferentialAudits); if (diffResult.hasDifference()) { if (storeDifferentialAudits) { @@ -1512,7 +1513,7 @@ private EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean AtlasEntityHeader entityHeaderWithClassifications = entityRetriever.toAtlasEntityHeaderWithClassifications(entity.getGuid()); AtlasEntityHeader entityHeader = new AtlasEntityHeader(entity); - if(CollectionUtils.isNotEmpty(entityHeaderWithClassifications.getClassifications())) { + if (CollectionUtils.isNotEmpty(entityHeaderWithClassifications.getClassifications())) { entityHeader.setClassifications(entityHeaderWithClassifications.getClassifications()); } @@ -1523,7 +1524,7 @@ private EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean if (skipAuthBaseConditions && (skipAuthMeaningsUpdate || skipAuthStarredDetailsUpdate)) { //do nothing, only diff is relationshipAttributes.meanings or starred, allow update } else { - AtlasAuthorizationUtils.verifyUpdateEntityAccess(typeRegistry, entityHeader,"update entity: type=" + entity.getTypeName()); + AtlasAuthorizationUtils.verifyUpdateEntityAccess(typeRegistry, entityHeader, "update entity: type=" + entity.getTypeName()); } } } @@ -1574,34 +1575,34 @@ private void executePreProcessor(EntityMutationContext context) throws AtlasBase for (AtlasEntity entity : copyOfCreated) { entityType = context.getType(entity.getGuid()); preProcessors = getPreProcessor(entityType.getTypeName()); - for(PreProcessor processor : preProcessors){ + for (PreProcessor processor : preProcessors) { processor.processAttributes(entity, context, CREATE); } } List copyOfUpdated = new ArrayList<>(context.getUpdatedEntities()); - for (AtlasEntity entity: copyOfUpdated) { + for (AtlasEntity entity : copyOfUpdated) { entityType = context.getType(entity.getGuid()); preProcessors = getPreProcessor(entityType.getTypeName()); - for(PreProcessor processor : preProcessors){ + for (PreProcessor processor : preProcessors) { processor.processAttributes(entity, context, UPDATE); } } List copyOfAppendRelationshipAttributes = new ArrayList<>(context.getUpdatedEntitiesForAppendRelationshipAttribute()); - for (AtlasEntity entity: copyOfAppendRelationshipAttributes) { + for (AtlasEntity entity : copyOfAppendRelationshipAttributes) { entityType = context.getType(entity.getGuid()); preProcessors = getPreProcessor(entityType.getTypeName()); - for(PreProcessor processor : preProcessors){ + for (PreProcessor processor : preProcessors) { processor.processAttributes(entity, context, UPDATE); } } List copyOfRemoveRelationshipAttributes = new ArrayList<>(context.getEntitiesUpdatedWithRemoveRelationshipAttribute()); - for (AtlasEntity entity: copyOfRemoveRelationshipAttributes) { + for (AtlasEntity entity : copyOfRemoveRelationshipAttributes) { entityType = context.getType(entity.getGuid()); preProcessors = getPreProcessor(entityType.getTypeName()); - for(PreProcessor processor : preProcessors){ + for (PreProcessor processor : preProcessors) { processor.processAttributes(entity, context, UPDATE); } } @@ -1610,10 +1611,10 @@ private void executePreProcessor(EntityMutationContext context) throws AtlasBase private EntityMutationContext preCreateOrUpdate(EntityStream entityStream, EntityGraphMapper entityGraphMapper, boolean isPartialUpdate) throws AtlasBaseException { MetricRecorder metric = RequestContext.get().startMetricRecord("preCreateOrUpdate"); this.graph.setEnableCache(RequestContext.get().isCacheEnabled()); - EntityGraphDiscovery graphDiscoverer = new AtlasEntityGraphDiscoveryV2(graph, typeRegistry, entityStream, entityGraphMapper); + EntityGraphDiscovery graphDiscoverer = new AtlasEntityGraphDiscoveryV2(graph, typeRegistry, entityStream, entityGraphMapper); EntityGraphDiscoveryContext discoveryContext = graphDiscoverer.discoverEntities(); - EntityMutationContext context = new EntityMutationContext(discoveryContext); - RequestContext requestContext = RequestContext.get(); + EntityMutationContext context = new EntityMutationContext(discoveryContext); + RequestContext requestContext = RequestContext.get(); Map referencedGuids = discoveryContext.getReferencedGuids(); for (Map.Entry element : referencedGuids.entrySet()) { @@ -1668,7 +1669,7 @@ private EntityMutationContext preCreateOrUpdate(EntityStream entityStream, Entit graphDiscoverer.validateAndNormalize(entity); - // Handle create flow here + // Handle create flow here //Create vertices which do not exist in the repository if (RequestContext.get().isImportInProgress() && AtlasTypeUtil.isAssignedGuid(entity.getGuid())) { vertex = entityGraphMapper.createVertexWithGuid(entity, entity.getGuid()); @@ -1798,7 +1799,7 @@ private void autoUpdateStarredDetailsAttributes(AtlasEntity entity, AtlasVertex private void addUserToStarredAttributes(String requestUser, long requestTime, Set starredBy, Set starredDetailsList) { //Check and update starredBy Attribute - if (!starredBy.contains(requestUser)){ + if (!starredBy.contains(requestUser)) { starredBy.add(requestUser); } @@ -1820,7 +1821,7 @@ private void addUserToStarredAttributes(String requestUser, long requestTime, Se private void removeUserFromStarredAttributes(String requestUser, Set starredBy, Set starredDetailsList) { //Check and update starredBy Attribute - if (starredBy.contains(requestUser)){ + if (starredBy.contains(requestUser)) { starredBy.remove(requestUser); } @@ -1958,6 +1959,11 @@ public List getPreProcessor(String typeName) { case ATLAS_DM_ATTRIBUTE_TYPE: preProcessors.add(new DMAttributePreprocessor(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore)); break; + case ATLAS_DM_ENTITY_ASSOCIATION_TYPE: + preProcessors.add(new DMEntityAssociationPreProcessor(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore)); + break; + case ATLAS_DM_ATTRIBUTE_ASSOCIATION_TYPE: + preProcessors.add(new DMEntityAssociationPreProcessor(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore)); } // The default global pre-processor for all AssetTypes @@ -1968,7 +1974,7 @@ public List getPreProcessor(String typeName) { private AtlasVertex getResolvedEntityVertex(EntityGraphDiscoveryContext context, AtlasEntity entity) throws AtlasBaseException { AtlasObjectId objectId = getAtlasObjectId(entity); - AtlasVertex ret = context.getResolvedEntityVertex(entity.getGuid()); + AtlasVertex ret = context.getResolvedEntityVertex(entity.getGuid()); if (ret != null) { @@ -2011,7 +2017,7 @@ private EntityMutationResponse deleteVertices(Collection deletionCa String typeName = getTypeName(vertex); List preProcessors = getPreProcessor(typeName); - for(PreProcessor processor : preProcessors){ + for (PreProcessor processor : preProcessors) { processor.processDelete(vertex); } @@ -2061,7 +2067,7 @@ private EntityMutationResponse deleteVertices(Collection deletionCa private EntityMutationResponse restoreVertices(Collection restoreCandidates) throws AtlasBaseException { EntityMutationResponse response = new EntityMutationResponse(); - RequestContext req = RequestContext.get(); + RequestContext req = RequestContext.get(); restoreHandlerV1.restoreEntities(restoreCandidates); @@ -2074,7 +2080,7 @@ private EntityMutationResponse restoreVertices(Collection restoreCa private EntityMutationResponse purgeVertices(Collection purgeCandidates) throws AtlasBaseException { EntityMutationResponse response = new EntityMutationResponse(); - RequestContext req = RequestContext.get(); + RequestContext req = RequestContext.get(); req.setDeleteType(DeleteType.HARD); req.setPurgeRequested(true); @@ -2112,15 +2118,15 @@ private void validateAndNormalize(AtlasClassification classification) throws Atl * @param classifications list of classifications to be associated */ private void validateEntityAssociations(String guid, List classifications) throws AtlasBaseException { - List entityClassifications = getClassificationNames(guid); - String entityTypeName = AtlasGraphUtilsV2.getTypeNameFromGuid(graph, guid); - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entityTypeName); + List entityClassifications = getClassificationNames(guid); + String entityTypeName = AtlasGraphUtilsV2.getTypeNameFromGuid(graph, guid); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entityTypeName); Set processedTagTypeNames = new HashSet<>(); - List copyList = new ArrayList<>(classifications); + List copyList = new ArrayList<>(classifications); for (AtlasClassification classification : copyList) { - if (processedTagTypeNames.contains(classification.getTypeName())){ + if (processedTagTypeNames.contains(classification.getTypeName())) { classifications.remove(classification); } else { String newClassification = classification.getTypeName(); @@ -2142,7 +2148,7 @@ private void validateEntityAssociations(String guid, List c } private List getClassificationNames(String guid) throws AtlasBaseException { - List ret = null; + List ret = null; List classifications = retrieveClassifications(guid); if (CollectionUtils.isNotEmpty(classifications)) { @@ -2222,12 +2228,12 @@ public BulkImportResponse bulkCreateOrUpdateBusinessAttributes(InputStream input throw new AtlasBaseException(AtlasErrorCode.FILE_NAME_NOT_FOUND, fileName); } - List fileData = FileUtils.readFileData(fileName, inputStream); + List fileData = FileUtils.readFileData(fileName, inputStream); Map attributesToAssociate = getBusinessMetadataDefList(fileData, ret); for (AtlasEntity entity : attributesToAssociate.values()) { Map> businessAttributes = entity.getBusinessAttributes(); - String guid = entity.getGuid(); + String guid = entity.getGuid(); try { addOrUpdateBusinessAttributes(guid, businessAttributes, true); @@ -2250,7 +2256,8 @@ public List getAccessors(List atlas for (AtlasAccessorRequest accessorRequest : atlasAccessorRequestList) { try { AtlasAccessorResponse result = null; - AtlasPrivilege action = AtlasPrivilege.valueOf(accessorRequest.getAction());; + AtlasPrivilege action = AtlasPrivilege.valueOf(accessorRequest.getAction()); + ; switch (action) { case ENTITY_READ: @@ -2368,13 +2375,13 @@ private AtlasEntityHeader extractEntityHeader(String guid, String qualifiedName, } private Map getBusinessMetadataDefList(List fileData, BulkImportResponse bulkImportResponse) throws AtlasBaseException { - Map ret = new HashMap<>(); - Map vertexCache = new HashMap<>(); - List failedMsgList = new ArrayList<>(); + Map ret = new HashMap<>(); + Map vertexCache = new HashMap<>(); + List failedMsgList = new ArrayList<>(); for (int lineIndex = 0; lineIndex < fileData.size(); lineIndex++) { - String[] record = fileData.get(lineIndex); - int lineIndexToLog = lineIndex + 2; + String[] record = fileData.get(lineIndex); + int lineIndexToLog = lineIndex + 2; boolean missingFields = record.length < FileUtils.UNIQUE_ATTR_NAME_COLUMN_INDEX || StringUtils.isBlank(record[FileUtils.TYPENAME_COLUMN_INDEX]) || @@ -2382,13 +2389,13 @@ private Map getBusinessMetadataDefList(List fileD StringUtils.isBlank(record[FileUtils.BM_ATTR_NAME_COLUMN_INDEX]) || StringUtils.isBlank(record[FileUtils.BM_ATTR_VALUE_COLUMN_INDEX]); - if (missingFields){ + if (missingFields) { failedMsgList.add("Line #" + lineIndexToLog + ": missing fields. " + Arrays.toString(record)); continue; } - String typeName = record[FileUtils.TYPENAME_COLUMN_INDEX]; + String typeName = record[FileUtils.TYPENAME_COLUMN_INDEX]; AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); if (entityType == null) { @@ -2397,10 +2404,10 @@ private Map getBusinessMetadataDefList(List fileD continue; } - String uniqueAttrValue = record[FileUtils.UNIQUE_ATTR_VALUE_COLUMN_INDEX]; - String bmAttribute = record[FileUtils.BM_ATTR_NAME_COLUMN_INDEX]; + String uniqueAttrValue = record[FileUtils.UNIQUE_ATTR_VALUE_COLUMN_INDEX]; + String bmAttribute = record[FileUtils.BM_ATTR_NAME_COLUMN_INDEX]; String bmAttributeValue = record[FileUtils.BM_ATTR_VALUE_COLUMN_INDEX]; - String uniqueAttrName = AtlasTypeUtil.ATTRIBUTE_QUALIFIED_NAME; + String uniqueAttrName = AtlasTypeUtil.ATTRIBUTE_QUALIFIED_NAME; if (record.length > FileUtils.UNIQUE_ATTR_NAME_COLUMN_INDEX && StringUtils.isNotBlank(record[FileUtils.UNIQUE_ATTR_NAME_COLUMN_INDEX])) { uniqueAttrName = record[FileUtils.UNIQUE_ATTR_NAME_COLUMN_INDEX]; @@ -2420,8 +2427,8 @@ private Map getBusinessMetadataDefList(List fileD continue; } - String vertexKey = uniqueAttribute.getVertexPropertyName() + "_" + uniqueAttrValue; - AtlasVertex vertex = vertexCache.get(vertexKey); + String vertexKey = uniqueAttribute.getVertexPropertyName() + "_" + uniqueAttrValue; + AtlasVertex vertex = vertexCache.get(vertexKey); if (vertex == null) { vertex = AtlasGraphUtilsV2.findByTypeAndUniquePropertyName(graph, typeName, uniqueAttribute.getVertexUniquePropertyName(), uniqueAttrValue); @@ -2438,7 +2445,7 @@ private Map getBusinessMetadataDefList(List fileD AtlasBusinessAttribute businessAttribute = entityType.getBusinesAAttribute(bmAttribute); if (businessAttribute == null) { - failedMsgList.add("Line #" + lineIndexToLog + ": invalid business-metadata '"+ bmAttribute + "' for entity type '" + entityType.getTypeName() + "'"); + failedMsgList.add("Line #" + lineIndexToLog + ": invalid business-metadata '" + bmAttribute + "' for entity type '" + entityType.getTypeName() + "'"); continue; } @@ -2447,12 +2454,12 @@ private Map getBusinessMetadataDefList(List fileD if (businessAttribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY) { AtlasArrayType arrayType = (AtlasArrayType) businessAttribute.getAttributeType(); - List arrayValue; + List arrayValue; if (arrayType.getElementType() instanceof AtlasEnumType) { - arrayValue = AtlasGraphUtilsV2.assignEnumValues(bmAttributeValue, (AtlasEnumType) arrayType.getElementType(), failedMsgList, lineIndex+1); + arrayValue = AtlasGraphUtilsV2.assignEnumValues(bmAttributeValue, (AtlasEnumType) arrayType.getElementType(), failedMsgList, lineIndex + 1); } else { - arrayValue = assignMultipleValues(bmAttributeValue, arrayType.getElementTypeName(), failedMsgList, lineIndex+1); + arrayValue = assignMultipleValues(bmAttributeValue, arrayType.getElementTypeName(), failedMsgList, lineIndex + 1); } attrValue = arrayValue; @@ -2465,8 +2472,8 @@ private Map getBusinessMetadataDefList(List fileD entity.setBusinessAttribute(businessAttribute.getDefinedInType().getTypeName(), businessAttribute.getName(), attrValue); } else { - AtlasEntity entity = new AtlasEntity(); - String guid = GraphHelper.getGuid(vertex); + AtlasEntity entity = new AtlasEntity(); + String guid = GraphHelper.getGuid(vertex); Map> businessAttributes = entityRetriever.getBusinessMetadata(vertex); entity.setGuid(guid); @@ -2531,14 +2538,14 @@ private List assignMultipleValues(String bmAttributeValues, String elementTypeNa return null; } - private boolean missingFieldsCheck(String[] record, BulkImportResponse bulkImportResponse, int lineIndex){ + private boolean missingFieldsCheck(String[] record, BulkImportResponse bulkImportResponse, int lineIndex) { boolean missingFieldsCheck = (record.length < FileUtils.UNIQUE_ATTR_NAME_COLUMN_INDEX) || StringUtils.isBlank(record[FileUtils.TYPENAME_COLUMN_INDEX]) || StringUtils.isBlank(record[FileUtils.UNIQUE_ATTR_VALUE_COLUMN_INDEX]) || StringUtils.isBlank(record[FileUtils.BM_ATTR_NAME_COLUMN_INDEX]) || StringUtils.isBlank(record[FileUtils.BM_ATTR_VALUE_COLUMN_INDEX]); - if(missingFieldsCheck){ + if (missingFieldsCheck) { LOG.error("Missing fields: " + Arrays.toString(record) + " at line #" + lineIndex); String failedTermMsgs = "Missing fields: " + Arrays.toString(record) + " at line #" + lineIndex; @@ -2584,7 +2591,7 @@ public void repairHasLineage(AtlasHasLineageRequests requests) throws AtlasBaseE edge = graphHelper.getEdge(processVertex, assetVertex, request.getLabel()); } else { LOG.warn("Skipping since vertex is null for processGuid {} and asset Guid {}" - ,request.getProcessGuid(),request.getEndGuid() ); + , request.getProcessGuid(), request.getEndGuid()); } } catch (RepositoryException re) { throw new AtlasBaseException(AtlasErrorCode.HAS_LINEAGE_GET_EDGE_FAILED, re); @@ -2639,7 +2646,7 @@ public void repairHasLineageWithAtlasEdges(Set inputOutputEdges) { for (AtlasEdge atlasEdge : inputOutputEdges) { if (getStatus(atlasEdge) != ACTIVE) { - LOG.warn("Edge id {} is not Active, so skipping " , getRelationshipGuid(atlasEdge)); + LOG.warn("Edge id {} is not Active, so skipping ", getRelationshipGuid(atlasEdge)); continue; } @@ -2718,7 +2725,7 @@ public void repairMeaningAttributeForTerms(List termGuid) { AtlasVertex termVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); - if(termVertex!= null && ATLAS_GLOSSARY_TERM_ENTITY_TYPE.equals(getTypeName(termVertex)) && + if (termVertex != null && ATLAS_GLOSSARY_TERM_ENTITY_TYPE.equals(getTypeName(termVertex)) && GraphHelper.getStatus(termVertex) == AtlasEntity.Status.ACTIVE) { Iterable edges = termVertex.getEdges(AtlasEdgeDirection.OUT, Constants.TERM_ASSIGNMENT_LABEL); // Get entity to tagged with term. @@ -2728,7 +2735,7 @@ public void repairMeaningAttributeForTerms(List termGuid) { if (GraphHelper.getStatus(edge) == AtlasEntity.Status.ACTIVE) { AtlasVertex entityVertex = edge.getInVertex(); if (entityVertex != null & getStatus(entityVertex) == AtlasEntity.Status.ACTIVE) { - if(!RequestContext.get().getProcessGuidIds().contains(getGuid(entityVertex))) { + if (!RequestContext.get().getProcessGuidIds().contains(getGuid(entityVertex))) { repairMeanings(entityVertex); } } @@ -2779,10 +2786,11 @@ private void repairMeanings(AtlasVertex assetVertex) { RequestContext.get().addProcessGuidIds(getGuid(assetVertex)); - LOG.info("Updated asset {} with term {} ", getGuid(assetVertex) , StringUtils.join(termNameList, ",")); + LOG.info("Updated asset {} with term {} ", getGuid(assetVertex), StringUtils.join(termNameList, ",")); } } + @Override public void repairAccesscontrolAlias(String guid) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("repairAlias"); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index 92f498384b..cf36c40d38 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -330,12 +330,13 @@ protected void copyAllAttributes(AtlasEntity source, AtlasEntity destination, lo } private void copyRelationshipAttributes(Map sourceAttributes, AtlasEntity destination, String entityType) { - if (MapUtils.isNotEmpty(sourceAttributes)) { + if (MapUtils.isEmpty(sourceAttributes)) { return; } Map destinationAttributes = new HashMap<>(); Set allowedRelations = allowedRelationshipsForEntityType(entityType); + for (String attribute : sourceAttributes.keySet()) { if (allowedRelations.contains(attribute)) { destinationAttributes.put(attribute, sourceAttributes.get(attribute)); @@ -405,6 +406,14 @@ protected Set allowedRelationshipsForEntityType(String entityType) { allowedRelationships.add("dMRelatedFromAttributes"); allowedRelationships.add("dMRelatedToAttributes"); break; + case ATLAS_DM_ENTITY_ASSOCIATION_TYPE: + allowedRelationships.add("dMEntityTo"); + allowedRelationships.add("dMEntityFrom"); + break; + case ATLAS_DM_ATTRIBUTE_ASSOCIATION_TYPE: + allowedRelationships.add("dMAttributeTo"); + allowedRelationships.add("dMAttributeFrom"); + break; } return allowedRelationships; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java index 47ca78c51f..3dda433863 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java @@ -105,7 +105,7 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt } else { modelENtityResponse = createEntity( attributeQualifiedNamePrefix + "_" + now, - ATLAS_DM_ATTRIBUTE_TYPE, + ATLAS_DM_ENTITY_TYPE, namespace, context ); From bc9ce3424bc9db4e994ff6dc27081b950f07965f Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 24 Sep 2024 13:07:39 +0530 Subject: [PATCH 39/55] add nam property while entity is created --- .../graph/v2/preprocessor/model/AbstractModelPreProcessor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index cf36c40d38..45795426e9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -118,6 +118,7 @@ protected void setName(AtlasEntity newEntity, AtlasVertex newVertex, Object valu protected ModelResponse createEntity(String qualifiedName, String entityType, String namespace, EntityMutationContext context) throws AtlasBaseException { String guid = UUID.randomUUID().toString(); AtlasEntity entity = new AtlasEntity(entityType); + entity.setAttribute(NAME, qualifiedName.substring(qualifiedName.lastIndexOf('/') + 1)); entity.setAttribute(VERSION_PROPERTY_KEY, 0); entity.setAttribute(QUALIFIED_NAME, qualifiedName); entity.setAttribute(ATLAS_DM_NAMESPACE, namespace); From 2c2c283c4a6bce15e8416e930dc6527ca84d1152 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 24 Sep 2024 14:53:48 +0530 Subject: [PATCH 40/55] add name attributes in preprocessors --- .../preprocessor/model/AbstractModelPreProcessor.java | 10 +++++++--- .../v2/preprocessor/model/DMAttributePreprocessor.java | 9 ++++++--- .../v2/preprocessor/model/DMEntityPreProcessor.java | 3 +++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index 45795426e9..019b4da214 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -124,6 +124,9 @@ protected ModelResponse createEntity(String qualifiedName, String entityType, St entity.setAttribute(ATLAS_DM_NAMESPACE, namespace); entity.setAttribute(ATLAS_DM_BUSINESS_DATE, RequestContext.get().getRequestTime()); entity.setAttribute(ATLAS_DM_SYSTEM_DATE, RequestContext.get().getRequestTime()); + if (entityType.equals(ATLAS_DM_ENTITY_TYPE) || entityType.equals(ATLAS_DM_ATTRIBUTE_TYPE)) { + entity.setAttribute(ATLAS_DM_QUALIFIED_NAME_PREFIX, qualifiedName.substring(0, qualifiedName.lastIndexOf('/'))); + } AtlasVertex versionVertex = entityGraphMapper.createVertexWithGuid(entity, guid); context.getDiscoveryContext().addResolvedGuid(guid, versionVertex); entity.setGuid(guid); @@ -662,7 +665,7 @@ protected ModelResponse updateDMAttribute(AtlasEntity entityAttribute, AtlasVert int lastIndex = attributeQualifiedNamePrefix.lastIndexOf("/"); String entityQualifiedNamePrefix = attributeQualifiedNamePrefix.substring(0, lastIndex); String namespace = (String) entityAttribute.getAttributes().get(ATLAS_DM_NAMESPACE); - String modelVersion = "v2"; + String modelVersion = "v1"; ModelResponse modelENtityResponse = null; AtlasVertex latestEntityVertex = AtlasGraphUtilsV2.findLatestEntityAttributeVerticesByType(ATLAS_DM_ENTITY_TYPE, entityQualifiedNamePrefix); @@ -686,13 +689,14 @@ protected ModelResponse updateDMAttribute(AtlasEntity entityAttribute, AtlasVert entityQualifiedNamePrefix, now ); + modelVersion = "v2"; if (modelENtityResponse.getExistingEntity() != null && modelENtityResponse.getExistingEntity().getRelationshipAttributes() != null) { existingAttributes = (List) modelENtityResponse.getExistingEntity().getAttributes().get("dMAttributes"); } } else { modelENtityResponse = createEntity( - attributeQualifiedNamePrefix + "_" + now, - ATLAS_DM_ATTRIBUTE_TYPE, + entityQualifiedNamePrefix + "_" + now, + ATLAS_DM_ENTITY_TYPE, namespace, context ); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java index 3dda433863..c564dd1392 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java @@ -75,7 +75,9 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt int lastIndex = attributeQualifiedNamePrefix.lastIndexOf("/"); String entityQualifiedNamePrefix = attributeQualifiedNamePrefix.substring(0, lastIndex); String namespace = (String) entityAttribute.getAttributes().get(ATLAS_DM_NAMESPACE); - String modelVersion = "v2"; + String modelVersion = "v1"; + + entityAttribute.setAttribute(NAME, attributeQualifiedNamePrefix + "_" + now); ModelResponse modelENtityResponse = null; AtlasVertex latestEntityVertex = AtlasGraphUtilsV2.findLatestEntityAttributeVerticesByType(ATLAS_DM_ENTITY_TYPE, entityQualifiedNamePrefix); @@ -91,6 +93,7 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt List existingAttributes = null; + // if (latestEntityVertex != null) { modelENtityResponse = replicateModelEntity( entityRetriever.toAtlasEntity(latestEntityVertex), @@ -98,13 +101,13 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt entityQualifiedNamePrefix, now ); - + modelVersion = "v2"; if (modelENtityResponse.getExistingEntity() != null && modelENtityResponse.getExistingEntity().getRelationshipAttributes() != null) { existingAttributes = (List) modelENtityResponse.getExistingEntity().getRelationshipAttributes().get("dMAttributes"); } } else { modelENtityResponse = createEntity( - attributeQualifiedNamePrefix + "_" + now, + entityQualifiedNamePrefix + "_" + now, ATLAS_DM_ENTITY_TYPE, namespace, context diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java index 4baf0d5675..fd432470b4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -76,6 +76,7 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati String qualifiedNamePrefix = (String) entity.getAttributes().get(ATLAS_DM_QUALIFIED_NAME_PREFIX); int lastIndex = qualifiedNamePrefix.lastIndexOf("/"); String modelQualifiedName = qualifiedNamePrefix.substring(0, lastIndex); + entity.setAttribute(NAME, qualifiedNamePrefix + "_" + now); Map attrValues = new HashMap<>(); attrValues.put(QUALIFIED_NAME, modelQualifiedName); @@ -88,8 +89,10 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati } String modelGuid = AtlasGraphUtilsV2.getIdFromVertex(modelVertex); + ModelResponse modelVersionResponse = replicateModelVersion(modelGuid, modelQualifiedName, now); + // create modelVersion if (modelVersionResponse.getCopyEntity() == null) { String namespace = (String) entity.getAttributes().get(ATLAS_DM_NAMESPACE); modelVersionResponse = createEntity( From 5b252bfdc7e3167961ac43de923bf26329b4ce6d Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 24 Sep 2024 17:12:55 +0530 Subject: [PATCH 41/55] fix preprocessor name --- .../atlas/repository/store/graph/v2/AtlasEntityStoreV2.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index e609d2ef3f..25be261f11 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1963,7 +1963,7 @@ public List getPreProcessor(String typeName) { preProcessors.add(new DMEntityAssociationPreProcessor(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore)); break; case ATLAS_DM_ATTRIBUTE_ASSOCIATION_TYPE: - preProcessors.add(new DMEntityAssociationPreProcessor(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore)); + preProcessors.add(new DMAttributePreprocessor(typeRegistry, entityRetriever, entityGraphMapper, atlasRelationshipStore)); } // The default global pre-processor for all AssetTypes From fea4ce4b4ff7313a0f835dbdca0826356747ed3b Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 25 Sep 2024 08:05:52 +0530 Subject: [PATCH 42/55] Remove name mappings --- .../model/AbstractModelPreProcessor.java | 12 +----------- .../preprocessor/model/DMAttributePreprocessor.java | 4 +--- .../v2/preprocessor/model/DMEntityPreProcessor.java | 2 -- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index 019b4da214..6482fc4068 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -109,12 +109,6 @@ protected void setQualifiedName(AtlasEntity newEntity, AtlasVertex newVertex, Ob AtlasGraphUtilsV2.setEncodedProperty(newVertex, QUALIFIED_NAME, value); } - protected void setName(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { - newEntity.setAttribute(NAME, value); - AtlasGraphUtilsV2.setEncodedProperty(newVertex, NAME, value); - } - - protected ModelResponse createEntity(String qualifiedName, String entityType, String namespace, EntityMutationContext context) throws AtlasBaseException { String guid = UUID.randomUUID().toString(); AtlasEntity entity = new AtlasEntity(entityType); @@ -130,7 +124,6 @@ protected ModelResponse createEntity(String qualifiedName, String entityType, St AtlasVertex versionVertex = entityGraphMapper.createVertexWithGuid(entity, guid); context.getDiscoveryContext().addResolvedGuid(guid, versionVertex); entity.setGuid(guid); - context.addCreated(guid, entity, typeRegistry.getEntityTypeByName(entityType), versionVertex); return new ModelResponse(entity, versionVertex); } @@ -172,7 +165,6 @@ protected ModelResponse replicateModelVersion(String modelGuid, String modelQual AtlasEntity copyModelVersion = entityRetriever.toAtlasEntity(copyModelVertex); copyAllAttributes(existingModelVersionEntity, copyModelVersion, now); setModelDates(copyModelVersion, copyModelVertex, now); - setName(copyModelVersion, copyModelVertex, modelVersion); setQualifiedName(copyModelVersion, copyModelVertex, modelQualifiedName + "/" + modelVersion); setModelExpiredAtDates(existingModelVersionEntity, existingModelVersionVertex, now); return new ModelResponse(existingModelVersionEntity, copyModelVersion, existingModelVersionVertex, copyModelVertex); @@ -182,7 +174,6 @@ protected ModelResponse replicateModelEntity(AtlasEntity existingEntity, AtlasVe AtlasVertex copyEntityVertex = entityGraphMapper.createVertex(existingEntity); AtlasEntity copyEntity = entityRetriever.toAtlasEntity(copyEntityVertex); copyAllAttributes(existingEntity, copyEntity, epoch); - setModelDates(copyEntity, copyEntityVertex, epoch); String entityQualifiedName = entityQualifiedNamePrefix + "_" + epoch; setQualifiedName(copyEntity, copyEntityVertex, entityQualifiedName); setModelDates(copyEntity, copyEntityVertex, epoch); @@ -194,7 +185,6 @@ protected ModelResponse replicateModelAttribute(AtlasEntity existingAttribute, A AtlasVertex copyAttributeVertex = entityGraphMapper.createVertex(existingAttribute); AtlasEntity copyAttributeEntity = entityRetriever.toAtlasEntity(copyAttributeVertex); copyAllAttributes(existingAttribute, copyAttributeEntity, epoch); - setModelDates(copyAttributeEntity, copyAttributeVertex, epoch); String attributeQualifiedName = attributeQualifiedNamePrefix + "_" + epoch; setQualifiedName(copyAttributeEntity, copyAttributeVertex, attributeQualifiedName); setModelDates(copyAttributeEntity, copyAttributeVertex, epoch); @@ -654,7 +644,7 @@ protected ModelResponse updateDMAttribute(AtlasEntity entityAttribute, AtlasVert String attributeName = (String) entityAttribute.getAttribute(NAME); if (StringUtils.isEmpty(attributeName) || isNameInvalid(attributeName)) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); + throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME, attributeName); } long now = RequestContext.get().getRequestTime(); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java index c564dd1392..5d571da955 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java @@ -77,8 +77,6 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt String namespace = (String) entityAttribute.getAttributes().get(ATLAS_DM_NAMESPACE); String modelVersion = "v1"; - entityAttribute.setAttribute(NAME, attributeQualifiedNamePrefix + "_" + now); - ModelResponse modelENtityResponse = null; AtlasVertex latestEntityVertex = AtlasGraphUtilsV2.findLatestEntityAttributeVerticesByType(ATLAS_DM_ENTITY_TYPE, entityQualifiedNamePrefix); lastIndex = entityQualifiedNamePrefix.lastIndexOf("/"); @@ -172,7 +170,7 @@ private void updateDMAttributes(AtlasEntity entityAttribute, AtlasVertex vertexA } if (entityAttribute.getRemoveRelationshipAttributes() != null) { - Map appendRelationshipAttributes = processRelationshipAttributesForAttribute(entityAttribute, entityAttribute.getAppendRelationshipAttributes(), context); + Map appendRelationshipAttributes = processRelationshipAttributesForAttribute(entityAttribute, entityAttribute.getRemoveRelationshipAttributes(), context); modelResponseParentEntity.getCopyEntity().setRemoveRelationshipAttributes(appendRelationshipAttributes); context.removeUpdatedWithDeleteRelationshipAttributes(entityAttribute); context.setUpdatedWithRemoveRelationshipAttributes(modelResponseParentEntity.getCopyEntity()); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java index fd432470b4..9ed8844724 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -76,8 +76,6 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati String qualifiedNamePrefix = (String) entity.getAttributes().get(ATLAS_DM_QUALIFIED_NAME_PREFIX); int lastIndex = qualifiedNamePrefix.lastIndexOf("/"); String modelQualifiedName = qualifiedNamePrefix.substring(0, lastIndex); - entity.setAttribute(NAME, qualifiedNamePrefix + "_" + now); - Map attrValues = new HashMap<>(); attrValues.put(QUALIFIED_NAME, modelQualifiedName); From 227bc87682cf30acaf9af6a8c3ee1cc433c9c23b Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 25 Sep 2024 14:58:21 +0530 Subject: [PATCH 43/55] Remove from context --- .../store/graph/v2/EntityMutationContext.java | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java index f6a5df70e8..fe209140ff 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java @@ -30,8 +30,8 @@ public class EntityMutationContext { private final EntityGraphDiscoveryContext context; private final List entitiesCreated = new ArrayList<>(); private final List entitiesUpdated = new ArrayList<>(); - private final List entitiesUpdatedWithAppendRelationshipAttribute = new ArrayList<>(); - private final List entitiesUpdatedWithRemoveRelationshipAttribute = new ArrayList<>(); + private List entitiesUpdatedWithAppendRelationshipAttribute = new ArrayList<>(); + private List entitiesUpdatedWithRemoveRelationshipAttribute = new ArrayList<>(); private final Map entityVsType = new HashMap<>(); private final Map entityVsVertex = new HashMap<>(); private final Map guidAssignments = new HashMap<>(); @@ -59,11 +59,11 @@ public void addCreated(String internalGuid, AtlasEntity entity, AtlasEntityType } } - public void setUpdatedWithRelationshipAttributes(AtlasEntity entity){ + public void setUpdatedWithRelationshipAttributes(AtlasEntity entity) { entitiesUpdatedWithAppendRelationshipAttribute.add(entity); } - public void setUpdatedWithRemoveRelationshipAttributes(AtlasEntity entity){ + public void setUpdatedWithRemoveRelationshipAttributes(AtlasEntity entity) { entitiesUpdatedWithRemoveRelationshipAttribute.add(entity); } @@ -97,12 +97,24 @@ public void removeUpdated(String internalGuid, AtlasEntity entity, AtlasEntityTy } } - public void removeUpdatedWithRelationshipAttributes(AtlasEntity entity){ - entitiesUpdatedWithAppendRelationshipAttribute.remove(entity); + public void removeUpdatedWithRelationshipAttributes(AtlasEntity entity) { + Iterator entities = entitiesUpdatedWithAppendRelationshipAttribute.iterator(); + while (entities.hasNext()) { + String guid = entities.next().getGuid(); + if (guid.equals(entity.getGuid())) { + entities.remove(); + } + } } - public void removeUpdatedWithDeleteRelationshipAttributes(AtlasEntity entity){ - entitiesUpdatedWithRemoveRelationshipAttribute.remove(entity); + public void removeUpdatedWithDeleteRelationshipAttributes(AtlasEntity entity) { + Iterator entities = entitiesUpdatedWithRemoveRelationshipAttribute.iterator(); + while (entities.hasNext()) { + String guid = entities.next().getGuid(); + if (guid.equals(entity.getGuid())) { + entities.remove(); + } + } } public void addEntityToRestore(AtlasVertex vertex) { From f22e1bdbb703ff2efa3ecad0ab66619d43f7c250 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 25 Sep 2024 14:59:34 +0530 Subject: [PATCH 44/55] Add utils --- .../model/AbstractModelPreProcessor.java | 61 ++++++++++++++++--- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index 6482fc4068..ff824f5c2f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -109,17 +109,33 @@ protected void setQualifiedName(AtlasEntity newEntity, AtlasVertex newVertex, Ob AtlasGraphUtilsV2.setEncodedProperty(newVertex, QUALIFIED_NAME, value); } - protected ModelResponse createEntity(String qualifiedName, String entityType, String namespace, EntityMutationContext context) throws AtlasBaseException { + protected void setName(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { + newEntity.setAttribute(NAME, value); + AtlasGraphUtilsV2.setEncodedProperty(newVertex, NAME, value); + } + + protected void setQualifiedNamePrefix(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { + newEntity.setAttribute(ATLAS_DM_QUALIFIED_NAME_PREFIX, value); + AtlasGraphUtilsV2.setEncodedProperty(newVertex, ATLAS_DM_QUALIFIED_NAME_PREFIX, value); + } + + protected void setNamespace(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { + newEntity.setAttribute(ATLAS_DM_NAMESPACE, value); + AtlasGraphUtilsV2.setEncodedProperty(newVertex, ATLAS_DM_NAMESPACE, value); + } + + protected ModelResponse createEntity(String qualifiedName, String name, String entityType, String namespace, EntityMutationContext context) throws AtlasBaseException { String guid = UUID.randomUUID().toString(); AtlasEntity entity = new AtlasEntity(entityType); - entity.setAttribute(NAME, qualifiedName.substring(qualifiedName.lastIndexOf('/') + 1)); + entity.setAttribute(NAME, name); entity.setAttribute(VERSION_PROPERTY_KEY, 0); entity.setAttribute(QUALIFIED_NAME, qualifiedName); entity.setAttribute(ATLAS_DM_NAMESPACE, namespace); entity.setAttribute(ATLAS_DM_BUSINESS_DATE, RequestContext.get().getRequestTime()); entity.setAttribute(ATLAS_DM_SYSTEM_DATE, RequestContext.get().getRequestTime()); if (entityType.equals(ATLAS_DM_ENTITY_TYPE) || entityType.equals(ATLAS_DM_ATTRIBUTE_TYPE)) { - entity.setAttribute(ATLAS_DM_QUALIFIED_NAME_PREFIX, qualifiedName.substring(0, qualifiedName.lastIndexOf('/'))); + String prefix = qualifiedName.substring(0, qualifiedName.indexOf("_")); + entity.setAttribute(ATLAS_DM_QUALIFIED_NAME_PREFIX, prefix); } AtlasVertex versionVertex = entityGraphMapper.createVertexWithGuid(entity, guid); context.getDiscoveryContext().addResolvedGuid(guid, versionVertex); @@ -166,6 +182,8 @@ protected ModelResponse replicateModelVersion(String modelGuid, String modelQual copyAllAttributes(existingModelVersionEntity, copyModelVersion, now); setModelDates(copyModelVersion, copyModelVertex, now); setQualifiedName(copyModelVersion, copyModelVertex, modelQualifiedName + "/" + modelVersion); + setName(copyModelVersion, copyModelVertex, modelVersion); + setNamespace(copyModelVersion, copyModelVertex, dataModel.getEntity().getAttribute(ATLAS_DM_NAMESPACE)); setModelExpiredAtDates(existingModelVersionEntity, existingModelVersionVertex, now); return new ModelResponse(existingModelVersionEntity, copyModelVersion, existingModelVersionVertex, copyModelVertex); } @@ -177,6 +195,9 @@ protected ModelResponse replicateModelEntity(AtlasEntity existingEntity, AtlasVe String entityQualifiedName = entityQualifiedNamePrefix + "_" + epoch; setQualifiedName(copyEntity, copyEntityVertex, entityQualifiedName); setModelDates(copyEntity, copyEntityVertex, epoch); + setName(copyEntity, copyEntityVertex, existingEntity.getAttribute(NAME)); + setNamespace(copyEntity, copyEntityVertex, existingEntity.getAttribute(ATLAS_DM_NAMESPACE)); + setQualifiedNamePrefix(copyEntity, copyEntityVertex, existingEntity.getAttribute(ATLAS_DM_QUALIFIED_NAME_PREFIX)); setModelExpiredAtDates(existingEntity, existingEntityVertex, epoch); return new ModelResponse(existingEntity, copyEntity, existingEntityVertex, copyEntityVertex); } @@ -188,6 +209,9 @@ protected ModelResponse replicateModelAttribute(AtlasEntity existingAttribute, A String attributeQualifiedName = attributeQualifiedNamePrefix + "_" + epoch; setQualifiedName(copyAttributeEntity, copyAttributeVertex, attributeQualifiedName); setModelDates(copyAttributeEntity, copyAttributeVertex, epoch); + setName(copyAttributeEntity, copyAttributeVertex, existingAttribute.getAttribute(NAME)); + setNamespace(copyAttributeEntity, copyAttributeVertex, existingAttribute.getAttribute(ATLAS_DM_NAMESPACE)); + setQualifiedNamePrefix(copyAttributeEntity, copyAttributeVertex, existingAttribute.getAttribute(ATLAS_DM_QUALIFIED_NAME_PREFIX)); setModelExpiredAtDates(existingAttribute, existingAttributeVertex, epoch); return new ModelResponse(existingAttribute, copyAttributeEntity, existingAttributeVertex, copyAttributeVertex); } @@ -318,17 +342,28 @@ protected void copyAllAttributes(AtlasEntity source, AtlasEntity destination, lo String entityType = source.getTypeName(); - copyRelationshipAttributes(source.getRelationshipAttributes(), destination, entityType); - copyRelationshipAttributes(source.getAppendRelationshipAttributes(), destination, entityType); - copyRelationshipAttributes(source.getRemoveRelationshipAttributes(), destination, entityType); + if (MapUtils.isNotEmpty(source.getRelationshipAttributes())) { + Map relationAttributes = copyRelationshipAttributes(source.getRelationshipAttributes(), destination, entityType); + destination.setRelationshipAttributes(relationAttributes); + } + if (MapUtils.isNotEmpty(source.getAppendRelationshipAttributes())) { + Map relationAttributes = copyRelationshipAttributes(source.getAppendRelationshipAttributes(), destination, entityType); + destination.setAppendRelationshipAttributes(relationAttributes); + } + if (MapUtils.isNotEmpty(source.getRemoveRelationshipAttributes())) { + Map relationAttributes = copyRelationshipAttributes(source.getRemoveRelationshipAttributes(), destination, entityType); + destination.setRemoveRelationshipAttributes(relationAttributes); + } + } - private void copyRelationshipAttributes(Map sourceAttributes, AtlasEntity destination, String entityType) { + private Map copyRelationshipAttributes(Map sourceAttributes, AtlasEntity destination, String entityType) { + Map destinationAttributes = new HashMap<>(); + if (MapUtils.isEmpty(sourceAttributes)) { - return; + return destinationAttributes; } - Map destinationAttributes = new HashMap<>(); Set allowedRelations = allowedRelationshipsForEntityType(entityType); for (String attribute : sourceAttributes.keySet()) { @@ -336,7 +371,8 @@ private void copyRelationshipAttributes(Map sourceAttributes, At destinationAttributes.put(attribute, sourceAttributes.get(attribute)); } } - destination.setAttributes(destinationAttributes); + + return destinationAttributes; } public static void replaceAttributes(Map existingAttributes, Map diffAttributes) { @@ -450,6 +486,7 @@ protected ModelResponse updateDMEntity(AtlasEntity entity, AtlasVertex vertex, E String namespace = (String) entity.getAttributes().get(ATLAS_DM_NAMESPACE); modelVersionResponse = createEntity( (modelQualifiedName + "/" + modelVersion), + modelVersion, ATLAS_DM_VERSION_TYPE, namespace, context); @@ -684,8 +721,11 @@ protected ModelResponse updateDMAttribute(AtlasEntity entityAttribute, AtlasVert existingAttributes = (List) modelENtityResponse.getExistingEntity().getAttributes().get("dMAttributes"); } } else { + int lastSlashIndex = entityQualifiedNamePrefix.lastIndexOf("/"); + String entityName = entityQualifiedNamePrefix.substring(lastSlashIndex + 1); modelENtityResponse = createEntity( entityQualifiedNamePrefix + "_" + now, + entityName, ATLAS_DM_ENTITY_TYPE, namespace, context @@ -697,6 +737,7 @@ protected ModelResponse updateDMAttribute(AtlasEntity entityAttribute, AtlasVert if (modelVersionResponse.getCopyEntity() == null) { modelVersionResponse = createEntity( (modelQualifiedName + "/" + modelVersion), + modelVersion, ATLAS_DM_VERSION_TYPE, namespace, context); From f24a0ac554818cdf8d4c86998f53646b43b9f29c Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 25 Sep 2024 15:00:26 +0530 Subject: [PATCH 45/55] remove from context --- .../preprocessor/model/DMAttributePreprocessor.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java index 5d571da955..bd36dbaea6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java @@ -104,8 +104,13 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt existingAttributes = (List) modelENtityResponse.getExistingEntity().getRelationshipAttributes().get("dMAttributes"); } } else { + int lastSlashIndex = entityQualifiedNamePrefix.lastIndexOf("/"); + + // Extract the substring after the last "/" + String name = entityQualifiedNamePrefix.substring(lastSlashIndex + 1); modelENtityResponse = createEntity( entityQualifiedNamePrefix + "_" + now, + name, ATLAS_DM_ENTITY_TYPE, namespace, context @@ -117,6 +122,7 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt if (modelVersionResponse.getCopyEntity() == null) { modelVersionResponse = createEntity( (modelQualifiedName + "/" + modelVersion), + modelVersion, ATLAS_DM_VERSION_TYPE, namespace, context); @@ -160,20 +166,17 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt private void updateDMAttributes(AtlasEntity entityAttribute, AtlasVertex vertexAttribute, EntityMutationContext context) throws AtlasBaseException { ModelResponse modelResponseParentEntity = updateDMAttribute(entityAttribute, vertexAttribute, context); - // case when a mapping is added if (entityAttribute.getAppendRelationshipAttributes() != null) { Map appendRelationshipAttributes = processRelationshipAttributesForAttribute(entityAttribute, entityAttribute.getAppendRelationshipAttributes(), context); - modelResponseParentEntity.getCopyEntity().setAppendRelationshipAttributes(appendRelationshipAttributes); + modelResponseParentEntity.getCopyEntity().setAppendRelationshipAttributes(new HashMap<>(appendRelationshipAttributes)); context.removeUpdatedWithRelationshipAttributes(entityAttribute); - context.setUpdatedWithRelationshipAttributes(modelResponseParentEntity.getCopyEntity()); } if (entityAttribute.getRemoveRelationshipAttributes() != null) { Map appendRelationshipAttributes = processRelationshipAttributesForAttribute(entityAttribute, entityAttribute.getRemoveRelationshipAttributes(), context); modelResponseParentEntity.getCopyEntity().setRemoveRelationshipAttributes(appendRelationshipAttributes); context.removeUpdatedWithDeleteRelationshipAttributes(entityAttribute); - context.setUpdatedWithRemoveRelationshipAttributes(modelResponseParentEntity.getCopyEntity()); } } From ca1188bbe068cc81fa6fb174f2f2e3093c22cfd9 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 25 Sep 2024 15:00:51 +0530 Subject: [PATCH 46/55] update entity name --- .../store/graph/v2/preprocessor/model/DMEntityPreProcessor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java index 9ed8844724..0396a8f701 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -95,6 +95,7 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati String namespace = (String) entity.getAttributes().get(ATLAS_DM_NAMESPACE); modelVersionResponse = createEntity( (modelQualifiedName + "/" + "v1"), + "v1", ATLAS_DM_VERSION_TYPE, namespace, context); From 17c3a6ef4583303ec245339ecc564fb4d8ef086e Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 26 Sep 2024 10:58:32 +0530 Subject: [PATCH 47/55] Improve formatting --- .../v2/preprocessor/model/AbstractModelPreProcessor.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index ff824f5c2f..748898f3b5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -60,7 +60,7 @@ protected static class ModelResponse { private AtlasVertex existingVertex; private AtlasVertex copyVertex; - protected ModelResponse(AtlasEntity existingEntity, AtlasEntity copyEntity, + public ModelResponse(AtlasEntity existingEntity, AtlasEntity copyEntity, AtlasVertex existingVertex, AtlasVertex copyVertex) { this.existingEntity = existingEntity; this.copyEntity = copyEntity; @@ -68,7 +68,7 @@ protected ModelResponse(AtlasEntity existingEntity, AtlasEntity copyEntity, this.copyVertex = copyVertex; } - protected ModelResponse(AtlasEntity copyEntity, AtlasVertex copyVertex) { + public ModelResponse(AtlasEntity copyEntity, AtlasVertex copyVertex) { this.copyEntity = copyEntity; this.copyVertex = copyVertex; } @@ -478,6 +478,10 @@ protected ModelResponse updateDMEntity(AtlasEntity entity, AtlasVertex vertex, E AtlasVertex modelVertex = AtlasGraphUtilsV2.findByUniqueAttributes( typeRegistry.getEntityTypeByName(ATLAS_DM_DATA_MODEL), attrValues); + if (modelVertex == null) { + throw new AtlasBaseException(AtlasErrorCode.DATA_MODEL_NOT_EXIST); + } + String modelGuid = AtlasGraphUtilsV2.getIdFromVertex(modelVertex); ModelResponse modelVersionResponse = replicateModelVersion(modelGuid, modelQualifiedName, now); From 8a4a1cb30d4ea60ab03e094959341d22491a971b Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 26 Sep 2024 10:59:05 +0530 Subject: [PATCH 48/55] Add atlas code --- intg/src/main/java/org/apache/atlas/AtlasErrorCode.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index 55ee1ff1a0..8bee2815be 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -297,7 +297,9 @@ public enum AtlasErrorCode { DATA_MODEL_NOT_EXIST(400, "ATLAS-400-00-117", "DataModel {0} does not exist"), QUALIFIED_NAME_PREFIX_NOT_EXIST(400, "ATLAS-400-00-118", "dMQualifiedNamePrefix is mandatory for DMEntity/DMAttribute"), NO_TYPE_EXISTS_FOR_QUALIFIED_NAME_PREFIX (400,"ATLAS-400-00-119", "No DMEntity/DMAttribute exists for dMQualifiedNamePrefix : {0}"), - NAME_NAMESPACE_NOT_EXIST (400, "ATLAS-400-00-120", "name/namespace are mandatory for DMEntity/DMAttribute"); + NAME_NAMESPACE_NOT_EXIST (400, "ATLAS-400-00-120", "name/namespace are mandatory for DMEntity/DMAttribute"), + QUALIFIED_NAME_PREFIX_OR_TYPE_NOT_FOUND(400, "ATLAS-400-00-121", "qualifiedName/entityType are mandatory"), + INVALID_ENTITY_TYPE(400, "ATLAS-400-00-122", "Invalid entity type"); private String errorCode; private String errorMessage; private Response.Status httpCode; From 93173503183235d697f415e7770638ce6726a0b3 Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 26 Sep 2024 18:30:41 +0530 Subject: [PATCH 49/55] Update abstarctModel Preprocessor --- .../model/AbstractModelPreProcessor.java | 94 ++++++------------- 1 file changed, 27 insertions(+), 67 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index 748898f3b5..0f32da92c1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -7,7 +7,6 @@ import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasRelatedObjectId; import org.apache.atlas.model.instance.AtlasRelationship; -import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; @@ -19,11 +18,8 @@ import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.ListUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.common.Strings; -import org.mockito.internal.util.collections.ListUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,42 +49,6 @@ public AbstractModelPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetr this.atlasRelationshipStore = atlasRelationshipStore; } - protected static class ModelResponse { - - private AtlasEntity existingEntity; - private AtlasEntity copyEntity; - private AtlasVertex existingVertex; - private AtlasVertex copyVertex; - - public ModelResponse(AtlasEntity existingEntity, AtlasEntity copyEntity, - AtlasVertex existingVertex, AtlasVertex copyVertex) { - this.existingEntity = existingEntity; - this.copyEntity = copyEntity; - this.existingVertex = existingVertex; - this.copyVertex = copyVertex; - } - - public ModelResponse(AtlasEntity copyEntity, AtlasVertex copyVertex) { - this.copyEntity = copyEntity; - this.copyVertex = copyVertex; - } - - public AtlasEntity getExistingEntity() { - return existingEntity; - } - - public AtlasEntity getCopyEntity() { - return copyEntity; - } - - public AtlasVertex getExistingVertex() { - return existingVertex; - } - - public AtlasVertex getCopyVertex() { - return copyVertex; - } - } protected void setModelDates(AtlasEntity newEntity, AtlasVertex newVertex, Object value) { newEntity.setAttribute(ATLAS_DM_SYSTEM_DATE, value); @@ -143,7 +103,7 @@ protected ModelResponse createEntity(String qualifiedName, String name, String e return new ModelResponse(entity, versionVertex); } - protected ModelResponse replicateModelVersion(String modelGuid, String modelQualifiedName, long now) throws AtlasBaseException { + public ModelResponse replicateModelVersion(String modelGuid, String modelQualifiedName, long now) throws AtlasBaseException { AtlasEntity.AtlasEntityWithExtInfo dataModel = entityRetriever.toAtlasEntityWithExtInfo(modelGuid, false); List existingModelVersions = (List) dataModel.getEntity().getRelationshipAttributes().get("dMVersions"); @@ -294,7 +254,7 @@ protected void createModelEntityModelAttributeRelation(AtlasVertex entity, Atlas } - protected void createModelModelVersionRelation(String modelGuid, String latestModelVersionGuid) throws AtlasBaseException { + public void createModelModelVersionRelation(String modelGuid, String latestModelVersionGuid) throws AtlasBaseException { AtlasRelationship modelVersionModelRelation = new AtlasRelationship("d_m_data_model_d_m_versions"); modelVersionModelRelation.setStatus(AtlasRelationship.Status.ACTIVE); modelVersionModelRelation.setEnd1( @@ -486,7 +446,7 @@ protected ModelResponse updateDMEntity(AtlasEntity entity, AtlasVertex vertex, E ModelResponse modelVersionResponse = replicateModelVersion(modelGuid, modelQualifiedName, now); // model is not replicated successfully - if (modelVersionResponse.getCopyEntity() == null) { + if (modelVersionResponse.getReplicaEntity() == null) { String namespace = (String) entity.getAttributes().get(ATLAS_DM_NAMESPACE); modelVersionResponse = createEntity( (modelQualifiedName + "/" + modelVersion), @@ -496,14 +456,14 @@ protected ModelResponse updateDMEntity(AtlasEntity entity, AtlasVertex vertex, E context); } - AtlasEntity latestModelVersionEntity = modelVersionResponse.getCopyEntity(); - AtlasVertex latestModelVersionVertex = modelVersionResponse.getCopyVertex(); + AtlasEntity latestModelVersionEntity = modelVersionResponse.getReplicaEntity(); + AtlasVertex latestModelVersionVertex = modelVersionResponse.getReplicaVertex(); AtlasEntity existingVersion = modelVersionResponse.getExistingEntity(); // create entity e1 ---> e1' ModelResponse modelEntityResponse = replicateModelEntity(existingEntity.getEntity(), vertex, qualifiedNamePrefix, now); - AtlasVertex copyEntityVertex = modelEntityResponse.getCopyVertex(); - AtlasEntity copyEntity = modelEntityResponse.getCopyEntity(); + AtlasVertex copyEntityVertex = modelEntityResponse.getReplicaVertex(); + AtlasEntity copyEntity = modelEntityResponse.getReplicaEntity(); applyDiffs(entity, copyEntity, ATLAS_DM_ENTITY_TYPE); unsetExpiredDates(copyEntity, copyEntityVertex); @@ -527,7 +487,7 @@ protected ModelResponse updateDMEntity(AtlasEntity entity, AtlasVertex vertex, E */ // previousEntity and previousModelVersion AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); - AtlasEntityType modelVersionType = typeRegistry.getEntityTypeByName(modelVersionResponse.getCopyEntity().getTypeName()); + AtlasEntityType modelVersionType = typeRegistry.getEntityTypeByName(modelVersionResponse.getReplicaEntity().getTypeName()); context.addCreated(copyEntity.getGuid(), copyEntity, entityType, copyEntityVertex); context.addCreated(latestModelVersionEntity.getGuid(), latestModelVersionEntity, modelVersionType, latestModelVersionVertex); @@ -576,10 +536,10 @@ protected Map processRelationshipAttributesForEntity(AtlasEntity context); //relationAttribute.put("guid", modelResponseRelatedEntity.getCopyEntity()); destMap = new HashMap<>(relationAttribute); - guid = modelResponseRelatedEntity.getCopyEntity().getGuid(); + guid = modelResponseRelatedEntity.getReplicaEntity().getGuid(); destMap.put("guid", guid); //destMap.put(QUALIFIED_NAME, ) - context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); + context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getReplicaVertex()); destList.add(destMap); } appendAttributesDestination.put(attribute, destList); @@ -596,8 +556,8 @@ protected Map processRelationshipAttributesForEntity(AtlasEntity Map destMap = new HashMap<>(attributeList); destMap.put("guid", guid); - guid = modelResponseRelatedEntity.getCopyEntity().getGuid(); - context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); + guid = modelResponseRelatedEntity.getReplicaEntity().getGuid(); + context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getReplicaVertex()); appendAttributesDestination.put(attribute, destMap); } } @@ -647,10 +607,10 @@ protected Map processRelationshipAttributesForAttribute(AtlasEnt context); //relationAttribute.put("guid", modelResponseRelatedEntity.getCopyEntity()); destMap = new HashMap<>(relationAttribute); - guid = modelResponseRelatedEntity.getCopyEntity().getGuid(); + guid = modelResponseRelatedEntity.getReplicaEntity().getGuid(); destMap.put("guid", guid); //destMap.put(QUALIFIED_NAME, ) - context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); + context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getReplicaVertex()); destList.add(destMap); } appendAttributesDestination.put(attribute, destList); @@ -667,8 +627,8 @@ protected Map processRelationshipAttributesForAttribute(AtlasEnt Map destMap = new HashMap<>(attributeList); destMap.put("guid", guid); - guid = modelResponseRelatedEntity.getCopyEntity().getGuid(); - context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getCopyVertex()); + guid = modelResponseRelatedEntity.getReplicaEntity().getGuid(); + context.getDiscoveryContext().addResolvedGuid(guid, modelResponseRelatedEntity.getReplicaVertex()); appendAttributesDestination.put(attribute, destMap); } } @@ -738,7 +698,7 @@ protected ModelResponse updateDMAttribute(AtlasEntity entityAttribute, AtlasVert ModelResponse modelVersionResponse = replicateModelVersion(modelGuid, modelQualifiedName, now); - if (modelVersionResponse.getCopyEntity() == null) { + if (modelVersionResponse.getReplicaEntity() == null) { modelVersionResponse = createEntity( (modelQualifiedName + "/" + modelVersion), modelVersion, @@ -746,8 +706,8 @@ protected ModelResponse updateDMAttribute(AtlasEntity entityAttribute, AtlasVert namespace, context); } - AtlasEntity latestModelVersionEntity = modelVersionResponse.getCopyEntity(); - AtlasVertex latestModelVersionVertex = modelVersionResponse.getCopyVertex(); + AtlasEntity latestModelVersionEntity = modelVersionResponse.getReplicaEntity(); + AtlasVertex latestModelVersionVertex = modelVersionResponse.getReplicaVertex(); List existingEntities = null; @@ -764,8 +724,8 @@ protected ModelResponse updateDMAttribute(AtlasEntity entityAttribute, AtlasVert attributeQualifiedNamePrefix, now); - AtlasVertex copyAttributeVertex = modelAttributeResponse.getCopyVertex(); - AtlasEntity copyAttribute = modelAttributeResponse.getCopyEntity(); + AtlasVertex copyAttributeVertex = modelAttributeResponse.getReplicaVertex(); + AtlasEntity copyAttribute = modelAttributeResponse.getReplicaEntity(); applyDiffs(entityAttribute, copyAttribute, ATLAS_DM_ATTRIBUTE_TYPE); unsetExpiredDates(copyAttribute, copyAttributeVertex); @@ -773,25 +733,25 @@ protected ModelResponse updateDMAttribute(AtlasEntity entityAttribute, AtlasVert createModelModelVersionRelation(modelGuid, latestModelVersionEntity.getGuid()); // create modelVersion-entity relationship [with new entity] - createModelVersionModelEntityRelationship(latestModelVersionVertex, modelENtityResponse.getCopyVertex()); + createModelVersionModelEntityRelationship(latestModelVersionVertex, modelENtityResponse.getReplicaVertex()); // create modelVersion-entity relationship [with existing entities] createModelVersionModelEntityRelationship(latestModelVersionVertex, existingEntities); // create entity - attribute relation [with new attribute] - createModelEntityModelAttributeRelation(modelENtityResponse.getCopyVertex(), copyAttributeVertex); + createModelEntityModelAttributeRelation(modelENtityResponse.getReplicaVertex(), copyAttributeVertex); // create entity - attribute relation [with existing attributes] - createModelEntityModelAttributeRelation(modelENtityResponse.getCopyVertex(), existingAttributes); + createModelEntityModelAttributeRelation(modelENtityResponse.getReplicaVertex(), existingAttributes); AtlasEntityType attributeType = typeRegistry.getEntityTypeByName(entityAttribute.getTypeName()); - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(modelENtityResponse.getCopyEntity().getTypeName()); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(modelENtityResponse.getReplicaEntity().getTypeName()); AtlasEntityType modelVersionType = typeRegistry.getEntityTypeByName(latestModelVersionEntity.getTypeName()); context.addCreated(copyAttribute.getGuid(), copyAttribute, attributeType, copyAttributeVertex); - context.addCreated(modelENtityResponse.getCopyEntity().getGuid(), modelENtityResponse.getCopyEntity(), - entityType, modelENtityResponse.getCopyVertex()); + context.addCreated(modelENtityResponse.getReplicaEntity().getGuid(), modelENtityResponse.getReplicaEntity(), + entityType, modelENtityResponse.getReplicaVertex()); context.addCreated(latestModelVersionEntity.getGuid(), latestModelVersionEntity, modelVersionType, latestModelVersionVertex); From 3bb6e19b72ef8bda9ca2a006cce8c1268234762d Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 26 Sep 2024 18:31:22 +0530 Subject: [PATCH 50/55] fix name --- .../model/DMAttributeAssociationPreprocessor.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributeAssociationPreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributeAssociationPreprocessor.java index 6341e7ff5c..5430d98451 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributeAssociationPreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributeAssociationPreprocessor.java @@ -6,7 +6,6 @@ import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.EntityMutations; -import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; @@ -20,7 +19,6 @@ import java.util.Map; import static org.apache.atlas.repository.Constants.*; -import static org.apache.atlas.repository.Constants.ATLAS_DM_ENTITY_ASSOCIATION_TYPE; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; public class DMAttributeAssociationPreprocessor extends AbstractModelPreProcessor{ @@ -80,8 +78,8 @@ private void updateDMAttributeAssociation(AtlasEntity entity, AtlasVertex vertex long now = RequestContext.get().getRequestTime(); ModelResponse modelResponse = replicateDMAssociation(entity, vertex, now); - AtlasEntity copyEntity = modelResponse.getCopyEntity(); - AtlasVertex copyVertex = modelResponse.getCopyVertex(); + AtlasEntity copyEntity = modelResponse.getReplicaEntity(); + AtlasVertex copyVertex = modelResponse.getReplicaVertex(); applyDiffs(entity, copyEntity, ATLAS_DM_ATTRIBUTE_ASSOCIATION_TYPE); unsetExpiredDates(copyEntity, copyVertex); @@ -89,16 +87,16 @@ private void updateDMAttributeAssociation(AtlasEntity entity, AtlasVertex vertex // case when a mapping is added if (entity.getAppendRelationshipAttributes() != null) { Map appendRelationshipAttributes = processRelationshipAttributesForAttribute(entity, entity.getAppendRelationshipAttributes(), context); - modelResponse.getCopyEntity().setAppendRelationshipAttributes(appendRelationshipAttributes); + modelResponse.getReplicaEntity().setAppendRelationshipAttributes(appendRelationshipAttributes); context.removeUpdatedWithRelationshipAttributes(entity); - context.setUpdatedWithRelationshipAttributes(modelResponse.getCopyEntity()); + context.setUpdatedWithRelationshipAttributes(modelResponse.getReplicaEntity()); } if (entity.getRemoveRelationshipAttributes() != null) { Map appendRelationshipAttributes = processRelationshipAttributesForAttribute(entity, entity.getRemoveRelationshipAttributes(), context); - modelResponse.getCopyEntity().setRemoveRelationshipAttributes(appendRelationshipAttributes); + modelResponse.getReplicaEntity().setRemoveRelationshipAttributes(appendRelationshipAttributes); context.removeUpdatedWithDeleteRelationshipAttributes(entity); - context.setUpdatedWithRemoveRelationshipAttributes(modelResponse.getCopyEntity()); + context.setUpdatedWithRemoveRelationshipAttributes(modelResponse.getReplicaEntity()); } } } From 05667f097b84850eb569054aeef73fda2776baea Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 26 Sep 2024 18:58:08 +0530 Subject: [PATCH 51/55] Update name --- .../model/DMAttributePreprocessor.java | 21 +++++++++---------- .../DMEntityAssociationPreProcessor.java | 12 +++++------ .../model/DMEntityPreProcessor.java | 16 +++++++------- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java index bd36dbaea6..c933f02519 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMAttributePreprocessor.java @@ -13,7 +13,6 @@ import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; -import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; @@ -119,7 +118,7 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt List existingEntities = null; ModelResponse modelVersionResponse = replicateModelVersion(modelGuid, modelQualifiedName, now); - if (modelVersionResponse.getCopyEntity() == null) { + if (modelVersionResponse.getReplicaEntity() == null) { modelVersionResponse = createEntity( (modelQualifiedName + "/" + modelVersion), modelVersion, @@ -127,15 +126,15 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt namespace, context); } - AtlasEntity latestModelVersionEntity = modelVersionResponse.getCopyEntity(); - AtlasVertex latestModelVersionVertex = modelVersionResponse.getCopyVertex(); + AtlasEntity latestModelVersionEntity = modelVersionResponse.getReplicaEntity(); + AtlasVertex latestModelVersionVertex = modelVersionResponse.getReplicaVertex(); // model --- modelVersion relation createModelModelVersionRelation(modelGuid, latestModelVersionEntity.getGuid()); // modelVersion --- entity relation - createModelVersionModelEntityRelationship(latestModelVersionVertex, modelENtityResponse.getCopyVertex()); + createModelVersionModelEntityRelationship(latestModelVersionVertex, modelENtityResponse.getReplicaVertex()); // modelVersion --- entitiesOfExistingModelVersion if (modelVersionResponse.getExistingEntity() != null && modelVersionResponse.getExistingEntity().getRelationshipAttributes() != null) { @@ -144,16 +143,16 @@ private void createDMAttribute(AtlasEntity entityAttribute, AtlasVertex vertexAt } // entity --- attributes of existingEntity relation - createModelEntityModelAttributeRelation(modelENtityResponse.getCopyVertex(), existingAttributes); + createModelEntityModelAttributeRelation(modelENtityResponse.getReplicaVertex(), existingAttributes); // latest entity ---- new attribute relation - createModelEntityModelAttributeRelation(modelENtityResponse.getCopyVertex(), vertexAttribute); + createModelEntityModelAttributeRelation(modelENtityResponse.getReplicaVertex(), vertexAttribute); context.addCreated(latestModelVersionEntity.getGuid(), latestModelVersionEntity, typeRegistry.getEntityTypeByName(ATLAS_DM_VERSION_TYPE), latestModelVersionVertex); - context.addCreated(modelENtityResponse.getCopyEntity().getGuid(), modelENtityResponse.getCopyEntity(), - typeRegistry.getEntityTypeByName(ATLAS_DM_ENTITY_TYPE), modelENtityResponse.getCopyVertex()); + context.addCreated(modelENtityResponse.getReplicaEntity().getGuid(), modelENtityResponse.getReplicaEntity(), + typeRegistry.getEntityTypeByName(ATLAS_DM_ENTITY_TYPE), modelENtityResponse.getReplicaVertex()); // resolve references context.getDiscoveryContext(). @@ -169,13 +168,13 @@ private void updateDMAttributes(AtlasEntity entityAttribute, AtlasVertex vertexA // case when a mapping is added if (entityAttribute.getAppendRelationshipAttributes() != null) { Map appendRelationshipAttributes = processRelationshipAttributesForAttribute(entityAttribute, entityAttribute.getAppendRelationshipAttributes(), context); - modelResponseParentEntity.getCopyEntity().setAppendRelationshipAttributes(new HashMap<>(appendRelationshipAttributes)); + modelResponseParentEntity.getReplicaEntity().setAppendRelationshipAttributes(new HashMap<>(appendRelationshipAttributes)); context.removeUpdatedWithRelationshipAttributes(entityAttribute); } if (entityAttribute.getRemoveRelationshipAttributes() != null) { Map appendRelationshipAttributes = processRelationshipAttributesForAttribute(entityAttribute, entityAttribute.getRemoveRelationshipAttributes(), context); - modelResponseParentEntity.getCopyEntity().setRemoveRelationshipAttributes(appendRelationshipAttributes); + modelResponseParentEntity.getReplicaEntity().setRemoveRelationshipAttributes(appendRelationshipAttributes); context.removeUpdatedWithDeleteRelationshipAttributes(entityAttribute); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityAssociationPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityAssociationPreProcessor.java index a7d8f6a189..1e549b4090 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityAssociationPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityAssociationPreProcessor.java @@ -79,8 +79,8 @@ private void updateDMEntityAssociation(AtlasEntity entity, AtlasVertex vertex, E long now = RequestContext.get().getRequestTime(); ModelResponse modelResponse = replicateDMAssociation(entity, vertex, now); - AtlasEntity copyEntity = modelResponse.getCopyEntity(); - AtlasVertex copyVertex = modelResponse.getCopyVertex(); + AtlasEntity copyEntity = modelResponse.getReplicaEntity(); + AtlasVertex copyVertex = modelResponse.getReplicaVertex(); applyDiffs(entity, copyEntity, ATLAS_DM_ENTITY_ASSOCIATION_TYPE); unsetExpiredDates(copyEntity, copyVertex); @@ -88,16 +88,16 @@ private void updateDMEntityAssociation(AtlasEntity entity, AtlasVertex vertex, E // case when a mapping is added if (entity.getAppendRelationshipAttributes() != null) { Map appendRelationshipAttributes = processRelationshipAttributesForEntity(entity, entity.getAppendRelationshipAttributes(), context); - modelResponse.getCopyEntity().setAppendRelationshipAttributes(appendRelationshipAttributes); + modelResponse.getReplicaEntity().setAppendRelationshipAttributes(appendRelationshipAttributes); context.removeUpdatedWithRelationshipAttributes(entity); - context.setUpdatedWithRelationshipAttributes(modelResponse.getCopyEntity()); + context.setUpdatedWithRelationshipAttributes(modelResponse.getReplicaEntity()); } if (entity.getRemoveRelationshipAttributes() != null) { Map appendRelationshipAttributes = processRelationshipAttributesForEntity(entity, entity.getRemoveRelationshipAttributes(), context); - modelResponse.getCopyEntity().setRemoveRelationshipAttributes(appendRelationshipAttributes); + modelResponse.getReplicaEntity().setRemoveRelationshipAttributes(appendRelationshipAttributes); context.removeUpdatedWithDeleteRelationshipAttributes(entity); - context.setUpdatedWithRemoveRelationshipAttributes(modelResponse.getCopyEntity()); + context.setUpdatedWithRemoveRelationshipAttributes(modelResponse.getReplicaEntity()); } } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java index 0396a8f701..3c7149dfbd 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/DMEntityPreProcessor.java @@ -18,12 +18,14 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; import java.util.*; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; +@Component public class DMEntityPreProcessor extends AbstractModelPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(DMEntityPreProcessor.class); @@ -91,7 +93,7 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati ModelResponse modelVersionResponse = replicateModelVersion(modelGuid, modelQualifiedName, now); // create modelVersion - if (modelVersionResponse.getCopyEntity() == null) { + if (modelVersionResponse.getReplicaEntity() == null) { String namespace = (String) entity.getAttributes().get(ATLAS_DM_NAMESPACE); modelVersionResponse = createEntity( (modelQualifiedName + "/" + "v1"), @@ -101,8 +103,8 @@ private void createDMEntity(AtlasEntity entity, AtlasVertex vertex, EntityMutati context); } - AtlasEntity latestModelVersionEntity = modelVersionResponse.getCopyEntity(); - AtlasVertex latestModelVersionVertex = modelVersionResponse.getCopyVertex(); + AtlasEntity latestModelVersionEntity = modelVersionResponse.getReplicaEntity(); + AtlasVertex latestModelVersionVertex = modelVersionResponse.getReplicaVertex(); // model --- modelVersion relation @@ -137,16 +139,16 @@ private void updateDMEntities(AtlasEntity entity, AtlasVertex vertex, EntityMuta // case when a mapping is added if (entity.getAppendRelationshipAttributes() != null) { Map appendRelationshipAttributes = processRelationshipAttributesForEntity(entity, entity.getAppendRelationshipAttributes(), context); - modelResponseParentEntity.getCopyEntity().setAppendRelationshipAttributes(appendRelationshipAttributes); + modelResponseParentEntity.getReplicaEntity().setAppendRelationshipAttributes(appendRelationshipAttributes); context.removeUpdatedWithRelationshipAttributes(entity); - context.setUpdatedWithRelationshipAttributes(modelResponseParentEntity.getCopyEntity()); + context.setUpdatedWithRelationshipAttributes(modelResponseParentEntity.getReplicaEntity()); } if (entity.getRemoveRelationshipAttributes() != null) { Map appendRelationshipAttributes = processRelationshipAttributesForEntity(entity, entity.getRemoveRelationshipAttributes(), context); - modelResponseParentEntity.getCopyEntity().setRemoveRelationshipAttributes(appendRelationshipAttributes); + modelResponseParentEntity.getReplicaEntity().setRemoveRelationshipAttributes(appendRelationshipAttributes); context.removeUpdatedWithDeleteRelationshipAttributes(entity); - context.setUpdatedWithRemoveRelationshipAttributes(modelResponseParentEntity.getCopyEntity()); + context.setUpdatedWithRemoveRelationshipAttributes(modelResponseParentEntity.getReplicaEntity()); } } } From 7edfbfaf7422f9d57053c07dcbd8ee457cc6b746 Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 26 Sep 2024 18:58:51 +0530 Subject: [PATCH 52/55] ModelResponse API --- .../v2/preprocessor/model/ModelResponse.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/ModelResponse.java diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/ModelResponse.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/ModelResponse.java new file mode 100644 index 0000000000..d6fd46afc3 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/ModelResponse.java @@ -0,0 +1,41 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.model; + +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.repository.graphdb.AtlasVertex; + +public class ModelResponse { + + private AtlasEntity existingEntity; + private AtlasEntity replicaEntity; + private AtlasVertex existingVertex; + private AtlasVertex replicaVertex; + + public ModelResponse(AtlasEntity existingEntity, AtlasEntity replicaEntity, + AtlasVertex existingVertex, AtlasVertex replicaVertex) { + this.existingEntity = existingEntity; + this.replicaEntity = replicaEntity; + this.existingVertex = existingVertex; + this.replicaVertex = replicaVertex; + } + + public ModelResponse(AtlasEntity replicaEntity, AtlasVertex replicaVertex) { + this.replicaEntity = replicaEntity; + this.replicaVertex = replicaVertex; + } + + public AtlasEntity getExistingEntity() { + return existingEntity; + } + + public AtlasEntity getReplicaEntity() { + return replicaEntity; + } + + public AtlasVertex getExistingVertex() { + return existingVertex; + } + + public AtlasVertex getReplicaVertex() { + return replicaVertex; + } +} From ba436849fac7b40be95b0567e8488c564b4bc6fb Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 27 Sep 2024 07:47:37 +0530 Subject: [PATCH 53/55] update method signaure --- .../v2/preprocessor/model/AbstractModelPreProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java index 0f32da92c1..2b39630150 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/model/AbstractModelPreProcessor.java @@ -148,7 +148,7 @@ public ModelResponse replicateModelVersion(String modelGuid, String modelQualifi return new ModelResponse(existingModelVersionEntity, copyModelVersion, existingModelVersionVertex, copyModelVertex); } - protected ModelResponse replicateModelEntity(AtlasEntity existingEntity, AtlasVertex existingEntityVertex, String entityQualifiedNamePrefix, long epoch) throws AtlasBaseException { + public ModelResponse replicateModelEntity(AtlasEntity existingEntity, AtlasVertex existingEntityVertex, String entityQualifiedNamePrefix, long epoch) throws AtlasBaseException { AtlasVertex copyEntityVertex = entityGraphMapper.createVertex(existingEntity); AtlasEntity copyEntity = entityRetriever.toAtlasEntity(copyEntityVertex); copyAllAttributes(existingEntity, copyEntity, epoch); @@ -176,7 +176,7 @@ protected ModelResponse replicateModelAttribute(AtlasEntity existingAttribute, A return new ModelResponse(existingAttribute, copyAttributeEntity, existingAttributeVertex, copyAttributeVertex); } - protected void createModelVersionModelEntityRelationship(AtlasVertex modelVersionVertex, + public void createModelVersionModelEntityRelationship(AtlasVertex modelVersionVertex, AtlasVertex modelEntityVertex) throws AtlasBaseException { AtlasRelationship modelVersionEntityRelation = new AtlasRelationship("d_m_version_d_m_entities"); modelVersionEntityRelation.setStatus(AtlasRelationship.Status.ACTIVE); From d345c492f8d8c730a1cebb6bcb8e8ecb535b0f81 Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 27 Sep 2024 07:48:35 +0530 Subject: [PATCH 54/55] Expose REST API for models --- .../org/apache/atlas/web/rest/ModelREST.java | 233 ++++++++++++++++-- 1 file changed, 208 insertions(+), 25 deletions(-) diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/ModelREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/ModelREST.java index d962b670e3..206b552ec6 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/ModelREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/ModelREST.java @@ -1,9 +1,5 @@ package org.apache.atlas.web.rest; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; @@ -12,7 +8,19 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.IndexSearchParams; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext; +import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.repository.store.graph.v2.preprocessor.model.AbstractModelPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.model.DMEntityPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.model.ModelResponse; import org.apache.atlas.searchlog.SearchLoggingManagement; +import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.atlas.utils.AtlasPerfTracer; @@ -30,10 +38,9 @@ import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import java.util.Arrays; -import java.util.Base64; -import java.util.HashSet; -import java.util.Set; +import java.util.*; + +import static org.apache.atlas.repository.Constants.*; @Path("model") @Singleton @@ -49,31 +56,34 @@ public class ModelREST { @Context private HttpServletRequest httpServletRequest; private final boolean enableSearchLogging; - private final AtlasTypeRegistry typeRegistry; + private final AbstractModelPreProcessor entityPreProcessor; private final AtlasDiscoveryService discoveryService; private final SearchLoggingManagement loggerManagement; + private final EntityGraphMapper entityGraphMapper; + private final EntityGraphRetriever graphRetriever; + + //private final Entity private static final String INDEXSEARCH_TAG_NAME = "indexsearch"; private static final Set TRACKING_UTM_TAGS = new HashSet<>(Arrays.asList("ui_main_list", "ui_popup_searchbar")); private static final String UTM_TAG_FROM_PRODUCT = "project_webapp"; @Inject - public ModelREST(AtlasTypeRegistry typeRegistry, AtlasDiscoveryService discoveryService, - SearchLoggingManagement loggerManagement) { + public ModelREST(AtlasTypeRegistry typeRegistry, AtlasDiscoveryService discoveryService, SearchLoggingManagement loggerManagement, EntityGraphMapper entityGraphMapper, EntityGraphRetriever graphRetriever, DMEntityPreProcessor entityPreProcessor) { this.typeRegistry = typeRegistry; this.discoveryService = discoveryService; this.loggerManagement = loggerManagement; + this.entityGraphMapper = entityGraphMapper; + this.graphRetriever = graphRetriever; + this.entityPreProcessor = entityPreProcessor; this.enableSearchLogging = AtlasConfiguration.ENABLE_SEARCH_LOGGER.getBoolean(); } @Path("/search") @POST @Timed - public AtlasSearchResult dataSearch(@QueryParam("namespace") String namespace, - @QueryParam("businessDate") String businessDate, - @QueryParam("systemDate") String systemDate, - @Context HttpServletRequest servletRequest, IndexSearchParams parameters) throws AtlasBaseException { + public AtlasSearchResult dataSearch(@QueryParam("namespace") String namespace, @QueryParam("businessDate") String businessDate, @QueryParam("systemDate") String systemDate, @Context HttpServletRequest servletRequest, IndexSearchParams parameters) throws AtlasBaseException { Servlets.validateQueryParamLength("namespace", namespace); Servlets.validateQueryParamLength("businessDate", businessDate); @@ -87,10 +97,7 @@ public AtlasSearchResult dataSearch(@QueryParam("namespace") String namespace, parameters = parameters == null ? new IndexSearchParams() : parameters; - String queryStringUsingFiltersAndUserDSL = ModelUtil.createQueryStringUsingFiltersAndUserDSL(namespace, - businessDate, - systemDate, - parameters.getQuery()); + String queryStringUsingFiltersAndUserDSL = ModelUtil.createQueryStringUsingFiltersAndUserDSL(namespace, businessDate, systemDate, parameters.getQuery()); if (StringUtils.isEmpty(queryStringUsingFiltersAndUserDSL)) { AtlasBaseException abe = new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Invalid model search query"); @@ -107,19 +114,15 @@ public AtlasSearchResult dataSearch(@QueryParam("namespace") String namespace, if (result == null) { return null; } - - return result; } catch (AtlasBaseException abe) { - if (enableSearchLogging && parameters.isSaveSearchLog() - ) { + if (enableSearchLogging && parameters.isSaveSearchLog()) { // logSearchLog(parameters, servletRequest, abe, System.currentTimeMillis() - startTime); } throw abe; } catch (Exception e) { AtlasBaseException abe = new AtlasBaseException(e.getMessage(), e.getCause()); - if (enableSearchLogging && parameters.isSaveSearchLog() - ) { + if (enableSearchLogging && parameters.isSaveSearchLog()) { //logSearchLog(parameters, servletRequest, abe, System.currentTimeMillis() - startTime); } throw abe; @@ -145,4 +148,184 @@ public AtlasSearchResult dataSearch(@QueryParam("namespace") String namespace, } } + @DELETE + @Path("/entity") + @Timed + public EntityMutationResponse deleteByQualifiedNamePrefix(@QueryParam("qualifiedNamePrefix") String qualifiedNamePrefix, + @QueryParam("entityType") String entityType, + @Context HttpServletRequest servletRequest) throws AtlasBaseException { + + Servlets.validateQueryParamLength("qualifiedNamePrefix", qualifiedNamePrefix); + Servlets.validateQueryParamLength("entityType", entityType); + AtlasPerfTracer perf = null; + + EntityGraphDiscoveryContext graphDiscoveryContext = new EntityGraphDiscoveryContext(typeRegistry, null); + EntityMutationContext entityMutationContext = new EntityMutationContext(graphDiscoveryContext); + + try { + if (StringUtils.isEmpty(qualifiedNamePrefix) || StringUtils.isEmpty(entityType)) { + throw new AtlasBaseException(AtlasErrorCode.QUALIFIED_NAME_PREFIX_NOT_EXIST); + } + + // check with chris how the label look like + // accordingly capitalize letters + AtlasEntityType atlasEntityType = typeRegistry.getEntityTypeByName(entityType); + + if (atlasEntityType == null) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_ENTITY_TYPE); + } + + AtlasVertex latestEntityVertex = AtlasGraphUtilsV2.findLatestEntityAttributeVerticesByType(entityType, qualifiedNamePrefix); + + if (latestEntityVertex == null) { + throw new AtlasBaseException(AtlasErrorCode.NO_TYPE_EXISTS_FOR_QUALIFIED_NAME_PREFIX, qualifiedNamePrefix); + } + + String modelQualifiedName; + + if (entityType.equals(ATLAS_DM_ENTITY_TYPE)) { + int lastIndex = qualifiedNamePrefix.lastIndexOf("/"); + modelQualifiedName = qualifiedNamePrefix.substring(0, lastIndex); + String entityGuid = AtlasGraphUtilsV2.getIdFromVertex(latestEntityVertex); + replicateModelVersionAndExcludeEntity(modelQualifiedName, entityGuid, entityMutationContext); + + } else if (entityType.equals(ATLAS_DM_ATTRIBUTE_TYPE)) { + int lastIndex = qualifiedNamePrefix.lastIndexOf("/"); + String entityQualifiedNamePrefix = qualifiedNamePrefix.substring(0, lastIndex); + String attributeGuid = AtlasGraphUtilsV2.getIdFromVertex(latestEntityVertex); + replicateModelVersionAndEntityAndExcludeAttribute(entityQualifiedNamePrefix, + attributeGuid, "", entityMutationContext); + } else { + throw new AtlasBaseException(AtlasErrorCode.INVALID_ENTITY_TYPE); + } + return entityGraphMapper.mapAttributesAndClassifications(entityMutationContext, + false, false, false, false); + } finally { + AtlasPerfTracer.log(perf); + } + } + + private void replicateModelVersionAndEntityAndExcludeAttribute(final String entityQualifiedNamePrefix, String deleteAttributeGuid, String deleteEntityGuid, EntityMutationContext entityMutationContext) throws AtlasBaseException { + int lastIndex = entityQualifiedNamePrefix.lastIndexOf("/"); + String modelQualifiedName = entityQualifiedNamePrefix.substring(0, lastIndex); + + // get entity + // replicate entity + AtlasVertex latestEntityVertex = AtlasGraphUtilsV2.findLatestEntityAttributeVerticesByType(ATLAS_DM_ENTITY_TYPE, entityQualifiedNamePrefix); + AtlasEntity latestEntity = graphRetriever.toAtlasEntity(latestEntityVertex); + + if (latestEntityVertex == null) { + throw new AtlasBaseException(AtlasErrorCode.DATA_ENTITY_NOT_EXIST); + } + + long now = RequestContext.get().getRequestTime(); + + ModelResponse modelResponse = entityPreProcessor.replicateModelEntity(latestEntity, + latestEntityVertex, entityQualifiedNamePrefix, now); + + AtlasEntity replicaEntity = modelResponse.getReplicaEntity(); + AtlasVertex replicaVertex = modelResponse.getReplicaVertex(); + + // exclude attribute from entity + Map relationshipAttributes = excludeEntityFromRelationshipAttribute(deleteAttributeGuid, + replicaEntity.getRelationshipAttributes()); + replicaEntity.setRelationshipAttributes(relationshipAttributes); + + + //replicate modelVersion + ModelResponse modelVersionResponse = replicateModelVersionAndExcludeEntity( + modelQualifiedName, "", entityMutationContext); + + // create entity-modelVersion relationship + entityPreProcessor.createModelVersionModelEntityRelationship( + modelVersionResponse.getReplicaVertex(), + replicaVertex); + + entityMutationContext.addCreated(replicaEntity.getGuid(), + replicaEntity, + typeRegistry.getEntityTypeByName(ATLAS_DM_ENTITY_TYPE), + replicaVertex); + + } + + private ModelResponse replicateModelVersionAndExcludeEntity(final String modelQualifiedName, String deleteEntityGuid, EntityMutationContext entityMutationContext) throws AtlasBaseException { + Map attrValues = new HashMap<>(); + attrValues.put(QUALIFIED_NAME, modelQualifiedName); + + AtlasVertex modelVertex = AtlasGraphUtilsV2.findByUniqueAttributes( + typeRegistry.getEntityTypeByName(ATLAS_DM_DATA_MODEL), attrValues); + + if (modelVertex == null) { + throw new AtlasBaseException(AtlasErrorCode.DATA_MODEL_NOT_EXIST); + } + + String modelGuid = AtlasGraphUtilsV2.getIdFromVertex(modelVertex); + + long now = RequestContext.get().getRequestTime(); + + ModelResponse modelResponse = entityPreProcessor.replicateModelVersion(modelGuid, modelQualifiedName, now); + AtlasEntity replicaModelVersionEntity = modelResponse.getReplicaEntity(); + AtlasVertex replicaModelVersionVertex = modelResponse.getReplicaVertex(); + String modelVersionGuid = replicaModelVersionEntity.getGuid(); + + Map relationshipAttributes = excludeEntityFromRelationshipAttribute(deleteEntityGuid, + replicaModelVersionEntity.getRelationshipAttributes()); + replicaModelVersionEntity.setRelationshipAttributes(relationshipAttributes); + entityPreProcessor.createModelModelVersionRelation(modelGuid, modelVersionGuid); + + entityMutationContext.addCreated(modelVersionGuid, replicaModelVersionEntity, + typeRegistry.getEntityTypeByName(ATLAS_DM_VERSION_TYPE), replicaModelVersionVertex); + + entityMutationContext.getDiscoveryContext().addResolvedGuid(modelGuid, modelVertex); + return modelResponse; + } + + private Map excludeEntityFromRelationshipAttribute(String entityGuid, Map relationshipAttributes) throws AtlasBaseException { + if (StringUtils.isEmpty(entityGuid)) { + return relationshipAttributes; + } + Map appendAttributesDestination = new HashMap<>(); + if (relationshipAttributes != null) { + Map appendAttributesSource = (Map) relationshipAttributes; + + String guid = ""; + + for (String attribute : appendAttributesSource.keySet()) { + + if (appendAttributesSource.get(attribute) instanceof List) { + + List> destList = new ArrayList<>(); + Map destMap = null; + + List> attributeList = (List>) appendAttributesSource.get(attribute); + + for (Map relationAttribute : attributeList) { + guid = (String) relationAttribute.get("guid"); + + if (guid.equals(entityGuid)) { + continue; + } + + destMap = new HashMap<>(relationAttribute); + destList.add(destMap); + } + appendAttributesDestination.put(attribute, destList); + } else { + if (appendAttributesSource.get(attribute) instanceof Map) { + LinkedHashMap attributeList = (LinkedHashMap) appendAttributesSource.get(attribute); + guid = (String) attributeList.get("guid"); + + // update end2 + if (guid.equals(entityGuid)) { + continue; + } + + Map destMap = new HashMap<>(attributeList); + appendAttributesDestination.put(attribute, destMap); + } + } + } + } + return appendAttributesDestination; + } } \ No newline at end of file From 467a647f890b496c3b3d6242be0dd922a65abb2a Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 27 Sep 2024 08:35:02 +0530 Subject: [PATCH 55/55] add safe check --- .../store/graph/v2/AtlasEntityStoreV2.java | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 25be261f11..a7b85635d3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1589,21 +1589,34 @@ private void executePreProcessor(EntityMutationContext context) throws AtlasBase } } + List copyOfAppendRelationshipAttributes = new ArrayList<>(context.getUpdatedEntitiesForAppendRelationshipAttribute()); for (AtlasEntity entity : copyOfAppendRelationshipAttributes) { entityType = context.getType(entity.getGuid()); - preProcessors = getPreProcessor(entityType.getTypeName()); - for (PreProcessor processor : preProcessors) { - processor.processAttributes(entity, context, UPDATE); + if( entityType.getTypeName().equals(ATLAS_DM_ENTITY_TYPE) || + entityType.getTypeName().equals(ATLAS_DM_ATTRIBUTE_TYPE) || + entityType.getTypeName().equals(ATLAS_DM_ENTITY_ASSOCIATION_TYPE) || + entity.getTypeName().equals(ATLAS_DM_ATTRIBUTE_ASSOCIATION_TYPE) + ){ + preProcessors = getPreProcessor(entityType.getTypeName()); + for (PreProcessor processor : preProcessors) { + processor.processAttributes(entity, context, UPDATE); + } } } List copyOfRemoveRelationshipAttributes = new ArrayList<>(context.getEntitiesUpdatedWithRemoveRelationshipAttribute()); for (AtlasEntity entity : copyOfRemoveRelationshipAttributes) { entityType = context.getType(entity.getGuid()); - preProcessors = getPreProcessor(entityType.getTypeName()); - for (PreProcessor processor : preProcessors) { - processor.processAttributes(entity, context, UPDATE); + if( entityType.getTypeName().equals(ATLAS_DM_ENTITY_TYPE) + || entityType.getTypeName().equals(ATLAS_DM_ATTRIBUTE_TYPE) || + entityType.getTypeName().equals(ATLAS_DM_ENTITY_ASSOCIATION_TYPE) || + entity.getTypeName().equals(ATLAS_DM_ATTRIBUTE_ASSOCIATION_TYPE) + ){ + preProcessors = getPreProcessor(entityType.getTypeName()); + for (PreProcessor processor : preProcessors) { + processor.processAttributes(entity, context, UPDATE); + } } } }