From 9d40667deb1d395d8acd382ff2969ae43bc21d0d Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 27 Feb 2025 22:04:27 +0530 Subject: [PATCH 01/11] Fix empty typename issue for attributes --- .../java/org/apache/atlas/repository/graph/GraphHelper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 73f9140e68..1be097c98a 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -2071,7 +2071,7 @@ private static Set parseLabelsString(String labels) { * @param vertex entity vertex * @return Iterator of children edges */ - public static Iterator getOnlyActiveEdges(AtlasVertex vertex, AtlasEdgeDirection direction) throws AtlasBaseException { + public Iterator getOnlyActiveEdges(AtlasVertex vertex, AtlasEdgeDirection direction) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("GraphHelper.getOnlyActiveEdges"); try { @@ -2109,7 +2109,7 @@ public Set> retrieveEdgeLabelsAndTypeName return new AbstractMap.SimpleEntry<>(labelStr, typeNameStr); }) - .filter(entry -> !entry.getKey().isEmpty() && !entry.getValue().isEmpty()) + .filter(entry -> !entry.getKey().isEmpty()) .distinct() .collect(Collectors.toSet()); From 923f9da0fd602082b5a4b5699f2e7ea35576f879 Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 27 Feb 2025 22:07:52 +0530 Subject: [PATCH 02/11] Do branch based deployment --- .github/workflows/maven.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 4afe869921..d99b44d45e 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -27,6 +27,7 @@ on: - development - master - lineageondemand + - hotfixindexsearch jobs: build: From 2297ec3e7b7e9492695ae4502ac9637a6bb39e9a Mon Sep 17 00:00:00 2001 From: aarshi Date: Sun, 2 Mar 2025 23:27:10 +0530 Subject: [PATCH 03/11] code cleanup --- .../atlas/repository/graph/GraphHelper.java | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 1be097c98a..e4215cba44 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -2065,30 +2065,6 @@ private static Set parseLabelsString(String labels) { return ret; } - - /** - * Get all the active edges - * @param vertex entity vertex - * @return Iterator of children edges - */ - public Iterator getOnlyActiveEdges(AtlasVertex vertex, AtlasEdgeDirection direction) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("GraphHelper.getOnlyActiveEdges"); - - try { - return vertex.query() - .direction(direction) - .has(STATE_PROPERTY_KEY, ACTIVE_STATE_VALUE) - .edges() - .iterator(); - } catch (Exception e) { - LOG.error("Error while getting active edges of vertex", e); - throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e); - } - finally { - RequestContext.get().endMetricRecord(metricRecorder); - } - } - public Set> retrieveEdgeLabelsAndTypeName(AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("GraphHelper.retrieveEdgeLabelsAndTypeName"); From 6b8211c4cf3c8c9f3f9d380d87e9fb0feecb373a Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 3 Mar 2025 17:30:43 +0530 Subject: [PATCH 04/11] Retrieve Policy using prefetch flow --- .../store/graph/v2/EntityGraphRetriever.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index c94b73a9aa..5204b9224e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1132,12 +1132,24 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, && typeName.equals("S3Bucket"); boolean shouldPrefetch = RequestContext.get().isInvokedByIndexSearch() - && !isPolicyAttribute(attributes) && !isS3Bucket && AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION.getBoolean(); + // remove isPolicyAttribute from shouldPrefetch check + // prefetch properties for policies + // if there is some exception in fetching properties, + // then we will fetch properties again without prefetch + if (shouldPrefetch) { - return mapVertexToAtlasEntityHeaderWithPrefetch(entityVertex, attributes); + try { + return mapVertexToAtlasEntityHeaderWithPrefetch(entityVertex, attributes); + } catch (AtlasBaseException e) { + if (isPolicyAttribute(attributes)) { + LOG.error("Error fetching properties for entity vertex: {}. Retrying without prefetch", entityVertex.getId(), e); + return mapVertexToAtlasEntityHeaderWithoutPrefetch(entityVertex, attributes); + } + throw e; + } } else { return mapVertexToAtlasEntityHeaderWithoutPrefetch(entityVertex, attributes); } From 23f06ae4086db10b4d013be6c7e83404128aa96b Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 4 Mar 2025 00:58:44 +0530 Subject: [PATCH 05/11] check conditions --- .../store/graph/v2/EntityGraphRetriever.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index c94b73a9aa..aebdf48719 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -285,6 +285,7 @@ public AtlasObjectId toAtlasObjectId(AtlasVertex entityVertex) throws AtlasBaseE AtlasObjectId ret = null; String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); + boolean enableJanusOptimisation = AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION.getBoolean(); if (entityType != null) { Map uniqueAttributes = new HashMap<>(); @@ -300,11 +301,21 @@ public AtlasObjectId toAtlasObjectId(AtlasVertex entityVertex) throws AtlasBaseE Map attributes = new HashMap<>(); Set relationAttributes = RequestContext.get().getRelationAttrsForSearch(); if (CollectionUtils.isNotEmpty(relationAttributes)) { + Map referenceVertexProperties= null; + if (enableJanusOptimisation){ + referenceVertexProperties = preloadProperties(entityVertex, entityType, relationAttributes); + } for (String attributeName : relationAttributes) { AtlasAttribute attribute = entityType.getAttribute(attributeName); if (attribute != null && !uniqueAttributes.containsKey(attributeName)) { - Object attrValue = getVertexAttribute(entityVertex, attribute); + Object attrValue= null; + if (enableJanusOptimisation) { +attrValue = getVertexAttributePreFetchCache(entityVertex, attribute, referenceVertexProperties); + }else { + attrValue = getVertexAttribute(entityVertex, attribute); + } + if (attrValue != null) { attributes.put(attribute.getName(), attrValue); } From edda0a3e1bc6747fe842f7399bf752a8dee71f7f Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 4 Mar 2025 13:41:23 +0530 Subject: [PATCH 06/11] Extend cassandra optimisation to relations and policies --- .../org/apache/atlas/AtlasConfiguration.java | 1 + .../store/graph/v2/EntityGraphRetriever.java | 17 +++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 8cba25f04d..caaa2c602a 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -122,6 +122,7 @@ public enum AtlasConfiguration { ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION("atlas.indexsearch.enable.janus.optimization", false), + ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION_FOR_RELATIONS("atlas.indexsearch.enable.janus.optimization_for_relations", false), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), DELTA_BASED_REFRESH_ENABLED("atlas.authorizer.enable.delta_based_refresh", false), diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 477aeacb1c..247348a7fe 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -285,7 +285,7 @@ public AtlasObjectId toAtlasObjectId(AtlasVertex entityVertex) throws AtlasBaseE AtlasObjectId ret = null; String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); - boolean enableJanusOptimisation = AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION.getBoolean(); + boolean enableJanusOptimisation = AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION_FOR_RELATIONS.getBoolean(); if (entityType != null) { Map uniqueAttributes = new HashMap<>(); @@ -301,19 +301,19 @@ public AtlasObjectId toAtlasObjectId(AtlasVertex entityVertex) throws AtlasBaseE Map attributes = new HashMap<>(); Set relationAttributes = RequestContext.get().getRelationAttrsForSearch(); if (CollectionUtils.isNotEmpty(relationAttributes)) { - Map referenceVertexProperties= null; - if (enableJanusOptimisation){ - referenceVertexProperties = preloadProperties(entityVertex, entityType, relationAttributes); + Map referenceVertexProperties = null; + if (enableJanusOptimisation) { + referenceVertexProperties = preloadProperties(entityVertex, entityType, relationAttributes); } for (String attributeName : relationAttributes) { AtlasAttribute attribute = entityType.getAttribute(attributeName); if (attribute != null && !uniqueAttributes.containsKey(attributeName)) { - Object attrValue= null; + Object attrValue = null; if (enableJanusOptimisation) { -attrValue = getVertexAttributePreFetchCache(entityVertex, attribute, referenceVertexProperties); - }else { - attrValue = getVertexAttribute(entityVertex, attribute); + attrValue = getVertexAttributePreFetchCache(entityVertex, attribute, referenceVertexProperties); + } else { + attrValue = getVertexAttribute(entityVertex, attribute); } if (attrValue != null) { @@ -1156,6 +1156,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, return mapVertexToAtlasEntityHeaderWithPrefetch(entityVertex, attributes); } catch (AtlasBaseException e) { if (isPolicyAttribute(attributes)) { + RequestContext.get().endMetricRecord(RequestContext.get().startMetricRecord("policiesPrefetchFailed")); LOG.error("Error fetching properties for entity vertex: {}. Retrying without prefetch", entityVertex.getId(), e); return mapVertexToAtlasEntityHeaderWithoutPrefetch(entityVertex, attributes); } From 41ea093c4d82ae39af963cbfb27f2405bbf8b70b Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 4 Mar 2025 13:43:08 +0530 Subject: [PATCH 07/11] Do branch based deployment --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 96094e5b14..8384fd3a2f 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -27,7 +27,7 @@ on: - development - master - lineageondemand - - hotfixindexsearch + - cassandrapoliciesoptimisation jobs: build: From ed824b2a7ae7e520a69c663708efbcc1abffd75e Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 4 Mar 2025 14:03:16 +0530 Subject: [PATCH 08/11] fix config name --- intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index caaa2c602a..2cd7b422ba 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -122,7 +122,7 @@ public enum AtlasConfiguration { ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION("atlas.indexsearch.enable.janus.optimization", false), - ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION_FOR_RELATIONS("atlas.indexsearch.enable.janus.optimization_for_relations", false), + ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION_FOR_RELATIONS("atlas.indexsearch.enable.janus.optimization.for.relationship", false), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), DELTA_BASED_REFRESH_ENABLED("atlas.authorizer.enable.delta_based_refresh", false), From 6a0810bf7579ba67f60b9318ff0b065106f99555 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Thu, 6 Mar 2025 01:36:22 +0530 Subject: [PATCH 09/11] preload properties just once for a vertex --- .../store/graph/v2/EntityGraphRetriever.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 247348a7fe..da356580c6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -286,10 +286,13 @@ public AtlasObjectId toAtlasObjectId(AtlasVertex entityVertex) throws AtlasBaseE String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); boolean enableJanusOptimisation = AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION_FOR_RELATIONS.getBoolean(); - + Map referenceVertexProperties = null; if (entityType != null) { Map uniqueAttributes = new HashMap<>(); - + Set relationAttributes = RequestContext.get().getRelationAttrsForSearch(); + if (enableJanusOptimisation) { + referenceVertexProperties = preloadProperties(entityVertex, entityType, relationAttributes); + } for (AtlasAttribute attribute : entityType.getUniqAttributes().values()) { Object attrValue = getVertexAttribute(entityVertex, attribute); @@ -299,12 +302,7 @@ public AtlasObjectId toAtlasObjectId(AtlasVertex entityVertex) throws AtlasBaseE } Map attributes = new HashMap<>(); - Set relationAttributes = RequestContext.get().getRelationAttrsForSearch(); if (CollectionUtils.isNotEmpty(relationAttributes)) { - Map referenceVertexProperties = null; - if (enableJanusOptimisation) { - referenceVertexProperties = preloadProperties(entityVertex, entityType, relationAttributes); - } for (String attributeName : relationAttributes) { AtlasAttribute attribute = entityType.getAttribute(attributeName); if (attribute != null From 94c8f4d55f377ec4c93b9687be9ccbe73cdb7e5c Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Thu, 6 Mar 2025 14:44:15 +0530 Subject: [PATCH 10/11] don't have to fetch edge label for non-root vertices --- .../store/graph/v2/EntityGraphRetriever.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index da356580c6..fa1c855cef 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -291,10 +291,11 @@ public AtlasObjectId toAtlasObjectId(AtlasVertex entityVertex) throws AtlasBaseE Map uniqueAttributes = new HashMap<>(); Set relationAttributes = RequestContext.get().getRelationAttrsForSearch(); if (enableJanusOptimisation) { - referenceVertexProperties = preloadProperties(entityVertex, entityType, relationAttributes); + //don't fetch edge labels for a relation attribute + referenceVertexProperties = preloadProperties(entityVertex, entityType, relationAttributes, false); } for (AtlasAttribute attribute : entityType.getUniqAttributes().values()) { - Object attrValue = getVertexAttribute(entityVertex, attribute); + Object attrValue = getVertexAttributePreFetchCache(entityVertex, attribute, referenceVertexProperties); if (attrValue != null) { uniqueAttributes.put(attribute.getName(), attrValue); @@ -1011,7 +1012,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex) return mapVertexToAtlasEntityHeader(entityVertex, Collections.emptySet()); } - private Map preloadProperties(AtlasVertex entityVertex, AtlasEntityType entityType, Set attributes) throws AtlasBaseException { + private Map preloadProperties(AtlasVertex entityVertex, AtlasEntityType entityType, Set attributes, boolean fetchEdgeLabels) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("preloadProperties"); try { @@ -1027,7 +1028,12 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt Map> relationshipsLookup = fetchEdgeNames(entityType); // Fetch edges in both directions - retrieveEdgeLabels(entityVertex, attributes, relationshipsLookup, propertiesMap); + + // if the vertex in scope is root then call below otherwise skip + // we don't support relation attributes of a relation + if (fetchEdgeLabels) { + retrieveEdgeLabels(entityVertex, attributes, relationshipsLookup, propertiesMap); + } // Iterate through the resulting VertexProperty objects while (traversal.hasNext()) { @@ -1257,7 +1263,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithPrefetch(AtlasVertex e //pre-fetching the properties String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); //properties.get returns null AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); // this is not costly - Map properties = preloadProperties(entityVertex, entityType, attributes); + Map properties = preloadProperties(entityVertex, entityType, attributes, true); String guid = (String) properties.get(Constants.GUID_PROPERTY_KEY); From f63491ce43c5cfe2303bc3b51a6c0e5ef18dd7b0 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Thu, 6 Mar 2025 18:05:48 +0530 Subject: [PATCH 11/11] call relationshipLookup only the fetchEdgeLabels condition meets --- .../repository/store/graph/v2/EntityGraphRetriever.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index fa1c855cef..1dd35ed8c6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1024,14 +1024,12 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt // Execute the traversal to fetch properties Iterator> traversal = ((AtlasJanusVertex)entityVertex).getWrappedElement().properties(); - // retrieve all the valid relationships for this entityType - Map> relationshipsLookup = fetchEdgeNames(entityType); - // Fetch edges in both directions - // if the vertex in scope is root then call below otherwise skip // we don't support relation attributes of a relation if (fetchEdgeLabels) { + // retrieve all the valid relationships for this entityType + Map> relationshipsLookup = fetchEdgeNames(entityType); retrieveEdgeLabels(entityVertex, attributes, relationshipsLookup, propertiesMap); }