Merge pull request #3967 from atlanhq/ns/fix/policy-last-updated

DG-1986 | Use policy update time as last refresh time
atlanhq · Jan 20, 2025 · 2ccaa16 · 2ccaa16
2 parents 9d4c07d + 49f8eae
commit 2ccaa16
Show file tree

Hide file tree

Showing 5 changed files with 11 additions and 6 deletions.
diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java
@@ -324,6 +324,7 @@ public void setPolicies(ServicePolicies policies) {
 						if (defaultSvcPolicies == null) {
 							LOG.error("Could not get default Service Policies. Keeping old policy-engine! This is a FATAL error as the old policy-engine is null!");
 							isNewEngineNeeded = false;
+							throw new RuntimeException("PolicyRefresher("+policies.getServiceName()+").setPolicies: fetched service policies contains no policies or delta and current policy engine is null");
 						} else {
 							defaultSvcPolicies.setPolicyVersion(policies.getPolicyVersion());
 							policies = defaultSvcPolicies;
@@ -419,6 +420,7 @@ public void setPolicies(ServicePolicies policies) {
 					if (this.refresher != null) {
 						this.refresher.saveToCache(usePolicyDeltas ? servicePolicies : policies);
 					}
+					LOG.info("New RangerPolicyEngine created with policy count:"+ (usePolicyDeltas? servicePolicies.getPolicies().size() : policies.getPolicies().size()));
 				}
 
 			} else {

diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java
@@ -295,7 +295,7 @@ private void loadPolicy() {
 				lastUpdatedTimeInMillis = -1;
 			}
 		} catch (Exception excp) {
-			LOG.error("Encountered unexpected exception!!!!!!!!!!!", excp);
+			LOG.error("Encountered unexpected exception!!!!!!!!!!! Message:" + excp.getMessage() + "Stacktrace: " + excp.getStackTrace().toString(), excp);
 		}
 
 		RangerPerfTracer.log(perf);

diff --git a/...s-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/...s-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java
@@ -174,6 +174,8 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map<String, EntityAu
 
                 ArrayList<String> policyGuids = new ArrayList<>(policyChanges.keySet());
                 List<AtlasEntityHeader> allAtlasPolicies = getAtlasPolicies(serviceName, POLICY_BATCH_SIZE, policyGuids);
+                Date latestUpdateTime = allAtlasPolicies.stream().map(AtlasEntityHeader::getUpdateTime).max(Date::compareTo).orElse(null);
+                servicePolicies.setPolicyUpdateTime(latestUpdateTime);
 
                 List<AtlasEntityHeader> atlasServicePolicies = allAtlasPolicies.stream().filter(x -> serviceName.equals(x.getAttribute(ATTR_POLICY_SERVICE_NAME))).collect(Collectors.toList());
                 List<RangerPolicyDelta> policiesDelta = getRangerPolicyDelta(service, policyChanges, atlasServicePolicies);

diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
@@ -2085,6 +2085,8 @@ private EntityMutationResponse deleteVertices(Collection<AtlasVertex> deletionCa
 
             MetricRecorder metric = RequestContext.get().startMetricRecord("filterCategoryVertices");
             for (AtlasVertex vertex : deletionCandidates) {
+                updateModificationMetadata(vertex);
+
                 String typeName = getTypeName(vertex);
 
                 List<PreProcessor> preProcessors = getPreProcessor(typeName);

diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java
@@ -158,13 +158,12 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s
             ServicePolicies ret;
             if (usePolicyDelta) {
                 List<EntityAuditEventV2> auditEvents = getPolicyAuditLogs(serviceName, lastUpdatedTime);
-                long lastEventTime = auditEvents.isEmpty() ? 0 : auditEvents.get(auditEvents.size() - 1).getCreated();
-                LOG.info("PolicyDelta: serviceName={}, lastUpdatedTime={}, audit events found={}", serviceName, lastEventTime, auditEvents.size());
+                LOG.info("PolicyDelta: serviceName={}, lastUpdatedTime={}, audit events found={}", serviceName, lastUpdatedTime, auditEvents.size());
                 if (auditEvents.isEmpty()) {
                     return null;
                 }
                 Map<String, EntityAuditEventV2.EntityAuditActionV2> policyChanges = policyTransformer.createPolicyChangeMap(serviceName, auditEvents);
-                ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges, lastEventTime);
+                ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges, lastUpdatedTime);
             } else {
                 if (!isPolicyUpdated(serviceName, lastUpdatedTime)) {
                     return null;
@@ -196,12 +195,12 @@ private List<EntityAuditEventV2> getPolicyAuditLogs(String serviceName, long las
         mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch)));
 
         lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime;
-        mustClauseList.add(getMap("range", getMap("created", getMap("gt", lastUpdatedTime))));
+        mustClauseList.add(getMap("range", getMap("timestamp", getMap("gt", lastUpdatedTime))));
 
         dsl.put("query", getMap("bool", getMap("must", mustClauseList)));
 
         List<Map<String, Object>> sortClause = new ArrayList<>();
-        sortClause.add(getMap("created", getMap("order", "asc")));
+        sortClause.add(getMap("timestamp", getMap("order", "asc")));
         dsl.put("sort", sortClause);
 
         int from = 0;